Added flake8 test to travis and applied flake8 rules to codebase

.travis.yml

@@ -24,3 +24,23 @@ branches:
   except:
     - stable
     - release-v2.0
+
+
+stages:
+  - Flake8
+  - test
+
+jobs:
+  include:
+    - stage: Flake8
+      env: Flake8=True
+      install:
+        - bash bin/travis-build.bash
+        - pip install flake8==3.5.0
+        - pip install pycodestyle==2.3.0
+      script:
+        - flake8 --version
+        # stop the build if there are Python syntax errors or undefined names
+        - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan,ckanext-harvest
+        # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
+        - flake8 . --count --max-line-length=127 --statistics --exclude ckan,ckanext-harvest

bin/ckan_pycsw.py

@@ -10,10 +10,15 @@
 import pycsw.config
 import pycsw.admin
 
+import os
+import argparse
+from ConfigParser import SafeConfigParser
+
 logging.basicConfig(format='%(message)s', level=logging.INFO)
 
 log = logging.getLogger(__name__)
 
+
 def setup_db(pycsw_config):
     """Setup database tables and indexes"""
 
@@ -28,9 +33,9 @@ def setup_db(pycsw_config):
     ]
 
     pycsw.admin.setup_db(database,
-        table_name, '',
-        create_plpythonu_functions=False,
-        extra_columns=ckan_columns)
+                         table_name, '',
+                         create_plpythonu_functions=False,
+                         extra_columns=ckan_columns)
 
 
 def set_keywords(pycsw_config_file, pycsw_config, ckan_url, limit=20):
@@ -63,7 +68,8 @@ def load(pycsw_config, ckan_url):
 
     log.info('Started gathering CKAN datasets identifiers: {0}'.format(str(datetime.datetime.now())))
 
-    query = 'api/search/dataset?qjson={"fl":"id,metadata_modified,extras_harvest_object_id,extras_metadata_source", "q":"harvest_object_id:[\\"\\" TO *]", "limit":1000, "start":%s}'
+    query = 'api/search/dataset?qjson={"fl":"id,metadata_modified,extras_harvest_object_id,' \
+            'extras_metadata_source", "q":"harvest_object_id:[\\"\\" TO *]", "limit":1000, "start":%s}'
 
     start = 0
 
@@ -75,7 +81,7 @@ def load(pycsw_config, ckan_url):
         response = requests.get(url)
         listing = response.json()
         if not isinstance(listing, dict):
-            raise RuntimeError, 'Wrong API response: %s' % listing
+            raise RuntimeError('Wrong API response: %s' % listing)
         results = listing.get('results')
         if not results:
             break
@@ -111,8 +117,7 @@ def load(pycsw_config, ckan_url):
     for ckan_id in deleted:
         try:
             repo.session.begin()
-            repo.session.query(repo.dataset.ckan_id).filter_by(
-            ckan_id=ckan_id).delete()
+            repo.session.query(repo.dataset.ckan_id).filter_by(ckan_id=ckan_id).delete()
             log.info('Deleted %s' % ckan_id)
             repo.session.commit()
         except Exception, err:
@@ -137,17 +142,16 @@ def load(pycsw_config, ckan_url):
         if not record:
             continue
         update_dict = dict([(getattr(repo.dataset, key),
-        getattr(record, key)) \
-        for key in record.__dict__.keys() if key != '_sa_instance_state'])
+                            getattr(record, key))
+                            for key in record.__dict__.keys() if key != '_sa_instance_state'])
         try:
             repo.session.begin()
-            repo.session.query(repo.dataset).filter_by(
-            ckan_id=ckan_id).update(update_dict)
+            repo.session.query(repo.dataset).filter_by(ckan_id=ckan_id).update(update_dict)
             repo.session.commit()
             log.info('Changed %s' % ckan_id)
         except Exception, err:
             repo.session.rollback()
-            raise RuntimeError, 'ERROR: %s' % str(err)
+            raise RuntimeError('ERROR: %s' % str(err))
 
 
 def clear(pycsw_config):
@@ -192,7 +196,7 @@ def get_record(context, repo, ckan_url, ckan_id, ckan_info):
     return record
 
 
-usage='''
+usage = '''
 Manages the CKAN-pycsw integration
 
     python ckan-pycsw.py setup [-p]
@@ -219,6 +223,7 @@ def get_record(context, repo, ckan_url, ckan_id, ckan_info):
 
 '''
 
+
 def _load_config(file_path):
     abs_path = os.path.abspath(file_path)
     if not os.path.exists(abs_path):
@@ -230,25 +235,18 @@ def _load_config(file_path):
     return config
 
 
-
-import os
-import argparse
-from ConfigParser import SafeConfigParser
-
 if __name__ == '__main__':
-    parser = argparse.ArgumentParser(
-        description='\n'.split(usage)[0],
-        usage=usage)
-    parser.add_argument('command',
-         help='Command to perform')
+    parser = argparse.ArgumentParser(description='\n'.split(usage)[0],
+                                     usage=usage)
+    parser.add_argument('command', help='Command to perform')
 
     parser.add_argument('-p', '--pycsw_config',
-         action='store', default='default.cfg',
-         help='pycsw config file to use.')
+                        action='store', default='default.cfg',
+                        help='pycsw config file to use.')
 
     parser.add_argument('-u', '--ckan_url',
-         action='store',
-         help='CKAN instance to import the datasets from.')
+                        action='store',
+                        help='CKAN instance to import the datasets from.')
 
     if len(sys.argv) <= 1:
         parser.print_usage()

ckanext/spatial/commands/__init__.py

@@ -3,4 +3,4 @@
     pkg_resources.declare_namespace(__name__)
 except ImportError:
     import pkgutil
-    __path__ = pkgutil.extend_path(__path__, __name__)
+    __path__ = pkgutil.extend_path(__path__, __name__)

ckanext/spatial/commands/csw.py

@@ -5,6 +5,8 @@
 
 from paste import script
 log = logging.getLogger(__name__)
+
+
 class Pycsw(script.command.Command):
     '''Manages the CKAN-pycsw integration
 
@@ -35,10 +37,14 @@ class Pycsw(script.command.Command):
     '''
 
     parser = script.command.Command.standard_parser(verbose=True)
-    parser.add_option('-p', '--pycsw-config', dest='pycsw_config',
-            default='default.cfg', help='pycsw config file to use.')
-    parser.add_option('-u', '--ckan-url', dest='ckan_url',
-            default='http://localhost', help='CKAN instance to import the datasets from.')
+    parser.add_option('-p', '--pycsw-config',
+                      dest='pycsw_config',
+                      default='default.cfg',
+                      help='pycsw config file to use.')
+    parser.add_option('-u', '--ckan-url',
+                      dest='ckan_url',
+                      default='http://localhost',
+                      help='CKAN instance to import the datasets from.')
 
     summary = __doc__.split('\n')[0]
     usage = __doc__

ckanext/spatial/commands/spatial.py

@@ -1,13 +1,12 @@
 import sys
-import re
-from pprint import pprint
 import logging
 
 from ckan.lib.cli import CkanCommand
 from ckan.lib.helpers import json
 from ckanext.spatial.lib import save_package_extent
 log = logging.getLogger(__name__)
 
+
 class Spatial(CkanCommand):
     '''Performs spatially related operations.
 
@@ -20,7 +19,7 @@ class Spatial(CkanCommand):
         spatial extents
             Creates or updates the extent geometry column for datasets with
             an extent defined in the 'spatial' extra.
-      
+
     The commands should be run from the ckanext-spatial directory and expect
     a development.ini file to be present. Most of the time you will
     specify the config explicitly though::
@@ -31,7 +30,7 @@ class Spatial(CkanCommand):
 
     summary = __doc__.split('\n')[0]
     usage = __doc__
-    max_args = 2 
+    max_args = 2
     min_args = 0
 
     def command(self):
@@ -43,7 +42,7 @@ def command(self):
             sys.exit(1)
         cmd = self.args[0]
         if cmd == 'initdb':
-            self.initdb()    
+            self.initdb()
         elif cmd == 'extents':
             self.update_extents()
         else:
@@ -56,16 +55,16 @@ def initdb(self):
             srid = None
 
         from ckanext.spatial.model import setup as db_setup
-        
+
         db_setup(srid)
 
         print 'DB tables created'
 
     def update_extents(self):
-        from ckan.model import PackageExtra, Package, Session
-        conn = Session.connection()
-        packages = [extra.package \
-                    for extra in \
+        from ckan.model import PackageExtra, Session
+        Session.connection()
+        packages = [extra.package
+                    for extra in
                     Session.query(PackageExtra).filter(PackageExtra.key == 'spatial').all()]
 
         errors = []
@@ -77,21 +76,19 @@ def update_extents(self):
                 geometry = json.loads(value)
 
                 count += 1
-            except ValueError,e:
-                errors.append(u'Package %s - Error decoding JSON object: %s' % (package.id,str(e)))
-            except TypeError,e:
-                errors.append(u'Package %s - Error decoding JSON object: %s' % (package.id,str(e)))
+            except ValueError, e:
+                errors.append(u'Package %s - Error decoding JSON object: %s' % (package.id, str(e)))
+            except TypeError, e:
+                errors.append(u'Package %s - Error decoding JSON object: %s' % (package.id, str(e)))
 
-            save_package_extent(package.id,geometry)
-
+            save_package_extent(package.id, geometry)
 
         Session.commit()
-        
+
         if errors:
             msg = 'Errors were found:\n%s' % '\n'.join(errors)
             print msg
 
-        msg = "Done. Extents generated for %i out of %i packages" % (count,len(packages))
+        msg = "Done. Extents generated for %i out of %i packages" % (count, len(packages))
 
         print msg
-

ckanext/spatial/commands/validation.py

@@ -1,5 +1,4 @@
 import sys
-import re
 import os
 from pprint import pprint
 import logging
@@ -10,6 +9,7 @@
 
 log = logging.getLogger(__name__)
 
+
 class Validation(CkanCommand):
     '''Validation commands
 
@@ -21,7 +21,7 @@ class Validation(CkanCommand):
         validation report-csv <filename>.csv
             Performs validation on all the harvested metadata in the db and
             writes a report in CSV format to the given filepath.
-      
+
         validation file <filename>.xml
             Performs validation on the given metadata file.
     '''
@@ -49,7 +49,6 @@ def command(self):
 
     def report(self):
         from ckan import model
-        from ckanext.harvest.model import HarvestObject
         from ckanext.spatial.lib.reports import validation_report
 
         if len(self.args) >= 2:
@@ -92,7 +91,7 @@ def validate_file(self):
             print 'ERROR: Unicode Error reading file \'%s\': %s' % \
                   (metadata_filepath, e)
             sys.exit(1)
-            #import pdb; pdb.set_trace()
+            # import pdb; pdb.set_trace()
         xml = etree.fromstring(xml_string)
 
         # XML validation
@@ -102,11 +101,11 @@ def validate_file(self):
         if valid:
             try:
                 iso_document = ISODocument(xml_string)
-                iso_values = iso_document.read_values()
+                iso_document.read_values()
             except Exception, e:
                 valid = False
                 errors.append('CKAN exception reading values from ISODocument: %s' % e)
-        
+
         print '***************'
         print 'Summary'
         print '***************'

ckanext/spatial/controllers/__init__.py

@@ -4,4 +4,3 @@
 except ImportError:
     import pkgutil
     __path__ = pkgutil.extend_path(__path__, __name__)
-

ckanext/spatial/controllers/api.py

@@ -26,7 +26,7 @@ def spatial_query(self):
         error_400_msg = \
             'Please provide a suitable bbox parameter [minx,miny,maxx,maxy]'
 
-        if not 'bbox' in request.params:
+        if 'bbox' not in request.params:
             abort(400, error_400_msg)
 
         bbox = validate_bbox(request.params['bbox'])
@@ -127,7 +127,7 @@ def display_xml_original(self, id):
         response.headers['Content-Type'] = 'application/xml; charset=utf-8'
         response.headers['Content-Length'] = len(content)
 
-        if not '<?xml' in content.split('\n')[0]:
+        if '<?xml' not in content.split('\n')[0]:
             content = u'<?xml version="1.0" encoding="UTF-8"?>\n' + content
         return content.encode('utf-8')
 

ckanext/spatial/geoalchemy_common.py

@@ -13,7 +13,7 @@
 if toolkit.check_ckan_version(min_version='2.3'):
     # CKAN >= 2.3, use GeoAlchemy2
 
-    from geoalchemy2.elements import WKTElement
+    from geoalchemy2.elements import WKTElement # noqa
     from geoalchemy2 import Geometry
     from sqlalchemy import func
     ST_Transform = func.ST_Transform
@@ -23,7 +23,7 @@
 else:
     # CKAN < 2.3, use GeoAlchemy
 
-    from geoalchemy import WKTSpatialElement as WKTElement
+    from geoalchemy import WKTSpatialElement as WKTElement # noqa
     from geoalchemy import functions
     ST_Transform = functions.transform
     ST_Equals = functions.equals

ckanext/spatial/harvesters/__init__.py

@@ -9,3 +9,5 @@
 from ckanext.spatial.harvesters.csw import CSWHarvester
 from ckanext.spatial.harvesters.waf import WAFHarvester
 from ckanext.spatial.harvesters.doc import DocHarvester
+
+__all__ = ['CSWHarvester', 'WAFHarvester', 'DocHarvester']