Upgrade ckan 2.10

.github/workflows/create-pr.yaml

@@ -6,6 +6,9 @@ on:
     workflows: [ "Build and push images" ]
     types:
       - completed
+  push:
+    branches:
+      - main-2.10
 
 jobs:
   create_pr:

.github/workflows/ghcr.yaml

@@ -10,7 +10,7 @@ on:
         default: false
   push:
     branches:
-      - main
+      - main-2.10
   schedule:
     - cron: '0 3 * * 0'
 

.gitignore

@@ -48,3 +48,4 @@ docs/_build/
 environment.sh
 .DS_Store
 .env*
+docker/src

ckan.ini

@@ -0,0 +1,338 @@
+#
+# CKAN configuration
+#
+# These are some of the configuration options available for your CKAN
+# instance. Check the documentation in 'doc/configuration.rst' or at the
+# following URL for a description of what they do and the full list of
+# available options:
+#
+# http://docs.ckan.org/en/latest/maintaining/configuration.html
+#
+# The %(here)s variable will be replaced with the parent directory of this file
+#
+[DEFAULT]
+
+## Default settings ############################################################
+## This enables the `Flask-DebugToolbar
+## <https://flask-debugtoolbar.readthedocs.io/>`_ in the web interface, makes
+## Webassets serve unminified JS and CSS files, and enables CKAN templates'
+## debugging features.
+## 
+## You will need to ensure the ``Flask-DebugToolbar`` python package is installed,
+## by activating your ckan virtual environment and then running::
+## 
+##     pip install -r /usr/lib/ckan/default/src/ckan/dev-requirements.txt
+## 
+## If you are running CKAN on Apache, you must change the WSGI
+## configuration to run a single process of CKAN. Otherwise
+## the execution will fail with: ``AssertionError: The EvalException
+## middleware is not usable in a multi-process environment``. Eg. change::
+## 
+##   WSGIDaemonProcess ckan_default display-name=ckan_default processes=2 threads=15
+##   to
+##   WSGIDaemonProcess ckan_default display-name=ckan_default threads=15
+## 
+## .. warning:: This option should be set to ``False`` for a public site.
+##    With debug mode enabled, a visitor to your site could execute malicious
+##    commands.
+debug = false
+
+[app:main]
+
+## General settings ############################################################
+use = egg:ckan
+ckan.legacy_route_mappings = {}
+config.mode = default
+
+## Development settings ########################################################
+ckan.devserver.host = localhost
+ckan.devserver.port = 5000
+ckan.devserver.threaded = false
+ckan.devserver.multiprocess = 1
+ckan.devserver.watch_patterns = 
+ckan.devserver.ssl_cert = 
+ckan.devserver.ssl_key = 
+
+## Session settings ############################################################
+ckan.user.last_active_interval = 600
+cache_dir = /tmp/%(ckan.site_id)s
+beaker.session.key = ckan
+beaker.session.secret = 9EZiPwkeS+cZpqb0VDWrN+Q0M
+beaker.session.auto = false
+beaker.session.cookie_expires = false
+# beaker.session.cookie_domain = .example.com
+beaker.session.save_accessed_time = true
+beaker.session.secure = false
+beaker.session.timeout = 600
+
+## Database settings ###########################################################
+sqlalchemy.url = postgresql://ckan:ckan@db/ckan
+sqlalchemy.pool_pre_ping = true
+# sqlalchemy.<OPTION> = 
+
+## Site Settings ###############################################################
+ckan.site_url = http://localhost:5003
+apikey_header_name = X-CKAN-API-Key
+ckan.cache_expires = 0
+ckan.cache_enabled = false
+ckan.mimetype_guess = file_ext
+ckan.static_max_age = 3600
+ckan.tracking_enabled = false
+ckan.valid_url_schemes = http https ftp
+ckan.requests.timeout = 5
+ckan.hide_version = false
+ckan.redirect_to_login_if_not_authorized = true
+
+## Authorization Settings ######################################################
+ckan.auth.anon_create_dataset = false
+ckan.auth.create_unowned_dataset = false
+ckan.auth.create_dataset_if_not_in_organization = false
+ckan.auth.user_create_groups = false
+ckan.auth.user_create_organizations = true
+ckan.auth.user_delete_groups = false
+ckan.auth.user_delete_organizations = true
+ckan.auth.create_user_via_api = true
+ckan.auth.create_user_via_web = true
+ckan.auth.roles_that_cascade_to_sub_groups = admin
+ckan.auth.public_user_details = true
+ckan.auth.public_activity_stream_detail = false
+ckan.auth.allow_dataset_collaborators = false
+ckan.auth.allow_admin_collaborators = false
+ckan.auth.allow_collaborators_to_change_owner_org = false
+ckan.auth.create_default_api_keys = false
+ckan.auth.login_view = user.login
+ckan.auth.reveal_private_datasets = false
+ckan.auth.enable_cookie_auth_in_api = true
+ckan.auth.route_after_login = dashboard.datasets
+
+## User Account Creation Setting
+; ckan.valid_email_regexes = .gov.uk$ .nhs.uk$ .nhs.net$ .ac.uk$ .os.uk$ .mod.uk$ .police.uk$ .bl.uk$
+
+## CSRF Protection #############################################################
+WTF_CSRF_ENABLED = true
+WTF_CSRF_CHECK_DEFAULT = true
+WTF_CSRF_SECRET_KEY = "wtf csrf secret key"
+WTF_CSRF_METHODS = POST PUT PATCH DELETE
+WTF_CSRF_FIELD_NAME = _csrf_token
+WTF_CSRF_HEADERS = X-CSRFToken X-CSRF-Token
+WTF_CSRF_TIME_LIMIT = 3600
+WTF_CSRF_SSL_STRICT = true
+WTF_I18N_ENABLED = true
+ckan.csrf_protection.ignore_extensions = true
+
+## Flask-Login Remember me cookie settings #####################################
+REMEMBER_COOKIE_NAME = remember_token
+REMEMBER_COOKIE_DURATION = 31536000
+# REMEMBER_COOKIE_DOMAIN = .example.com
+REMEMBER_COOKIE_PATH = /
+REMEMBER_COOKIE_SECURE = false
+REMEMBER_COOKIE_HTTPONLY = true
+REMEMBER_COOKIE_REFRESH_EACH_REQUEST = false
+REMEMBER_COOKIE_SAMESITE = None
+
+## API Token Settings ##########################################################
+api_token.nbytes = 32
+api_token.jwt.encode.secret = string:%(beaker.session.secret)s
+api_token.jwt.decode.secret = string:%(beaker.session.secret)s
+api_token.jwt.algorithm = HS256
+
+## Search Settings #############################################################
+ckan.site_id = dgu
+solr_url = http://solr:8983/solr/ckan
+solr_user = 
+solr_password = 
+ckan.search.remove_deleted_packages = true
+ckan.search.solr_commit = true
+ckan.search.show_all_types = dataset
+ckan.search.default_include_private = true
+ckan.search.default_package_sort = score desc, metadata_modified desc
+search.facets = organization groups tags res_format license_id
+search.facets.limit = 50
+search.facets.default = 10
+ckan.extra_resource_fields = 
+ckan.search.rows_max = 1000
+ckan.group_and_organization_list_max = 1000
+ckan.group_and_organization_list_all_fields_max = 25
+solr_timeout = 60
+
+# Harvesting settings
+ckan.harvest.mq.type = redis
+ckan.harvest.mq.hostname = redis
+ckan.harvest.mq.port = 6379
+ckan.harvest.mq.redis_db = 1
+
+# Spatial settings
+ckan.spatial.validator.profiles = iso19139eden,constraints-1.4,gemini2-1.3
+ckan.spatial.validator.reject = true
+
+## Redis Settings ##############################################################
+ckan.redis.url = redis://redis/0
+
+## CORS Settings ###############################################################
+ckan.cors.origin_allow_all = true
+ckan.cors.origin_whitelist = 
+
+## Plugins Settings ############################################################
+ckan.plugins = activity datagovuk_publisher_form datagovuk dcat harvest ckan_harvester dcat_rdf_harvester dcat_json_harvester dcat_json_interface spatial_metadata spatial_query spatial_harvest_metadata_api gemini_csw_harvester gemini_waf_harvester gemini_doc_harvester inventory_harvester
+ckan.resource_proxy.timeout = 5
+
+## Front-End Settings ##########################################################
+ckan.site_title = dev.data.gov.uk
+ckan.site_description = Data publisher
+; ckan.favicon = /images/icons/ckan.ico
+ckan.gravatar_default = identicon
+ckan.preview.direct = png jpg gif
+ckan.preview.loadable = html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json
+ckan.site_intro_text = 
+ckan.site_logo = /base/images/ckan-logo.png
+ckan.site_about = 
+ckan.theme = css/main
+ckan.favicon = /base/images/ckan.ico
+ckan.datasets_per_page = 20
+package_hide_extras = 
+ckan.dumps_url = 
+ckan.dumps_format = 
+ckan.recaptcha.publickey = 
+ckan.recaptcha.privatekey = 
+ckan.featured_groups = 
+ckan.featured_orgs = 
+ckan.default_group_sort = title
+ckan.debug_supress_header = false
+ckan.homepage_style = 1
+ckan.site_custom_css = 
+
+## Resource Views Settings #####################################################
+ckan.views.default_views = image_view datatables_view
+
+## Theming Settings ############################################################
+ckan.template_head_end = 
+ckan.template_footer_end = 
+ckan.template_title_delimiter = -
+extra_template_paths = 
+extra_public_paths = 
+; ckan.base_public_folder = public
+; ckan.base_templates_folder = templates
+ckan.base_public_folder=public-bs3
+ckan.base_templates_folder=templates-bs3
+ckan.default.package_type = dataset
+ckan.default.group_type = group
+ckan.default.organization_type = organization
+ckan.admin_tabs = {}
+
+## Storage Settings ############################################################
+ckan.storage_path = /var/lib/ckan
+ckan.max_resource_size = 50
+; ckan.max_image_size = 2
+
+## Uploader Settings ###########################################################
+ckan.upload.user.types = 
+ckan.upload.user.mimetypes = 
+ckan.upload.group.types = 
+ckan.upload.group.mimetypes = 
+
+## Webassets Settings ##########################################################
+ckan.webassets.path = 
+ckan.webassets.use_x_sendfile = false
+
+## User Settings ###############################################################
+ckan.user_list_limit = 20
+ckan.user_reset_landing_page = home.index
+
+## Activity Streams Settings ###################################################
+ckan.activity_streams_enabled = true
+ckan.activity_streams_email_notifications = false
+ckan.activity_list_limit = 31
+ckan.activity_list_limit_max = 100
+ckan.email_notifications_since = 2 days
+ckan.hide_activity_from_users = %(ckan.site_id)s
+
+## Feeds Settings ##############################################################
+ckan.feeds.author_name = 
+ckan.feeds.author_link = 
+ckan.feeds.authority_name = 
+ckan.feeds.date = 
+ckan.feeds.limit = 20
+
+## Internationalisation Settings ###############################################
+ckan.locale_default = en_GB
+ckan.locale_order = en_GB
+ckan.locales_offered = en_GB
+ckan.locales_filtered_out = en_US
+ckan.i18n_directory = 
+ckan.i18n.extra_directory = 
+ckan.i18n.extra_gettext_domain = 
+ckan.i18n.extra_locales = 
+ckan.i18n.rtl_languages = he ar fa_IR
+ckan.i18n.rtl_theme = css/main-rtl
+ckan.display_timezone = UTC
+ckan.root_path = 
+ckan.resource_formats = /usr/lib/ckan/venv/src/ckan/ckan/config/resource_formats.json
+
+## Form Settings ###############################################################
+ckan.dataset.create_on_ui_requires_resources = true
+package_new_return_url = 
+package_edit_return_url = 
+licenses_group_url = 
+
+## Email settings ##############################################################
+smtp.server = localhost
+smtp.starttls = false
+smtp.user = 
+smtp.password = 
+smtp.mail_from = 
+smtp.reply_to = 
+email_to = 
+error_email_from = 
+
+## Background Job Settings #####################################################
+ckan.jobs.timeout = 180
+
+## S3 Settngs
+ckan.datagovuk.s3_aws_access_key_id = xx
+ckan.datagovuk.s3_aws_secret_access_key = xx
+ckan.datagovuk.s3_bucket_name = xx
+ckan.datagovuk.s3_url_prefix = xx
+ckan.datagovuk.s3_aws_region_name = xx
+
+## DGU test data settings
+ckan.mock_harvest_source = http://static-mock-harvest-source:11088/
+## Logging configuration
+[loggers]
+keys = root, ckan, ckanext, werkzeug
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARNING
+handlers = console
+
+[logger_werkzeug]
+level = WARNING
+handlers = console
+qualname = werkzeug
+propagate = 0
+
+[logger_ckan]
+level = INFO
+handlers = console
+qualname = ckan
+propagate = 0
+
+[logger_ckanext]
+level = DEBUG
+handlers = console
+qualname = ckanext
+propagate = 0
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s

ckanext/datagovuk/forms/publisher.py

@@ -19,11 +19,11 @@ def form_to_db_schema(self):
         schema = group_form_schema()
         for mandatory_extra in ['category']:
             schema.update({mandatory_extra: [toolkit.get_converter('convert_to_extras'),
-                                         str]})
+                                         toolkit.get_validator('unicode_safe')]})
         for optional_extra in ['contact-name', 'contact-email', 'contact-phone', 'foi-name', 'foi-email', 'foi-web', 'foi-phone']:
             schema.update({optional_extra: [toolkit.get_validator('ignore_missing'),
                                          toolkit.get_converter('convert_to_extras'),
-                                         str]})
+                                         toolkit.get_validator('unicode_safe')]})
         return schema
 
     def db_to_form_schema(self, package_type=None):
@@ -32,7 +32,7 @@ def db_to_form_schema(self, package_type=None):
         for optional_extra in ['category', 'contact-name', 'contact-email', 'contact-phone', 'foi-name', 'foi-email', 'foi-web', 'foi-phone']:
             schema.update({optional_extra: [toolkit.get_converter('convert_from_extras'),
                                          toolkit.get_validator('ignore_missing'),
-                                         str]})
+                                         toolkit.get_validator('unicode_safe')]})
         schema['num_followers'] = []
         schema['created'] = []
         schema['display_name'] = []

ckanext/datagovuk/harvesters/dgu_base.py

@@ -253,9 +253,8 @@ def get_extra(extras, key):
             # Defer constraints and flush so the dataset can be indexed with
             # the harvest object id (on the after_show hook from the harvester
             # plugin)
-            if model.engine_is_pg():
-                model.Session.execute('SET CONSTRAINTS harvest_object_package_id_fkey DEFERRED')
-                model.Session.flush()
+            model.Session.execute('SET CONSTRAINTS harvest_object_package_id_fkey DEFERRED')
+            model.Session.flush()
 
             if source_config.get('private_datasets', False):
                 package_dict['private'] = True