Skip to content

Commit

Permalink
Merge branch 'master' into drop-support-old-versions
Browse files Browse the repository at this point in the history
  • Loading branch information
@pdelboca
pdelboca committed Mar 15, 2023
2 parents c22f7dd + 89a98d7 commit a168881
Show file tree
Hide file tree
Showing 2 changed files with 71 additions and 61 deletions.
112 changes: 61 additions & 51 deletions ckanext/harvest/plugin.py
View file
Expand Up @@ -116,7 +116,40 @@ def before_dataset_search(self, search_params):

return search_params

def _add_or_update_harvest_metadata(self, key, value, data_dict):
"""Adds extras fields or updates them if already exist."""
if not data_dict.get("extras"):
data_dict["extras"] = []

for e in data_dict.get("extras"):
if e.get("key") == key:
e.update({"value": value})
break
else:
data_dict["extras"].append({"key": key, "value": value})

def before_dataset_index(self, pkg_dict):
"""Adds harvest metadata to the extra field of the dataset.
This method will add or update harvest related metadata in `pkg_dict`,
`data_dict` and `validated_data_dict` so it can be obtained when
calling package_show API (that depends on Solr data). This metadata will
be stored in the `extras` field of the dictionaries ONLY if it does not
already exist in the root schema.
Note: If another extension adds any harvest extra to the `package_show`
schema then this method will not add them again in the `extras` field to avoid
validation errors when updating a package.
If the harvest extra has been added to the root schema, then we will not update
them since it is responsibility of the package validators to do it.
"""
# Fix to support Solr8
if isinstance(pkg_dict.get('status'), dict):
try:
pkg_dict['status'] = json.dumps(pkg_dict['status'])
except ValueError:
pkg_dict.pop('status', None)

harvest_object = model.Session.query(HarvestObject) \
.filter(HarvestObject.package_id == pkg_dict["id"]) \
Expand All @@ -125,59 +158,36 @@ def before_dataset_index(self, pkg_dict):
).order_by(HarvestObject.import_finished.desc()) \
.first()

if harvest_object:

data_dict = json.loads(pkg_dict["data_dict"])

validated_data_dict = json.loads(pkg_dict["validated_data_dict"])

harvest_extras = [
("harvest_object_id", harvest_object.id),
("harvest_source_id", harvest_object.source.id),
("harvest_source_title", harvest_object.source.title),
]

for key, value in harvest_extras:

# If the harvest extras are there, update them. This can
# happen eg when calling package_update or resource_update,
# which call package_show
harvest_not_found = True
harvest_not_found_validated = True
if not data_dict.get("extras"):
data_dict["extras"] = []

for e in data_dict.get("extras"):
if e.get("key") == key:
e.update({"value": value})
harvest_not_found = False
if harvest_not_found:
data_dict["extras"].append({"key": key, "value": value})

if not validated_data_dict.get("extras"):
validated_data_dict["extras"] = []

for e in validated_data_dict.get("extras"):
if e.get("key") == key:
e.update({"value": value})
harvest_not_found_validated = False
if harvest_not_found_validated:
validated_data_dict["extras"].append({"key": key, "value": value})

# The commented line isn't cataloged correctly, if we pass the
# basic key the extras are prepended and the system works as
# expected.
# pkg_dict['extras_{0}'.format(key)] = value
if not harvest_object:
return pkg_dict

harvest_extras = [
("harvest_object_id", harvest_object.id),
("harvest_source_id", harvest_object.source.id),
("harvest_source_title", harvest_object.source.title),
]

data_dict = json.loads(pkg_dict["data_dict"])
for key, value in harvest_extras:
if key in data_dict.keys():
data_dict[key] = value
continue
self._add_or_update_harvest_metadata(key, value, data_dict)

validated_data_dict = json.loads(pkg_dict["validated_data_dict"])
for key, value in harvest_extras:
if key in validated_data_dict.keys():
validated_data_dict[key] = value
continue
self._add_or_update_harvest_metadata(key, value, validated_data_dict)

# Add harvest extras to main indexed pkg_dict
for key, value in harvest_extras:
if key not in pkg_dict.keys():
pkg_dict[key] = value

pkg_dict["data_dict"] = json.dumps(data_dict)
pkg_dict["validated_data_dict"] = json.dumps(validated_data_dict)

if isinstance(pkg_dict.get('status'), dict):
try:
pkg_dict['status'] = json.dumps(pkg_dict['status'])
except ValueError:
pkg_dict.pop('status', None)
pkg_dict["data_dict"] = json.dumps(data_dict)
pkg_dict["validated_data_dict"] = json.dumps(validated_data_dict)

return pkg_dict

Expand Down
20 changes: 10 additions & 10 deletions ckanext/harvest/utils.py
View file
Expand Up @@ -506,7 +506,7 @@ def _get_source_for_job(source_id):
except tk.ObjectNotFound:
return tk.abort(404, _('Harvest source not found'))
except tk.NotAuthorized:
return tk.abort(401, _not_auth_message())
return tk.abort(403, _not_auth_message())
except Exception as e:
msg = 'An error occurred: [%s]' % str(e)
return tk.abort(500, msg)
Expand All @@ -526,7 +526,7 @@ def admin_view(id):
except tk.ObjectNotFound:
return tk.abort(404, _('Harvest source not found'))
except tk.NotAuthorized:
return tk.abort(401, _not_auth_message())
return tk.abort(403, _not_auth_message())


def job_show_last_view(source):
Expand Down Expand Up @@ -568,7 +568,7 @@ def job_show_view(id, source_dict=False, is_last=False):
except tk.ObjectNotFound:
return tk.abort(404, _('Harvest job not found'))
except tk.NotAuthorized:
return tk.abort(401, _not_auth_message())
return tk.abort(403, _not_auth_message())
except Exception as e:
msg = 'An error occurred: [%s]' % str(e)
return tk.abort(500, msg)
Expand Down Expand Up @@ -596,7 +596,7 @@ def job_list_view(source):
except tk.ObjectNotFound:
return tk.abort(404, _('Harvest source not found'))
except tk.NotAuthorized:
return tk.abort(401, _not_auth_message())
return tk.abort(403, _not_auth_message())
except Exception as e:
msg = 'An error occurred: [%s]' % str(e)
return tk.abort(500, msg)
Expand All @@ -614,7 +614,7 @@ def about_view(id):
except tk.ObjectNotFound:
return tk.abort(404, _('Harvest source not found'))
except tk.NotAuthorized:
return tk.abort(401, _not_auth_message())
return tk.abort(403, _not_auth_message())


def job_abort_view(source, id):
Expand All @@ -627,7 +627,7 @@ def job_abort_view(source, id):
except tk.ObjectNotFound:
return tk.abort(404, _('Harvest job not found'))
except tk.NotAuthorized:
return tk.abort(401, _not_auth_message())
return tk.abort(403, _not_auth_message())
except Exception as e:
msg = 'An error occurred: [%s]' % str(e)
return tk.abort(500, msg)
Expand All @@ -648,7 +648,7 @@ def refresh_view(id):
except tk.ObjectNotFound:
return tk.abort(404, _('Harvest source not found'))
except tk.NotAuthorized:
return tk.abort(401, _not_auth_message())
return tk.abort(403, _not_auth_message())
except HarvestSourceInactiveError:
h.flash_error(
_('Cannot create new harvest jobs on inactive '
Expand All @@ -674,7 +674,7 @@ def clear_view(id):
except tk.ObjectNotFound:
return tk.abort(404, _('Harvest source not found'))
except tk.NotAuthorized:
return tk.abort(401, _not_auth_message())
return tk.abort(403, _not_auth_message())
except Exception as e:
msg = 'An error occurred: [%s]' % str(e)
h.flash_error(msg)
Expand Down Expand Up @@ -705,7 +705,7 @@ def delete_view(id):
except tk.ObjectNotFound:
return tk.abort(404, _('Harvest source not found'))
except tk.NotAuthorized:
return tk.abort(401, _not_auth_message())
return tk.abort(403, _not_auth_message())


def object_show_view(id, ref_type, response):
Expand Down Expand Up @@ -755,7 +755,7 @@ def object_show_view(id, ref_type, response):
except tk.ObjectNotFound as e:
return tk.abort(404, _(str(e)))
except tk.NotAuthorized:
return tk.abort(401, _not_auth_message())
return tk.abort(403, _not_auth_message())
except Exception as e:
msg = 'An error occurred: [%s]' % str(e)
return tk.abort(500, msg)

0 comments on commit a168881

Please sign in to comment.