Allow purge the harvest source

Make a new list item that can purge the harvest source so that one can
directly delete the data in the harvest source and package table when a
harvest source is no longer needed.

CHANGELOG.rst

@@ -10,6 +10,16 @@ and this project adheres to `Semantic Versioning <http://semver.org/>`_
 ***********
 Unreleased_
 ***********
+Changed
+-------
+
+- Add a new list item "Deleted and purge source" in templates/source/new_source_form.html that can purge the harvest source
+
+
+Fixed
+-----
+
+- remove confirm action in list item "Deleted and clear source" because confirm action will make cause the delete view can not pass the "clear = true"
 
 ***********
 1.4.1_ - 2022-09-20

ckanext/harvest/controllers/view.py

@@ -22,6 +22,9 @@ def refresh(self, id):
     def clear(self, id):
         return utils.clear_view(id)
 
+    def purge(id):
+        return utils.purge_view(id)
+
     def show_object(self, id, ref_type='object'):
         _, content = utils.object_show_view(id, ref_type, response)
         return content

ckanext/harvest/logic/action/delete.py

@@ -21,7 +21,13 @@ def harvest_source_delete(context, data_dict):
 
     p.toolkit.get_action('package_delete')(context, data_dict)
 
-    if context.get('clear_source', False):
+    if context.get('purge_resource', False):
+
+        # We need the id. The name won't work.
+        package_dict = p.toolkit.get_action('package_show')(context, data_dict)
+        p.toolkit.get_action('purge_harvest_source')(
+            context, {'id': package_dict['id']})
+    elif context.get('clear_source', False):
 
         # We need the id. The name won't work.
         package_dict = p.toolkit.get_action('package_show')(context, data_dict)

ckanext/harvest/logic/action/update.py

@@ -237,6 +237,38 @@ def harvest_source_clear(context, data_dict):
     return {'id': harvest_source_id}
 
 
+def purge_harvest_source(context, data_dict):
+    '''
+    Clears all datasets, jobs and objects related to a harvest source, includes
+    the source itself.  This is useful to clean history of long running
+    harvest sources to start again fresh.
+    :param id: the id of the harvest source to clear
+    :type id: string
+    '''
+
+    check_access('purge_harvest_source', context, data_dict)
+
+    harvest_source_id = data_dict.get('id')
+
+    source = HarvestSource.get(harvest_source_id)
+    if not source:
+        log.error('Harvest source %s does not exist', harvest_source_id)
+        raise NotFound('Harvest source %s does not exist' % harvest_source_id)
+
+    #Clears all datasets, jobs and objects related to a harvest source
+    get_action('harvest_source_clear')(context, data_dict)
+
+    harvest_source_id = source.id
+
+    #Delete harvest source itself
+    source.delete()
+    # Refresh the index for this source to update the status object
+    get_action('harvest_source_reindex')(context, {'id': harvest_source_id})
+    #Purge the dataset itself
+    get_action("dataset_purge")(context,{'id':harvest_source_id})
+    return {'id': harvest_source_id}
+
+
 def harvest_abort_failed_jobs(context, data_dict):
     session = context['session']
 

ckanext/harvest/logic/auth/update.py

@@ -49,6 +49,25 @@ def harvest_source_clear(context, data_dict):
     return harvest_source_update(context, data_dict)
 
 
+def purge_harvest_sources(context, data_dict):
+    '''
+        Authorization check for purging for all harvest sources
+        Only sysadmins can do it
+    '''
+    if not user_is_sysadmin(context):
+        return {'success': False, 'msg': pt._('Only sysadmins can purge for all harvest source')}
+    else:
+        return {'success': True}
+
+
+def purge_harvest_source(context, data_dict):
+    '''
+        Authorization check for purging a harvest source
+        It forwards to harvest_source_update
+    '''
+    return harvest_source_update(context, data_dict)
+
+
 def harvest_objects_import(context, data_dict):
     '''
         Authorization check reimporting all harvest objects

ckanext/harvest/plugin/__init__.py

@@ -294,7 +294,7 @@ def update_config(self, config):
             })
             bp_routes = [
                 "delete", "refresh", "admin", "about",
-                "clear", "job_list", "job_show_last", "job_show",
+                "clear", "purge", "job_list", "job_show_last", "job_show",
                 "job_abort", "object_show"
             ]
             mappings.update({

ckanext/harvest/templates/source/new_source_form.html

@@ -106,10 +106,15 @@
         </a>
       </li>
       <li>
-        <a href="{% url_for 'harvest_delete', id=data.name %}?clear=True" data-module="confirm-action" data-module-i18n="{{ locale_clear }}">
+        <a href="{% url_for 'harvest_delete', id=data.name %}?clear=True">
           {{ _('Delete and clear source') }}
         </a>
       </li>
+      <li>
+        <a href="{% url_for 'harvest_delete', id=data.name %}?purge=True">
+          {{ _('Delete and purge source') }}
+        </a>
+      </li>
     </ul>
   </div>
       {% endif %}

ckanext/harvest/utils.py

@@ -232,6 +232,18 @@ def clear_harvest_source_history(source_id, keep_current):
             len(cleared_sources_dicts))
 
 
+def purge_harvest_source(source_id_or_name):
+    context = {
+        "model": model,
+        "session": model.Session,
+        "user": _admin_user()["name"],
+    }
+    source = tk.get_action("harvest_source_show")(context, {
+        "id": source_id_or_name
+    })
+    tk.get_action("purge_harvest_source")(context, {"id": source["id"]})
+
+
 def abort_failed_jobs(job_life_span, include, exclude):
     context = {
         "model": model,
@@ -694,6 +706,22 @@ def clear_view(id):
         h.url_for('{0}_admin'.format(DATASET_TYPE_NAME), id=id))
 
 
+def purge_view(id):
+    try:
+        context = {'model': model, 'user': tk.c.user, 'session': model.Session}
+        tk.get_action('purge_harvest_source')(context, {'id': id})
+        h.flash_success(_('Harvest source purged'))
+    except tk.ObjectNotFound:
+        return tk.abort(404, _('Harvest source not found'))
+    except tk.NotAuthorized:
+        return tk.abort(401, _not_auth_message())
+    except Exception as e:
+        msg = 'An error occurred: [%s]' % str(e)
+        h.flash_error(msg)
+
+    return h.redirect_to('/harvest')
+
+
 def delete_view(id):
     try:
         context = {'model': model, 'user': tk.c.user}
@@ -703,9 +731,18 @@ def delete_view(id):
                                                             u'true',
                                                             u'1',
                                                         )
+        context['purge_resource'] = tk.request.params.get('purge',
+                                                        '').lower() in (
+                                                            u'true',
+                                                            u'1',
+                                                        )
 
         tk.get_action('harvest_source_delete')(context, {'id': id})
 
+        if context['purge_resource']:
+            h.flash_success(_('Harvesting source successfully purged'))
+            return h.redirect_to('/harvest')
+
         if context['clear_source']:
             h.flash_success(_('Harvesting source successfully cleared'))
         else:

ckanext/harvest/views.py

@@ -33,6 +33,10 @@ def clear(id):
     return utils.clear_view(id)
 
 
+def purge(id):
+    return utils.purge_view(id)
+
+
 def job_list(source):
     return utils.job_list_view(source)
 
@@ -73,6 +77,9 @@ def object_show(id, ref_type):
 harvester.add_url_rule("/" + utils.DATASET_TYPE_NAME + "/clear/<id>",
                        view_func=clear,
                        methods=(u'POST', u'GET'))
+harvester.add_url_rule("/" + utils.DATASET_TYPE_NAME + "/purge/<id>",
+                       view_func=purge,
+                       methods=(u'POST', u'GET'))
 harvester.add_url_rule(
     "/" + utils.DATASET_TYPE_NAME + "/<source>/job",
     view_func=job_list,