ENH Introduce remove_many

This makes optimizes the removal of multiple entries. With packs, doing
it one by one was extraordinarily slow as it kept repacking

jug/backends/base.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright (C) 2011-2021, Luis Pedro Coelho <luis@luispedro.org>
+# Copyright (C) 2011-2023, Luis Pedro Coelho <luis@luispedro.org>
 # vim: set ts=4 sts=4 sw=4 expandtab smartindent:
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -107,6 +107,33 @@ def load(self, name):
             The object that was saved under ``name``
         '''
 
+    def remove_many(self, names):
+        '''
+        removed = store.remove_many(names)
+
+        Remove the entry associated with ``names``.
+
+        Returns set of names that were actually removed.
+
+        Default implementation calls ``remove`` for each name, but this can be
+        overridden for efficiency.
+
+        Parameters
+        ----------
+        names : iterable of str
+            Keys
+
+        Returns
+        -------
+        removed : set of str
+            Keys that were actually removed
+        '''
+        removed = []
+        for name in names:
+            if self.remove(name):
+                removed.append(name)
+        return removed
+
     @abstractmethod
     def remove(self, name):
         '''

jug/backends/file_store.py

@@ -277,6 +277,26 @@ def load(self, name):
             return decode_from(ifile)
 
 
+    def remove_many(self, names):
+        '''
+        store.remove_many(names)
+
+        Removes the objects with the given names from the store.
+        '''
+        removed = set()
+        for name in names:
+            if name in self.packed:
+                del self.packed[name]
+                removed.add(name)
+            try:
+                fname = self._getfname(name)
+                os.unlink(fname)
+                removed.add(name)
+            except OSError:
+                pass
+        self.resave_pack()
+        return removed
+
     def remove(self, name):
         '''
         was_removed = store.remove(name)
@@ -285,17 +305,7 @@ def remove(self, name):
 
         Returns whether any entry was actually removed.
         '''
-        removed = False
-        if name in self.packed:
-            del self.packed[name]
-            self.resave_pack()
-            removed = True
-        try:
-            fname = self._getfname(name)
-            os.unlink(fname)
-            return True
-        except OSError:
-            return removed
+        return bool(self.remove_many([name]))
 
 
     def cleanup(self, active, keeplocks=False):

jug/subcommands/invalidate.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
-# Copyright (C) 2008-2015, Luis Pedro Coelho <luis@luispedro.org>
+# Copyright (C) 2008-2023, Luis Pedro Coelho <luis@luispedro.org>
 # vim: set ts=4 sts=4 sw=4 expandtab smartindent:
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -77,8 +77,9 @@ def isinvalid(t):
             options.print_out('No results invalidated.')
             return
         task_counts = defaultdict(int)
+        removed = store.remove_many(t.hash() for t in invalid)
         for t in invalid:
-            if store.remove(t.hash()):
+            if t.hash() in removed:
                 task_counts[t.name] += 1
         if sum(task_counts.values()) == 0:
             options.print_out('Tasks invalidated, but no results removed')