fix: report error and possible cause if metadata cleanup fails (#1554)

snakemake/persistence.py

@@ -186,7 +186,7 @@ def cleanup_locks(self):
         shutil.rmtree(self._lockdir)
 
     def cleanup_metadata(self, path):
-        self._delete_record(self._metadata_path, path)
+        return self._delete_record(self._metadata_path, path)
 
     def cleanup_shadow(self):
         if os.path.exists(self.shadow_path):
@@ -403,9 +403,14 @@ def _delete_record(self, subject, id):
             recdirs = os.path.relpath(os.path.dirname(recpath), start=subject)
             if recdirs != ".":
                 os.removedirs(recdirs)
+            return True
         except OSError as e:
-            if e.errno != 2:  # not missing
+            if e.errno != 2:
+                # not missing
                 raise e
+            else:
+                # file is missing, report failure
+                return False
 
     @lru_cache()
     def _read_record_cached(self, subject, id):

snakemake/workflow.py

@@ -764,8 +764,18 @@ def files(items):
             self.persistence.deactivate_cache()
 
         if cleanup_metadata:
+            failed = []
             for f in cleanup_metadata:
-                self.persistence.cleanup_metadata(f)
+                success = self.persistence.cleanup_metadata(f)
+                if not success:
+                    failed.append(f)
+            if failed:
+                logger.warning(
+                    "Failed to clean up metadata for the following files because the metadata was not present.\n"
+                    "If this is expected, there is nothing to do.\nOtherwise, the reason might be file system latency "
+                    "or still running jobs.\nConsider running metadata cleanup again.\nFiles:\n"
+                    + "\n".join(failed)
+                )
             return True
 
         if unlock:

tests/tests.py

@@ -1589,3 +1589,7 @@ def test_github_issue1500():
 
 def test_github_issue1542():
     run(dpath("test_github_issue1542"), dryrun=True)
+
+
+def test_cleanup_metadata_fail():
+    run(dpath("test09"), cleanup_metadata=["xyz"])