From 68661341efa0a3de4e03de3fb1b8f3117de66efe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Johannes=20K=C3=B6ster?= <johannes.koester@tu-dortmund.de>
Date: Fri, 1 Apr 2022 13:01:29 +0200
Subject: [PATCH] fix: report error and possible cause if metadata cleanup
 fails (#1554)

---
 snakemake/persistence.py |  9 +++++++--
 snakemake/workflow.py    | 12 +++++++++++-
 tests/tests.py           |  4 ++++
 3 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/snakemake/persistence.py b/snakemake/persistence.py
index 68853cf8a..c292c2c0a 100755
--- a/snakemake/persistence.py
+++ b/snakemake/persistence.py
@@ -186,7 +186,7 @@ def cleanup_locks(self):
         shutil.rmtree(self._lockdir)
 
     def cleanup_metadata(self, path):
-        self._delete_record(self._metadata_path, path)
+        return self._delete_record(self._metadata_path, path)
 
     def cleanup_shadow(self):
         if os.path.exists(self.shadow_path):
@@ -403,9 +403,14 @@ def _delete_record(self, subject, id):
             recdirs = os.path.relpath(os.path.dirname(recpath), start=subject)
             if recdirs != ".":
                 os.removedirs(recdirs)
+            return True
         except OSError as e:
-            if e.errno != 2:  # not missing
+            if e.errno != 2:
+                # not missing
                 raise e
+            else:
+                # file is missing, report failure
+                return False
 
     @lru_cache()
     def _read_record_cached(self, subject, id):
diff --git a/snakemake/workflow.py b/snakemake/workflow.py
index 4b2adbdfe..488b8dda7 100644
--- a/snakemake/workflow.py
+++ b/snakemake/workflow.py
@@ -764,8 +764,18 @@ def files(items):
             self.persistence.deactivate_cache()
 
         if cleanup_metadata:
+            failed = []
             for f in cleanup_metadata:
-                self.persistence.cleanup_metadata(f)
+                success = self.persistence.cleanup_metadata(f)
+                if not success:
+                    failed.append(f)
+            if failed:
+                logger.warning(
+                    "Failed to clean up metadata for the following files because the metadata was not present.\n"
+                    "If this is expected, there is nothing to do.\nOtherwise, the reason might be file system latency "
+                    "or still running jobs.\nConsider running metadata cleanup again.\nFiles:\n"
+                    + "\n".join(failed)
+                )
             return True
 
         if unlock:
diff --git a/tests/tests.py b/tests/tests.py
index b4885dc2e..12495ee51 100644
--- a/tests/tests.py
+++ b/tests/tests.py
@@ -1589,3 +1589,7 @@ def test_github_issue1500():
 
 def test_github_issue1542():
     run(dpath("test_github_issue1542"), dryrun=True)
+
+
+def test_cleanup_metadata_fail():
+    run(dpath("test09"), cleanup_metadata=["xyz"])