fix: Adds fixes for the first two MREs in #823 (#1215)

* add failing tests 823

* fix mistakes

* black

* Fix the first two MREs from #823.

- Incorporates @epruesse's fix for MRE #1
- Adds a fix for MRE #2 - properly marks group jobs as finished
- Some minor updates to tests

* Fix tests on Windows

* Skip MRE 2 from 823 on Windows due to `pipe()` output

Co-authored-by: Maarten-vd-Sande <maartenvandersande@hotmail.com>
Co-authored-by: Johannes Köster <johannes.koester@uni-due.de>

snakemake/dag.py

@@ -617,8 +617,13 @@ def unneeded_files():
                 yield from filterfalse(partial(needed, job_), tempfiles & files)
 
             # temp output
-            if not job.dynamic_output and (
-                job not in self.targetjobs or job.rule.name == self.workflow.first_rule
+            if (
+                not job.dynamic_output
+                and not job.is_checkpoint
+                and (
+                    job not in self.targetjobs
+                    or job.rule.name == self.workflow.first_rule
+                )
             ):
                 tempfiles = (
                     f
@@ -1360,7 +1365,11 @@ def finish(self, job, update_dynamic=True):
         self._running.remove(job)
 
         # turn off this job's Reason
-        self.reason(job).mark_finished()
+        if job.is_group():
+            for j in job:
+                self.reason(j).mark_finished()
+        else:
+            self.reason(job).mark_finished()
 
         try:
             self._ready_jobs.remove(job)

tests/test_issue823_1/Snakefile

@@ -0,0 +1,27 @@
+rule all:
+    input:
+        "b.txt"
+
+checkpoint a:
+    output:
+        # to reproduce the bug:
+        # >1 output required
+        # one or more temp() outputs required
+        # exception: will work if only the first output is temp()
+        temp('a1.txt'),
+        temp('a2.txt'),
+    shell:
+        "touch {output}"
+
+def _checkpoint_output(wildcards):
+    out = checkpoints.a.get(**wildcards).output
+    return out
+
+rule b:
+    input:
+        _checkpoint_output,
+    output:
+        'b.txt'
+    shell:
+        "touch {output}"
+

tests/test_issue823_2/Snakefile

@@ -0,0 +1,36 @@
+rule all:
+    input:
+        "c.txt",
+
+checkpoint a:
+    output:
+        "a.txt"
+    shell:
+        "touch {output}"
+
+rule b1:
+    output:
+        pipe("b.pipe")
+    shell:
+        "echo test > {output}"
+
+rule b2:
+    input:
+        "b.pipe"
+    output:
+        "b.txt"
+    shell:
+        """
+        cat {input} > /dev/null
+        touch {output}
+        """
+
+rule c:
+    input:
+        "a.txt",
+        "b.txt",
+    output:
+        "c.txt"
+    shell:
+        "touch {output}"
+

tests/tests.py

@@ -798,6 +798,15 @@ def test_issue805():
     run(dpath("test_issue805"), shouldfail=True)
 
 
+def test_issue823_1():
+    run(dpath("test_issue823_1"))
+
+
+@skip_on_windows
+def test_issue823_2():
+    run(dpath("test_issue823_2"))
+
+
 @skip_on_windows
 def test_pathlib():
     run(dpath("test_pathlib"))