From cfd2f890a0af57628f7b9278d8d43f59b7006825 Mon Sep 17 00:00:00 2001
From: Don Freed <DonFreed@users.noreply.github.com>
Date: Mon, 25 Oct 2021 12:29:40 -0700
Subject: [PATCH] fix: Adds fixes for the first two MREs in #823 (#1215)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* add failing tests 823

* fix mistakes

* black

* Fix the first two MREs from #823.

- Incorporates @epruesse's fix for MRE #1
- Adds a fix for MRE #2 - properly marks group jobs as finished
- Some minor updates to tests

* Fix tests on Windows

* Skip MRE 2 from 823 on Windows due to `pipe()` output

Co-authored-by: Maarten-vd-Sande <maartenvandersande@hotmail.com>
Co-authored-by: Johannes Köster <johannes.koester@uni-due.de>
---
 snakemake/dag.py                             | 15 ++++++--
 tests/test_issue823_1/Snakefile              | 27 +++++++++++++++
 tests/test_issue823_1/expected-results/b.txt |  0
 tests/test_issue823_2/Snakefile              | 36 ++++++++++++++++++++
 tests/test_issue823_2/expected-results/c.txt |  0
 tests/tests.py                               |  9 +++++
 6 files changed, 84 insertions(+), 3 deletions(-)
 create mode 100644 tests/test_issue823_1/Snakefile
 create mode 100644 tests/test_issue823_1/expected-results/b.txt
 create mode 100644 tests/test_issue823_2/Snakefile
 create mode 100644 tests/test_issue823_2/expected-results/c.txt

diff --git a/snakemake/dag.py b/snakemake/dag.py
index e60eaa7c0..a8f5e8578 100755
--- a/snakemake/dag.py
+++ b/snakemake/dag.py
@@ -617,8 +617,13 @@ def unneeded_files():
                 yield from filterfalse(partial(needed, job_), tempfiles & files)
 
             # temp output
-            if not job.dynamic_output and (
-                job not in self.targetjobs or job.rule.name == self.workflow.first_rule
+            if (
+                not job.dynamic_output
+                and not job.is_checkpoint
+                and (
+                    job not in self.targetjobs
+                    or job.rule.name == self.workflow.first_rule
+                )
             ):
                 tempfiles = (
                     f
@@ -1360,7 +1365,11 @@ def finish(self, job, update_dynamic=True):
         self._running.remove(job)
 
         # turn off this job's Reason
-        self.reason(job).mark_finished()
+        if job.is_group():
+            for j in job:
+                self.reason(j).mark_finished()
+        else:
+            self.reason(job).mark_finished()
 
         try:
             self._ready_jobs.remove(job)
diff --git a/tests/test_issue823_1/Snakefile b/tests/test_issue823_1/Snakefile
new file mode 100644
index 000000000..86f518376
--- /dev/null
+++ b/tests/test_issue823_1/Snakefile
@@ -0,0 +1,27 @@
+rule all:
+    input:
+        "b.txt"
+
+checkpoint a:
+    output:
+        # to reproduce the bug:
+        # >1 output required
+        # one or more temp() outputs required
+        # exception: will work if only the first output is temp()
+        temp('a1.txt'),
+        temp('a2.txt'),
+    shell:
+        "touch {output}"
+
+def _checkpoint_output(wildcards):
+    out = checkpoints.a.get(**wildcards).output
+    return out
+
+rule b:
+    input:
+        _checkpoint_output,
+    output:
+        'b.txt'
+    shell:
+        "touch {output}"
+
diff --git a/tests/test_issue823_1/expected-results/b.txt b/tests/test_issue823_1/expected-results/b.txt
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/test_issue823_2/Snakefile b/tests/test_issue823_2/Snakefile
new file mode 100644
index 000000000..507f495b6
--- /dev/null
+++ b/tests/test_issue823_2/Snakefile
@@ -0,0 +1,36 @@
+rule all:
+    input:
+        "c.txt",
+
+checkpoint a:
+    output:
+        "a.txt"
+    shell:
+        "touch {output}"
+
+rule b1:
+    output:
+        pipe("b.pipe")
+    shell:
+        "echo test > {output}"
+
+rule b2:
+    input:
+        "b.pipe"
+    output:
+        "b.txt"
+    shell:
+        """
+        cat {input} > /dev/null
+        touch {output}
+        """
+
+rule c:
+    input:
+        "a.txt",
+        "b.txt",
+    output:
+        "c.txt"
+    shell:
+        "touch {output}"
+
diff --git a/tests/test_issue823_2/expected-results/c.txt b/tests/test_issue823_2/expected-results/c.txt
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/tests.py b/tests/tests.py
index 0223c0bc2..f1ec34c28 100644
--- a/tests/tests.py
+++ b/tests/tests.py
@@ -798,6 +798,15 @@ def test_issue805():
     run(dpath("test_issue805"), shouldfail=True)
 
 
+def test_issue823_1():
+    run(dpath("test_issue823_1"))
+
+
+@skip_on_windows
+def test_issue823_2():
+    run(dpath("test_issue823_2"))
+
+
 @skip_on_windows
 def test_pathlib():
     run(dpath("test_pathlib"))