From d9a56aaf75c5f70ba0217d9d461d839fa3013f2e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Johannes=20K=C3=B6ster?= <johannes.koester@tu-dortmund.de>
Date: Wed, 30 Mar 2022 14:24:14 +0200
Subject: [PATCH] fix: correct handling of exceptions in input functions that
 are generators (#1536)

* fix: correct handling of exceptions in input functions that are generators

* fmt
---
 snakemake/exceptions.py                       |  2 +-
 snakemake/executors/__init__.py               |  4 ++-
 snakemake/rules.py                            |  6 +++++
 tests/test_github_issue261/Snakefile          | 25 +++++++++++++++++++
 .../test1/target1/config1.done                |  0
 tests/tests.py                                |  5 ++++
 6 files changed, 40 insertions(+), 2 deletions(-)
 create mode 100644 tests/test_github_issue261/Snakefile
 create mode 100644 tests/test_github_issue261/expected-results/test1/target1/config1.done

diff --git a/snakemake/exceptions.py b/snakemake/exceptions.py
index 288bec9c3..591a670c9 100644
--- a/snakemake/exceptions.py
+++ b/snakemake/exceptions.py
@@ -510,7 +510,7 @@ class CheckSumMismatchException(WorkflowError):
 class IncompleteCheckpointException(Exception):
     def __init__(self, rule, targetfile):
         super().__init__(
-            "The requested checkpoint output is not yet created."
+            "The requested checkpoint output is not yet created. "
             "If you see this error, you have likely tried to use "
             "checkpoint output outside of an input function, or "
             "you have tried to call an input function directly "
diff --git a/snakemake/executors/__init__.py b/snakemake/executors/__init__.py
index 109f4a1c0..60d788fff 100644
--- a/snakemake/executors/__init__.py
+++ b/snakemake/executors/__init__.py
@@ -465,6 +465,9 @@ def get_python_executable(self):
     def get_envvar_declarations(self):
         return ""
 
+    def get_job_args(self, job, **kwargs):
+        return f"{super().get_job_args(job, **kwargs)} --quiet"
+
     def run(self, job, callback=None, submit_callback=None, error_callback=None):
         super()._run(job)
 
@@ -581,7 +584,6 @@ def run_group_job(self, job):
 
     def spawn_job(self, job):
         cmd = self.format_job_exec(job)
-        print(cmd)
         try:
             subprocess.check_call(cmd, shell=True)
         except subprocess.CalledProcessError as e:
diff --git a/snakemake/rules.py b/snakemake/rules.py
index 26a0ba05c..b02d55666 100644
--- a/snakemake/rules.py
+++ b/snakemake/rules.py
@@ -5,6 +5,7 @@
 
 import os
 import re
+import types
 from snakemake.path_modifier import PATH_MODIFIER_FLAG
 import sys
 import inspect
@@ -753,6 +754,11 @@ def apply_input_function(
 
         try:
             value = func(Wildcards(fromdict=wildcards), **_aux_params)
+            if isinstance(value, types.GeneratorType):
+                # generators should be immediately collected here,
+                # otherwise we would miss any exceptions and
+                # would have to capture them again later.
+                value = list(value)
         except IncompleteCheckpointException as e:
             value = incomplete_checkpoint_func(e)
             incomplete = True
diff --git a/tests/test_github_issue261/Snakefile b/tests/test_github_issue261/Snakefile
new file mode 100644
index 000000000..79f89303f
--- /dev/null
+++ b/tests/test_github_issue261/Snakefile
@@ -0,0 +1,25 @@
+import random
+
+checkpoint random:
+    output:
+        target = 'test1/{target}/{config}.txt'
+
+    run:
+        with open(output.target, 'w') as ftg:
+            for i in range(2, 10):
+                ftg.write(f'test1/{wildcards.target}/{wildcards.config}/v{i}\n')
+
+rule process:
+    output:
+        touch('test1/{target}/{config}/v{index}')
+
+def genetate_inputs(wildcards):
+    with checkpoints.random.get(**wildcards).output[0].open() as fran:
+        for line in fran.readlines():
+            yield line.rstrip()
+
+rule generate:
+    input:
+        genetate_inputs
+    output:
+        touch('test1/{target}/{config}.done')
diff --git a/tests/test_github_issue261/expected-results/test1/target1/config1.done b/tests/test_github_issue261/expected-results/test1/target1/config1.done
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/tests.py b/tests/tests.py
index 4783b024d..bd5702a0e 100644
--- a/tests/tests.py
+++ b/tests/tests.py
@@ -1567,6 +1567,11 @@ def test_incomplete_params():
     run(dpath("test_incomplete_params"), dryrun=True, printshellcmds=True)
 
 
+@skip_on_windows
+def test_github_issue261():
+    run(dpath("test_github_issue261"), targets=["test1/target1/config1.done"])
+
+
 @skip_on_windows  # no pipe support on windows
 def test_pipe_depend():
     run(dpath("test_pipe_depend"), shouldfail=True)