fix: more robust handling of incompletely evaluated parameters (any i…

…nteraction with them will result in a string <TBD> now). (#1525)

* fix: catch any errors when formatting job information

* Detect incomplete params evaluation and avoid printing the shell command in such a case

* More robust TBD handling, working for any interaction with the <TBD> object.

snakemake/common/__init__.py

@@ -16,13 +16,14 @@
 from pathlib import Path
 
 from snakemake._version import get_versions
+from snakemake.common.tbdstring import TBDString
 
 __version__ = get_versions()["version"]
 del get_versions
 
 
 MIN_PY_VERSION = (3, 7)
-DYNAMIC_FILL = "__snakemake_dynamic__"
+DYNAMIC_FILL = "__othernakemake_dynamic__"
 SNAKEMAKE_SEARCHPATH = str(Path(__file__).parent.parent.parent)
 UUID_NAMESPACE = uuid.uuid5(uuid.NAMESPACE_URL, "https://snakemake.readthedocs.io")
 NOTHING_TO_BE_DONE_MSG = (
@@ -47,13 +48,6 @@ def async_run(coroutine):
         asyncio.create_task(coroutine)
 
 
-# A string that prints as TBD
-class TBDString(str):
-    # the second arg is necessary to avoid problems when pickling
-    def __new__(cls, _=None):
-        return str.__new__(cls, "<TBD>")
-
-
 APPDIRS = None
 
 
@@ -178,7 +172,7 @@ class Mode:
 
 
 class lazy_property(property):
-    __slots__ = ["method", "cached", "__doc__"]
+    __otherlots__ = ["method", "cached", "__doc__"]
 
     @staticmethod
     def clean(instance, method):

snakemake/common/tbdstring.py

@@ -0,0 +1,99 @@
+# A string that prints as TBD
+# whatever interaction happens on this class, <TBD> shall be returned
+class TBDString(str):
+    # the second arg is necessary to avoid problems when pickling
+    def __new__(cls, _=None):
+        return str.__new__(cls, "<TBD>")
+
+    def __getitem__(self, __item):
+        return self
+
+    def __getattribute__(self, __name):
+        return self
+
+    def __bool__(self):
+        return False
+
+    def __add__(self, __other):
+        return self
+
+    def __sub__(self, __other):
+        return self
+
+    def __mul__(self, __other):
+        return self
+
+    def __matmul__(self, __other):
+        return self
+
+    def __truediv__(self, __other):
+        return self
+
+    def __floordiv__(self, __other):
+        return self
+
+    def __mod__(self, __other):
+        return self
+
+    def __divmod__(self, __other):
+        return self
+
+    def __pow__(self, __other):
+        return self
+
+    def __lshift__(self, __other):
+        return self
+
+    def __rshift__(self, __other):
+        return self
+
+    def __and__(self, __other):
+        return self
+
+    def __xor__(self, __other):
+        return self
+
+    def __or__(self, __other):
+        return self
+
+    def __neg__(self):
+        return self
+
+    def __pos__(self):
+        return self
+
+    def __abs__(self):
+        return self
+
+    def __invert__(self):
+        return self
+
+    def __complex__(self):
+        return self
+
+    def __int__(self):
+        return self
+
+    def __float__(self):
+        return self
+
+    def __index__(self):
+        return self
+
+    def __round__(self, ndigits=0):
+        return self
+
+    def __trunc__(self):
+        return self
+
+    def __floor__(self):
+        return self
+
+    def __ceil__(self):
+        return self
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, __exc_type, __exc_value, __traceback):
+        return self

snakemake/jobs.py

@@ -24,7 +24,11 @@
     wait_for_files,
 )
 from snakemake.utils import format, listfiles
-from snakemake.exceptions import RuleException, ProtectedOutputException, WorkflowError
+from snakemake.exceptions import (
+    RuleException,
+    ProtectedOutputException,
+    WorkflowError,
+)
 from snakemake.logging import logger
 from snakemake.common import (
     DYNAMIC_FILL,
@@ -904,6 +908,10 @@ def format_wildcards(self, string, **variables):
             raise RuleException("NameError: " + str(ex), rule=self.rule)
         except IndexError as ex:
             raise RuleException("IndexError: " + str(ex), rule=self.rule)
+        except Exception as ex:
+            raise WorkflowError(
+                f"Error when formatting '{string}' for rule {self.rule.name}. {ex}"
+            )
 
     def properties(self, omit_resources=["_cores", "_nodes"], **aux_properties):
         resources = {
@@ -1469,6 +1477,10 @@ def format_wildcards(self, string, **variables):
             raise WorkflowError(
                 "IndexError with group job {}: {}".format(self.jobid, str(ex))
             )
+        except Exception as ex:
+            raise WorkflowError(
+                f"Error when formatting {string} for group job {self.jobid}: {ex}"
+            )
 
     @property
     def threads(self):

tests/test_incomplete_params/Snakefile

@@ -0,0 +1,27 @@
+def get_x(wildcards, input):
+    with open(input[0]) as infile:
+        return {"foo": infile.read()}
+
+
+def get_mem_mb(wildcards, input):
+    return os.path.getsize(input[0]) / 1024.0
+
+
+rule a:
+    input:
+        "test.in",
+    output:
+        "test.out",
+    params:
+        x=get_x,
+    resources:
+        mem_mb=get_mem_mb,
+    shell:
+        "echo {params.x[foo]} > {output}"
+
+
+rule b:
+    output:
+        "test.in",
+    shell:
+        "touch {output}"

tests/tests.py

@@ -1561,3 +1561,7 @@ def test_groupid_expand_cluster():
 @skip_on_windows
 def test_service_jobs():
     run(dpath("test_service_jobs"), check_md5=False)
+
+
+def test_incomplete_params():
+    run(dpath("test_incomplete_params"), dryrun=True, printshellcmds=True)