From 3d4c768aafbdca67a9032ad9e3b73449a1fadb0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Mon, 28 Mar 2022 16:54:57 +0200 Subject: [PATCH] fix: more robust handling of incompletely evaluated parameters (any interaction with them will result in a string now). (#1525) * fix: catch any errors when formatting job information * Detect incomplete params evaluation and avoid printing the shell command in such a case * More robust TBD handling, working for any interaction with the object. --- snakemake/common/__init__.py | 12 +-- snakemake/common/tbdstring.py | 99 +++++++++++++++++++ snakemake/jobs.py | 14 ++- tests/test_incomplete_params/Snakefile | 27 +++++ .../expected-results/.gitkeep | 0 tests/tests.py | 4 + 6 files changed, 146 insertions(+), 10 deletions(-) create mode 100644 snakemake/common/tbdstring.py create mode 100644 tests/test_incomplete_params/Snakefile create mode 100644 tests/test_incomplete_params/expected-results/.gitkeep diff --git a/snakemake/common/__init__.py b/snakemake/common/__init__.py index 3119ec81c..b09feb91d 100644 --- a/snakemake/common/__init__.py +++ b/snakemake/common/__init__.py @@ -16,13 +16,14 @@ from pathlib import Path from snakemake._version import get_versions +from snakemake.common.tbdstring import TBDString __version__ = get_versions()["version"] del get_versions MIN_PY_VERSION = (3, 7) -DYNAMIC_FILL = "__snakemake_dynamic__" +DYNAMIC_FILL = "__othernakemake_dynamic__" SNAKEMAKE_SEARCHPATH = str(Path(__file__).parent.parent.parent) UUID_NAMESPACE = uuid.uuid5(uuid.NAMESPACE_URL, "https://snakemake.readthedocs.io") NOTHING_TO_BE_DONE_MSG = ( @@ -47,13 +48,6 @@ def async_run(coroutine): asyncio.create_task(coroutine) -# A string that prints as TBD -class TBDString(str): - # the second arg is necessary to avoid problems when pickling - def __new__(cls, _=None): - return str.__new__(cls, "") - - APPDIRS = None @@ -178,7 +172,7 @@ class Mode: class lazy_property(property): - __slots__ = ["method", "cached", "__doc__"] + __otherlots__ = ["method", "cached", "__doc__"] @staticmethod def clean(instance, method): diff --git a/snakemake/common/tbdstring.py b/snakemake/common/tbdstring.py new file mode 100644 index 000000000..9440098c1 --- /dev/null +++ b/snakemake/common/tbdstring.py @@ -0,0 +1,99 @@ +# A string that prints as TBD +# whatever interaction happens on this class, shall be returned +class TBDString(str): + # the second arg is necessary to avoid problems when pickling + def __new__(cls, _=None): + return str.__new__(cls, "") + + def __getitem__(self, __item): + return self + + def __getattribute__(self, __name): + return self + + def __bool__(self): + return False + + def __add__(self, __other): + return self + + def __sub__(self, __other): + return self + + def __mul__(self, __other): + return self + + def __matmul__(self, __other): + return self + + def __truediv__(self, __other): + return self + + def __floordiv__(self, __other): + return self + + def __mod__(self, __other): + return self + + def __divmod__(self, __other): + return self + + def __pow__(self, __other): + return self + + def __lshift__(self, __other): + return self + + def __rshift__(self, __other): + return self + + def __and__(self, __other): + return self + + def __xor__(self, __other): + return self + + def __or__(self, __other): + return self + + def __neg__(self): + return self + + def __pos__(self): + return self + + def __abs__(self): + return self + + def __invert__(self): + return self + + def __complex__(self): + return self + + def __int__(self): + return self + + def __float__(self): + return self + + def __index__(self): + return self + + def __round__(self, ndigits=0): + return self + + def __trunc__(self): + return self + + def __floor__(self): + return self + + def __ceil__(self): + return self + + def __enter__(self): + return self + + def __exit__(self, __exc_type, __exc_value, __traceback): + return self diff --git a/snakemake/jobs.py b/snakemake/jobs.py index 56c0f5af5..c6c7d32a7 100644 --- a/snakemake/jobs.py +++ b/snakemake/jobs.py @@ -24,7 +24,11 @@ wait_for_files, ) from snakemake.utils import format, listfiles -from snakemake.exceptions import RuleException, ProtectedOutputException, WorkflowError +from snakemake.exceptions import ( + RuleException, + ProtectedOutputException, + WorkflowError, +) from snakemake.logging import logger from snakemake.common import ( DYNAMIC_FILL, @@ -904,6 +908,10 @@ def format_wildcards(self, string, **variables): raise RuleException("NameError: " + str(ex), rule=self.rule) except IndexError as ex: raise RuleException("IndexError: " + str(ex), rule=self.rule) + except Exception as ex: + raise WorkflowError( + f"Error when formatting '{string}' for rule {self.rule.name}. {ex}" + ) def properties(self, omit_resources=["_cores", "_nodes"], **aux_properties): resources = { @@ -1469,6 +1477,10 @@ def format_wildcards(self, string, **variables): raise WorkflowError( "IndexError with group job {}: {}".format(self.jobid, str(ex)) ) + except Exception as ex: + raise WorkflowError( + f"Error when formatting {string} for group job {self.jobid}: {ex}" + ) @property def threads(self): diff --git a/tests/test_incomplete_params/Snakefile b/tests/test_incomplete_params/Snakefile new file mode 100644 index 000000000..342775788 --- /dev/null +++ b/tests/test_incomplete_params/Snakefile @@ -0,0 +1,27 @@ +def get_x(wildcards, input): + with open(input[0]) as infile: + return {"foo": infile.read()} + + +def get_mem_mb(wildcards, input): + return os.path.getsize(input[0]) / 1024.0 + + +rule a: + input: + "test.in", + output: + "test.out", + params: + x=get_x, + resources: + mem_mb=get_mem_mb, + shell: + "echo {params.x[foo]} > {output}" + + +rule b: + output: + "test.in", + shell: + "touch {output}" diff --git a/tests/test_incomplete_params/expected-results/.gitkeep b/tests/test_incomplete_params/expected-results/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/tests.py b/tests/tests.py index 4908b1f93..f95c2109e 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -1561,3 +1561,7 @@ def test_groupid_expand_cluster(): @skip_on_windows def test_service_jobs(): run(dpath("test_service_jobs"), check_md5=False) + + +def test_incomplete_params(): + run(dpath("test_incomplete_params"), dryrun=True, printshellcmds=True)