From 3ac8e858a7b908326922c8f68cae512b1250e906 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Tue, 26 Oct 2021 15:02:43 +0200 Subject: [PATCH] fix: env file usage after changes to source file handling (inspired by #1233 and #1211). (#1236) * fix: more informative nothing to be done message * fmt * fmt, fixes and additional tests * implement fspath --- snakemake/deployment/containerize.py | 16 ++++-- snakemake/sourcecache.py | 3 ++ snakemake/workflow.py | 2 +- tests/common.py | 74 ++++++++++++++-------------- tests/tests.py | 9 ++++ 5 files changed, 62 insertions(+), 42 deletions(-) diff --git a/snakemake/deployment/containerize.py b/snakemake/deployment/containerize.py index c4a831801..755133826 100644 --- a/snakemake/deployment/containerize.py +++ b/snakemake/deployment/containerize.py @@ -7,6 +7,7 @@ from snakemake.exceptions import WorkflowError from snakemake.deployment import conda from snakemake.logging import logger +from snakemake.sourcecache import LocalSourceFile CONDA_ENV_PATH = "/conda-envs" @@ -22,7 +23,11 @@ def containerize(workflow): "Containerization of conda based workflows is not allowed if any conda env definition contains a wildcard." ) - relfile = lambda env: os.path.relpath(env.file, os.getcwd()) + def relfile(env): + if isinstance(env.file, LocalSourceFile): + return os.path.relpath(env.file.get_path_or_uri(), os.getcwd()) + else: + return env.file.get_path_or_uri() envs = sorted( set( @@ -59,11 +64,12 @@ def containerize(workflow): "\n".join(map("# {}".format, env.content.decode().strip().split("\n"))) ) get_env_cmds.append("RUN mkdir -p {}".format(prefix)) - if env.file.startswith("https://"): - # get_env_cmds.append("RUN curl -sSL {} > {}".format(env.file, env_target_path)) - get_env_cmds.append("ADD {} {}".format(env.file, env_target_path)) - else: + if isinstance(env.file, LocalSourceFile): get_env_cmds.append("COPY {} {}".format(env_source_path, env_target_path)) + else: + get_env_cmds.append( + "ADD {} {}".format(env.file.get_path_or_uri(), env_target_path) + ) generate_env_cmds.append( "mamba env create --prefix {} --file {} &&".format(prefix, env_target_path) diff --git a/snakemake/sourcecache.py b/snakemake/sourcecache.py index f5fd4658e..c8dab9c88 100644 --- a/snakemake/sourcecache.py +++ b/snakemake/sourcecache.py @@ -105,6 +105,9 @@ def isabs(self): def simplify_path(self): return utils.simplify_path(self.path) + def __fspath__(self): + return self.path + class LocalGitFile(SourceFile): def __init__( diff --git a/snakemake/workflow.py b/snakemake/workflow.py index eeaa171a6..93121023d 100644 --- a/snakemake/workflow.py +++ b/snakemake/workflow.py @@ -958,7 +958,7 @@ def files(items): for env in set(job.conda_env for job in dag.jobs): if env: print( - simplify_path(env.file), + env.file.simplify_path(), env.container_img_url or "", simplify_path(env.path), sep="\t", diff --git a/tests/common.py b/tests/common.py index faca55eb3..f1a179fcc 100644 --- a/tests/common.py +++ b/tests/common.py @@ -88,6 +88,7 @@ def run( subpath=None, no_tmpdir=False, check_md5=True, + check_results=True, cores=3, set_pythonpath=True, cleanup=True, @@ -181,42 +182,43 @@ def run( assert not success, "expected error on execution" else: assert success, "expected successful execution" - for resultfile in get_expected_files(results_dir): - if resultfile in [".gitignore", ".gitkeep"] or not os.path.isfile( - os.path.join(results_dir, resultfile) - ): - # this means tests cannot use directories as output files - continue - targetfile = join(tmpdir, resultfile) - expectedfile = join(results_dir, resultfile) - - if ON_WINDOWS: - if os.path.exists(join(results_dir, resultfile + "_WIN")): - continue # Skip test if a Windows specific file exists - if resultfile.endswith("_WIN"): - targetfile = join(tmpdir, resultfile[:-4]) - elif resultfile.endswith("_WIN"): - # Skip win specific result files on Posix platforms - continue - - assert os.path.exists(targetfile), 'expected file "{}" not produced'.format( - resultfile - ) - if check_md5: - md5expected = md5sum(expectedfile, ignore_newlines=ON_WINDOWS) - md5target = md5sum(targetfile, ignore_newlines=ON_WINDOWS) - if md5target != md5expected: - with open(expectedfile) as expected: - expected_content = expected.read() - with open(targetfile) as target: - content = target.read() - assert ( - False - ), "wrong result produced for file '{resultfile}':\n------found------\n{content}\n-----expected-----\n{expected_content}\n-----------------".format( - resultfile=resultfile, - content=content, - expected_content=expected_content, - ) + if check_results: + for resultfile in get_expected_files(results_dir): + if resultfile in [".gitignore", ".gitkeep"] or not os.path.isfile( + os.path.join(results_dir, resultfile) + ): + # this means tests cannot use directories as output files + continue + targetfile = join(tmpdir, resultfile) + expectedfile = join(results_dir, resultfile) + + if ON_WINDOWS: + if os.path.exists(join(results_dir, resultfile + "_WIN")): + continue # Skip test if a Windows specific file exists + if resultfile.endswith("_WIN"): + targetfile = join(tmpdir, resultfile[:-4]) + elif resultfile.endswith("_WIN"): + # Skip win specific result files on Posix platforms + continue + + assert os.path.exists( + targetfile + ), 'expected file "{}" not produced'.format(resultfile) + if check_md5: + md5expected = md5sum(expectedfile, ignore_newlines=ON_WINDOWS) + md5target = md5sum(targetfile, ignore_newlines=ON_WINDOWS) + if md5target != md5expected: + with open(expectedfile) as expected: + expected_content = expected.read() + with open(targetfile) as target: + content = target.read() + assert ( + False + ), "wrong result produced for file '{resultfile}':\n------found------\n{content}\n-----expected-----\n{expected_content}\n-----------------".format( + resultfile=resultfile, + content=content, + expected_content=expected_content, + ) if not cleanup: return tmpdir diff --git a/tests/tests.py b/tests/tests.py index f1ec34c28..0fbea1540 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -409,6 +409,10 @@ def test_conda(): run(dpath("test_conda"), use_conda=True) +def test_conda_list_envs(): + run(dpath("test_conda"), list_conda_envs=True, check_results=False) + + def test_upstream_conda(): run(dpath("test_conda"), use_conda=True, conda_frontend="conda") @@ -1230,6 +1234,11 @@ def test_containerized(): run(dpath("test_containerized"), use_conda=True, use_singularity=True) +@skip_on_windows +def test_containerize(): + run(dpath("test_conda"), containerize=True, check_results=False) + + def test_long_shell(): run(dpath("test_long_shell"))