From 9325f0a1358e03eedfe672841535402f6cccb2db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Wed, 16 Mar 2022 11:46:02 +0100 Subject: [PATCH 1/2] fix: do not skip local conda env creation per se when having no shared FS, because it is still needed for local jobs. Instead, decide for each env whether it is needed locally or not. --- snakemake/workflow.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/snakemake/workflow.py b/snakemake/workflow.py index a4ddc278f..a03b05ed3 100644 --- a/snakemake/workflow.py +++ b/snakemake/workflow.py @@ -965,11 +965,10 @@ def files(items): dryrun=dryrun or list_conda_envs, quiet=list_conda_envs ) if self.use_conda: - if self.assume_shared_fs: - dag.create_conda_envs( - dryrun=dryrun or list_conda_envs or conda_cleanup_envs, - quiet=list_conda_envs, - ) + dag.create_conda_envs( + dryrun=dryrun or list_conda_envs or conda_cleanup_envs, + quiet=list_conda_envs, + ) if conda_create_envs_only: return True From c52594959812cc8cdf5c8c3ec2dc5ddc88414e83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Wed, 16 Mar 2022 11:49:10 +0100 Subject: [PATCH 2/2] extend test case to cover this issue --- tests/test_tibanna/Snakefile | 49 +++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/tests/test_tibanna/Snakefile b/tests/test_tibanna/Snakefile index 8717edc58..61faeb810 100644 --- a/tests/test_tibanna/Snakefile +++ b/tests/test_tibanna/Snakefile @@ -9,73 +9,88 @@ # Output of step2 is set remote. from snakemake.remote.S3 import RemoteProvider as S3RemoteProvider + S3 = S3RemoteProvider() -#envvars: +# envvars: # "TEST_ENVVAR1", # "TEST_ENVVAR2" + +localrules: + all, + + rule all: + conda: + "env.yml" input: - final = S3.remote("snakemake-tibanna-test/1/final_message"), + final=S3.remote("snakemake-tibanna-test/1/final_message"), # final = "snakemake-tibanna-test/1/final_message" - final2 = S3.remote("snakemake-tibanna-test2/1/final_message") + final2=S3.remote("snakemake-tibanna-test2/1/final_message"), + shell: + "echo test" rule step1: conda: "env.yml" - group: "mygroup" + group: + "mygroup" input: - S3.remote("snakemake-tibanna-test/1/somefile") + S3.remote("snakemake-tibanna-test/1/somefile"), output: - message = "message1" + message="message1", shell: """ echo {config[message]} > message1 cat snakemake-tibanna-test/1/somefile >> message1 """ + rule step1a: conda: "env.yml" - group: "mygroup" + group: + "mygroup" input: - message = "message1" + message="message1", output: - next_message = "next_message" + next_message="next_message", shell: """ cat message1 message1 > next_message """ + rule step1b: conda: "env.yml" input: - S3.remote("snakemake-tibanna-test/1/somefile2") + S3.remote("snakemake-tibanna-test/1/somefile2"), output: - message = "message2" + message="message2", log: - "log.txt" + "log.txt", benchmark: "benchmark.txt" threads: 1 resources: - mem_mb=1024 + mem_mb=1024, shell: """ echo "abcdefg" > message2 cat snakemake-tibanna-test/1/somefile2 >> message2 """ + rule step2: input: - message1 = "next_message", - message2 = "message2" + message1="next_message", + message2="message2", output: - final = S3.remote("snakemake-tibanna-test/1/final_message"), + final=S3.remote("snakemake-tibanna-test/1/final_message"), # final = "snakemake-tibanna-test/1/final_message" - final2 = S3.remote("snakemake-tibanna-test2/1/final_message") + final2=S3.remote("snakemake-tibanna-test2/1/final_message"), script: "scripts/step2.py"