From 58739f82e1e35a831ae9a0f4de828cfd59d8606e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20M=C3=B6lder?= Date: Mon, 31 Jan 2022 08:40:25 +0100 Subject: [PATCH 1/9] fix: hash post-deploy scripts --- snakemake/deployment/conda.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/snakemake/deployment/conda.py b/snakemake/deployment/conda.py index 59a91a556..5bab3902d 100644 --- a/snakemake/deployment/conda.py +++ b/snakemake/deployment/conda.py @@ -102,7 +102,11 @@ def _get_content(self): def _get_content_deploy(self): self.check_is_file_based() deploy_file = Path(self.file).with_suffix(".post-deploy.sh") - return self.workflow.sourcecache.open(deploy_file, "rb").read() + if deploy_file.exists(): + return self.workflow.sourcecache.open( + infer_source_file(deploy_file), "rb" + ).read() + return None @property def _env_archive_dir(self): @@ -139,6 +143,9 @@ def hash(self): md5hash.update(env_dir.encode()) if self._container_img: md5hash.update(self._container_img.url.encode()) + content_deploy = self.content_deploy + if content_deploy: + md5hash.update(self.content_deploy) md5hash.update(self.content) self._hash = md5hash.hexdigest() return self._hash @@ -148,6 +155,9 @@ def content_hash(self): if self._content_hash is None: md5hash = hashlib.md5() md5hash.update(self.content) + content_deploy = self.content_deploy + if content_deploy: + md5hash.update(content_deploy) self._content_hash = md5hash.hexdigest() return self._content_hash From b49cebe561506eb59ad0960061fb0d71e5a5ff36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20M=C3=B6lder?= Date: Mon, 31 Jan 2022 09:20:29 +0100 Subject: [PATCH 2/9] Add testcase --- tests/test_deploy_hashing/Snakefile | 19 +++++++++++++++++++ tests/test_deploy_hashing/a.post-deploy.sh | 3 +++ tests/test_deploy_hashing/a.yaml | 5 +++++ tests/test_deploy_hashing/b.yaml | 5 +++++ .../expected-results/a.txt | 0 .../expected-results/b.txt | 0 tests/tests.py | 7 +++++++ 7 files changed, 39 insertions(+) create mode 100644 tests/test_deploy_hashing/Snakefile create mode 100644 tests/test_deploy_hashing/a.post-deploy.sh create mode 100644 tests/test_deploy_hashing/a.yaml create mode 100644 tests/test_deploy_hashing/b.yaml create mode 100644 tests/test_deploy_hashing/expected-results/a.txt create mode 100644 tests/test_deploy_hashing/expected-results/b.txt diff --git a/tests/test_deploy_hashing/Snakefile b/tests/test_deploy_hashing/Snakefile new file mode 100644 index 000000000..9fe47395a --- /dev/null +++ b/tests/test_deploy_hashing/Snakefile @@ -0,0 +1,19 @@ +rule all: + input: + expand("{s}.txt", s=["a", "b"]) + +rule a: + output: + "a.txt" + conda: + "a.yaml" + shell: + "touch {output}" + +rule b: + output: + "b.txt" + conda: + "b.yaml" + shell: + "touch {output}" \ No newline at end of file diff --git a/tests/test_deploy_hashing/a.post-deploy.sh b/tests/test_deploy_hashing/a.post-deploy.sh new file mode 100644 index 000000000..ebca5bab2 --- /dev/null +++ b/tests/test_deploy_hashing/a.post-deploy.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "test" > $CONDA_PREFIX/a.txt \ No newline at end of file diff --git a/tests/test_deploy_hashing/a.yaml b/tests/test_deploy_hashing/a.yaml new file mode 100644 index 000000000..50d99e291 --- /dev/null +++ b/tests/test_deploy_hashing/a.yaml @@ -0,0 +1,5 @@ +channels: + - bioconda + - conda-forge +dependencies: + - python =3.10 diff --git a/tests/test_deploy_hashing/b.yaml b/tests/test_deploy_hashing/b.yaml new file mode 100644 index 000000000..50d99e291 --- /dev/null +++ b/tests/test_deploy_hashing/b.yaml @@ -0,0 +1,5 @@ +channels: + - bioconda + - conda-forge +dependencies: + - python =3.10 diff --git a/tests/test_deploy_hashing/expected-results/a.txt b/tests/test_deploy_hashing/expected-results/a.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_deploy_hashing/expected-results/b.txt b/tests/test_deploy_hashing/expected-results/b.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tests/tests.py b/tests/tests.py index 2e54c4c2f..ba4a43fc4 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -434,6 +434,13 @@ def test_deploy_script(): run(dpath("test_deploy_script"), use_conda=True) +@skip_on_windows +def test_deploy_hashing(): + tmpdir = run(dpath("test_deploy_hashing"), use_conda=True, cleanup=False) + print(f"Bla {tmpdir}") + assert len(next(os.walk(os.path.join(tmpdir, ".snakemake/conda")))[1]) == 2 + + def test_conda_custom_prefix(): run( dpath("test_conda_custom_prefix"), From a68f41e2e4dae0f41145a26943535113fa0678dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20M=C3=B6lder?= Date: Mon, 31 Jan 2022 09:31:43 +0100 Subject: [PATCH 3/9] Remove printing --- tests/tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/tests.py b/tests/tests.py index ba4a43fc4..293c55747 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -437,7 +437,6 @@ def test_deploy_script(): @skip_on_windows def test_deploy_hashing(): tmpdir = run(dpath("test_deploy_hashing"), use_conda=True, cleanup=False) - print(f"Bla {tmpdir}") assert len(next(os.walk(os.path.join(tmpdir, ".snakemake/conda")))[1]) == 2 From 1473afb081959cd4191943a4bbca9dc376ee8804 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20M=C3=B6lder?= Date: Mon, 31 Jan 2022 09:37:12 +0100 Subject: [PATCH 4/9] removed redundant call --- snakemake/deployment/conda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snakemake/deployment/conda.py b/snakemake/deployment/conda.py index 5bab3902d..3184f4bab 100644 --- a/snakemake/deployment/conda.py +++ b/snakemake/deployment/conda.py @@ -145,7 +145,7 @@ def hash(self): md5hash.update(self._container_img.url.encode()) content_deploy = self.content_deploy if content_deploy: - md5hash.update(self.content_deploy) + md5hash.update(content_deploy) md5hash.update(self.content) self._hash = md5hash.hexdigest() return self._hash From 2f3ec3c1380e54ef8908aaf0d7b08604a3b99a6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20M=C3=B6lder?= Date: Mon, 31 Jan 2022 10:07:23 +0100 Subject: [PATCH 5/9] fix wrapper test --- snakemake/deployment/conda.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/snakemake/deployment/conda.py b/snakemake/deployment/conda.py index 3184f4bab..815c22e08 100644 --- a/snakemake/deployment/conda.py +++ b/snakemake/deployment/conda.py @@ -101,7 +101,8 @@ def _get_content(self): def _get_content_deploy(self): self.check_is_file_based() - deploy_file = Path(self.file).with_suffix(".post-deploy.sh") + print(str(self.file)) + deploy_file = Path(str(self.file)).with_suffix(".post-deploy.sh") if deploy_file.exists(): return self.workflow.sourcecache.open( infer_source_file(deploy_file), "rb" From 2f6bf14cbbb1673c146ca9943018c4e36a6d817b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20M=C3=B6lder?= Date: Mon, 31 Jan 2022 10:07:35 +0100 Subject: [PATCH 6/9] fix wrapper test --- snakemake/deployment/conda.py | 1 - 1 file changed, 1 deletion(-) diff --git a/snakemake/deployment/conda.py b/snakemake/deployment/conda.py index 815c22e08..4709a10e4 100644 --- a/snakemake/deployment/conda.py +++ b/snakemake/deployment/conda.py @@ -101,7 +101,6 @@ def _get_content(self): def _get_content_deploy(self): self.check_is_file_based() - print(str(self.file)) deploy_file = Path(str(self.file)).with_suffix(".post-deploy.sh") if deploy_file.exists(): return self.workflow.sourcecache.open( From 2e954a4757faf885fa753ae3de384953d56bcc25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20M=C3=B6lder?= Date: Mon, 31 Jan 2022 10:08:56 +0100 Subject: [PATCH 7/9] propper fix --- snakemake/deployment/conda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snakemake/deployment/conda.py b/snakemake/deployment/conda.py index 4709a10e4..108e71e3e 100644 --- a/snakemake/deployment/conda.py +++ b/snakemake/deployment/conda.py @@ -101,7 +101,7 @@ def _get_content(self): def _get_content_deploy(self): self.check_is_file_based() - deploy_file = Path(str(self.file)).with_suffix(".post-deploy.sh") + deploy_file = Path(self.file.get_path_or_uri()).with_suffix(".post-deploy.sh") if deploy_file.exists(): return self.workflow.sourcecache.open( infer_source_file(deploy_file), "rb" From d4bbbdfdc9009a361a906af3331c84e373060f79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20M=C3=B6lder?= Date: Mon, 31 Jan 2022 10:31:00 +0100 Subject: [PATCH 8/9] Refactoring --- snakemake/deployment/conda.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/snakemake/deployment/conda.py b/snakemake/deployment/conda.py index 108e71e3e..2c44f4821 100644 --- a/snakemake/deployment/conda.py +++ b/snakemake/deployment/conda.py @@ -62,11 +62,15 @@ def __init__( ): self.file = None self.name = None + self.post_deploy_file = None if env_file is not None: self.file = infer_source_file(env_file) if env_name is not None: assert env_file is None, "bug: both env_file and env_name specified" self.name = env_name + deploy_file = Path(self.file.get_path_or_uri()).with_suffix(".post-deploy.sh") + if deploy_file.exists(): + self.post_deploy_file = infer_source_file(deploy_file) self.frontend = workflow.conda_frontend self.workflow = workflow @@ -101,11 +105,8 @@ def _get_content(self): def _get_content_deploy(self): self.check_is_file_based() - deploy_file = Path(self.file.get_path_or_uri()).with_suffix(".post-deploy.sh") - if deploy_file.exists(): - return self.workflow.sourcecache.open( - infer_source_file(deploy_file), "rb" - ).read() + if self.post_deploy_file: + return self.workflow.sourcecache.open(self.post_deploy_file, "rb").read() return None @property @@ -327,11 +328,7 @@ def create(self, dryrun=False): tmp.write(self.content) env_file = tmp.name tmp_env_file = tmp.name - if ( - Path(self.file.get_path_or_uri()) - .with_suffix(".post-deploy.sh") - .exists() - ): + if self.post_deploy_file: with tempfile.NamedTemporaryFile( delete=False, suffix=".post-deploy.sh" ) as tmp: @@ -341,8 +338,7 @@ def create(self, dryrun=False): tmp_deploy_file = tmp.name else: env_file = env_file.get_path_or_uri() - if Path(env_file).with_suffix(".post-deploy.sh").exists(): - deploy_file = Path(env_file).with_suffix(".post-deploy.sh") + deploy_file = self.post_deploy_file env_path = self.address From 6c29cdef7bd7ad4fca07e04b60570d8bbac978d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20M=C3=B6lder?= Date: Mon, 31 Jan 2022 11:08:01 +0100 Subject: [PATCH 9/9] limit to env file --- snakemake/deployment/conda.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/snakemake/deployment/conda.py b/snakemake/deployment/conda.py index 2c44f4821..8bb7327b9 100644 --- a/snakemake/deployment/conda.py +++ b/snakemake/deployment/conda.py @@ -65,12 +65,14 @@ def __init__( self.post_deploy_file = None if env_file is not None: self.file = infer_source_file(env_file) + deploy_file = Path(self.file.get_path_or_uri()).with_suffix( + ".post-deploy.sh" + ) + if deploy_file.exists(): + self.post_deploy_file = infer_source_file(deploy_file) if env_name is not None: assert env_file is None, "bug: both env_file and env_name specified" self.name = env_name - deploy_file = Path(self.file.get_path_or_uri()).with_suffix(".post-deploy.sh") - if deploy_file.exists(): - self.post_deploy_file = infer_source_file(deploy_file) self.frontend = workflow.conda_frontend self.workflow = workflow