Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: consider post-deploy script for env hashing #1363

Merged
merged 9 commits into from Jan 31, 2022
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
24 changes: 15 additions & 9 deletions snakemake/deployment/conda.py
Expand Up @@ -62,11 +62,15 @@ def __init__(
):
self.file = None
self.name = None
self.post_deploy_file = None
if env_file is not None:
self.file = infer_source_file(env_file)
if env_name is not None:
assert env_file is None, "bug: both env_file and env_name specified"
self.name = env_name
deploy_file = Path(self.file.get_path_or_uri()).with_suffix(".post-deploy.sh")
if deploy_file.exists():
self.post_deploy_file = infer_source_file(deploy_file)

self.frontend = workflow.conda_frontend
self.workflow = workflow
Expand Down Expand Up @@ -101,8 +105,9 @@ def _get_content(self):

def _get_content_deploy(self):
self.check_is_file_based()
deploy_file = Path(self.file).with_suffix(".post-deploy.sh")
return self.workflow.sourcecache.open(deploy_file, "rb").read()
if self.post_deploy_file:
return self.workflow.sourcecache.open(self.post_deploy_file, "rb").read()
return None

@property
def _env_archive_dir(self):
Expand Down Expand Up @@ -139,6 +144,9 @@ def hash(self):
md5hash.update(env_dir.encode())
if self._container_img:
md5hash.update(self._container_img.url.encode())
content_deploy = self.content_deploy
if content_deploy:
md5hash.update(content_deploy)
md5hash.update(self.content)
self._hash = md5hash.hexdigest()
return self._hash
Expand All @@ -148,6 +156,9 @@ def content_hash(self):
if self._content_hash is None:
md5hash = hashlib.md5()
md5hash.update(self.content)
content_deploy = self.content_deploy
if content_deploy:
md5hash.update(content_deploy)
self._content_hash = md5hash.hexdigest()
return self._content_hash

Expand Down Expand Up @@ -317,11 +328,7 @@ def create(self, dryrun=False):
tmp.write(self.content)
env_file = tmp.name
tmp_env_file = tmp.name
if (
Path(self.file.get_path_or_uri())
.with_suffix(".post-deploy.sh")
.exists()
):
if self.post_deploy_file:
with tempfile.NamedTemporaryFile(
delete=False, suffix=".post-deploy.sh"
) as tmp:
Expand All @@ -331,8 +338,7 @@ def create(self, dryrun=False):
tmp_deploy_file = tmp.name
else:
env_file = env_file.get_path_or_uri()
if Path(env_file).with_suffix(".post-deploy.sh").exists():
deploy_file = Path(env_file).with_suffix(".post-deploy.sh")
deploy_file = self.post_deploy_file

env_path = self.address

Expand Down
19 changes: 19 additions & 0 deletions tests/test_deploy_hashing/Snakefile
@@ -0,0 +1,19 @@
rule all:
input:
expand("{s}.txt", s=["a", "b"])

rule a:
output:
"a.txt"
conda:
"a.yaml"
shell:
"touch {output}"

rule b:
output:
"b.txt"
conda:
"b.yaml"
shell:
"touch {output}"
3 changes: 3 additions & 0 deletions tests/test_deploy_hashing/a.post-deploy.sh
@@ -0,0 +1,3 @@
#!/bin/bash

echo "test" > $CONDA_PREFIX/a.txt
5 changes: 5 additions & 0 deletions tests/test_deploy_hashing/a.yaml
@@ -0,0 +1,5 @@
channels:
- bioconda
- conda-forge
dependencies:
- python =3.10
5 changes: 5 additions & 0 deletions tests/test_deploy_hashing/b.yaml
@@ -0,0 +1,5 @@
channels:
- bioconda
- conda-forge
dependencies:
- python =3.10
Empty file.
Empty file.
6 changes: 6 additions & 0 deletions tests/tests.py
Expand Up @@ -434,6 +434,12 @@ def test_deploy_script():
run(dpath("test_deploy_script"), use_conda=True)


@skip_on_windows
def test_deploy_hashing():
tmpdir = run(dpath("test_deploy_hashing"), use_conda=True, cleanup=False)
assert len(next(os.walk(os.path.join(tmpdir, ".snakemake/conda")))[1]) == 2


def test_conda_custom_prefix():
run(
dpath("test_conda_custom_prefix"),
Expand Down