Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support for post deploy scripts #1325

Merged
merged 18 commits into from Jan 14, 2022
Merged
24 changes: 24 additions & 0 deletions docs/snakefiles/deployment.rst
Expand Up @@ -262,6 +262,30 @@ Conda deployment also works well for offline or air-gapped environments. Running

.. _singularity:


-------------------------
Providing post-deployment scripts
-------------------------

With Snakemake 9.14 post-deployment shell-scripts can be provided to perform additional adjustments of a conda environment.
FelixMoelder marked this conversation as resolved.
Show resolved Hide resolved
This might be helpful in case a conda package is missing components or requires further configuration for execution.
post-deployment scripts must be placed next to their corresponding environment-file and require the suffix ``.post-deploy.sh``.
FelixMoelder marked this conversation as resolved.
Show resolved Hide resolved

.. code-block:: python

rule NAME:
input:
"seqs.fastq"
output:
"results.tsv"
conda:
"envs/interproscan.yaml"
"envs/interproscan.post-deploy.sh
FelixMoelder marked this conversation as resolved.
Show resolved Hide resolved
shell:
"interproscan.sh -i {input} -f tsv -o {output}"

The path of the conda environment can be accessed within the script via ``$CONDA_PREFIX``.

--------------------------
Running jobs in containers
--------------------------
Expand Down
28 changes: 28 additions & 0 deletions snakemake/deployment/conda.py
Expand Up @@ -4,6 +4,7 @@
__license__ = "MIT"

import os
from pathlib import Path
import re
from snakemake.sourcecache import LocalGitFile, LocalSourceFile, infer_source_file
import subprocess
Expand Down Expand Up @@ -211,6 +212,29 @@ def create_archive(self):
raise e
return env_archive

def execute_deployment_script(self, env_file):
"""Execute post-deployment script if present"""
from snakemake.shell import shell

post_deploy_script = Path(env_file).with_suffix(".post-deploy.sh")
FelixMoelder marked this conversation as resolved.
Show resolved Hide resolved
if post_deploy_script.exists():
if ON_WINDOWS:
raise WorkflowError(
"Post deploy script {} provided for conda env {} but unsupported on windows.".format(
post_deploy_script, env_file
)
)
logger.info(
"Running post-deploy script {}...".format(
post_deploy_script.relative_to(os.getcwd())
)
)
conda = Conda(self._container_img)
shell.check_output(
conda.shellcmd(self.path, "sh {}".format(post_deploy_script)),
stderr=subprocess.STDOUT,
)

def create(self, dryrun=False):
"""Create the conda enviroment."""
from snakemake.shell import shell
Expand Down Expand Up @@ -374,6 +398,10 @@ def create(self, dryrun=False):
"Cleaning up conda package tarballs and package cache."
)
shell.check_output("conda clean -y --tarballs --packages")

# Execute post-deplay script if present
self.execute_deployment_script(env_file)

# Touch "done" flag file
with open(os.path.join(env_path, "env_setup_done"), "a") as f:
pass
Expand Down
17 changes: 17 additions & 0 deletions tests/test_deploy_script/Snakefile
@@ -0,0 +1,17 @@
shell.executable("bash")

rule all:
input:
"test.out"

rule a:
output:
"test.out"
conda:
"test-env.yaml"
shell:
"""
if [ -f $CONDA_PREFIX/test.txt ] ;then
touch {output}
fi
"""
Empty file.
3 changes: 3 additions & 0 deletions tests/test_deploy_script/test-env.post-deploy.sh
@@ -0,0 +1,3 @@
#!/bin/bash

touch $CONDA_PREFIX/test.txt
5 changes: 5 additions & 0 deletions tests/test_deploy_script/test-env.yaml
@@ -0,0 +1,5 @@
channels:
- bioconda
- conda-forge
dependencies:
- python <3.10
5 changes: 5 additions & 0 deletions tests/tests.py
Expand Up @@ -429,6 +429,11 @@ def test_upstream_conda():
run(dpath("test_conda"), use_conda=True, conda_frontend="conda")


@skip_on_windows
def test_deploy_script():
run(dpath("test_deploy_script"), use_conda=True)


def test_conda_custom_prefix():
run(
dpath("test_conda_custom_prefix"),
Expand Down