feat: support for post deploy scripts (#1325)

* feat: support for post deploy scripts (skeleton)

* fixes

* fmt with black

* Add info message

* Add documentation

* move code

* Moved to method

* Fix variable

* Add test

* Add result

* Add missing folder

* Fix name

* Fixed typo

* Update docs

* write to tmp

* fix patho

* transform to path

Co-authored-by: Felix Mölder <felix.moelder@uni-due.de>

docs/snakefiles/deployment.rst

@@ -262,6 +262,36 @@ Conda deployment also works well for offline or air-gapped environments. Running
 
 .. _singularity:
 
+
+-------------------------
+Providing post-deployment scripts
+-------------------------
+
+From Snakemake 6.14 onwards post-deployment shell-scripts can be provided to perform additional adjustments of a conda environment.
+This might be helpful in case a conda package is missing components or requires further configuration for execution.
+Post-deployment scripts must be placed next to their corresponding environment-file and require the suffix ``.post-deploy.sh``, e.g.:
+
+.. code-block:: python
+
+    rule NAME:
+        input:
+            "seqs.fastq"
+        output:
+            "results.tsv"
+        conda:
+            "envs/interproscan.yaml"
+        shell:
+            "interproscan.sh -i {input} -f tsv -o {output}"
+
+.. code-block:: none
+
+    ├── Snakefile
+    └── envs
+        ├── interproscan.yaml
+        └── interproscan.post-deploy.sh
+
+The path of the conda environment can be accessed within the script via ``$CONDA_PREFIX``.
+
 --------------------------
 Running jobs in containers
 --------------------------

snakemake/deployment/conda.py

@@ -4,6 +4,7 @@
 __license__ = "MIT"
 
 import os
+from pathlib import Path
 import re
 from snakemake.sourcecache import LocalGitFile, LocalSourceFile, infer_source_file
 import subprocess
@@ -60,6 +61,7 @@ def __init__(
         self._hash = None
         self._content_hash = None
         self._content = None
+        self._content_deploy = None
         self._path = None
         self._archive_file = None
         self._cleanup = cleanup
@@ -68,6 +70,10 @@ def __init__(
     def _get_content(self):
         return self.workflow.sourcecache.open(self.file, "rb").read()
 
+    def _get_content_deploy(self):
+        deploy_file = Path(self.file).with_suffix(".post-deploy.sh")
+        return self.workflow.sourcecache.open(deploy_file, "rb").read()
+
     @property
     def _env_archive_dir(self):
         return self.workflow.persistence.conda_env_archive_path
@@ -82,6 +88,12 @@ def content(self):
             self._content = self._get_content()
         return self._content
 
+    @property
+    def content_deploy(self):
+        if self._content_deploy is None:
+            self._content_deploy = self._get_content_deploy()
+        return self._content_deploy
+
     @property
     def hash(self):
         if self._hash is None:
@@ -211,13 +223,36 @@ def create_archive(self):
             raise e
         return env_archive
 
+    def execute_deployment_script(self, env_file, deploy_file):
+        """Execute post-deployment script if present"""
+        from snakemake.shell import shell
+
+        if ON_WINDOWS:
+            raise WorkflowError(
+                "Post deploy script {} provided for conda env {} but unsupported on windows.".format(
+                    deploy_file, env_file
+                )
+            )
+        logger.info(
+            "Running post-deploy script {}...".format(
+                Path(deploy_file).relative_to(os.getcwd())
+            )
+        )
+        conda = Conda(self._container_img)
+        shell.check_output(
+            conda.shellcmd(self.path, "sh {}".format(deploy_file)),
+            stderr=subprocess.STDOUT,
+        )
+
     def create(self, dryrun=False):
         """Create the conda enviroment."""
         from snakemake.shell import shell
 
         # Read env file and create hash.
         env_file = self.file
-        tmp_file = None
+        deploy_file = None
+        tmp_env_file = None
+        tmp_deploy_file = None
 
         if not isinstance(env_file, LocalSourceFile) or isinstance(
             env_file, LocalGitFile
@@ -226,9 +261,23 @@ def create(self, dryrun=False):
                 # write to temp file such that conda can open it
                 tmp.write(self.content)
                 env_file = tmp.name
-                tmp_file = tmp.name
+                tmp_env_file = tmp.name
+            if (
+                Path(self.file.get_path_or_uri())
+                .with_suffix(".post-deploy.sh")
+                .exists()
+            ):
+                with tempfile.NamedTemporaryFile(
+                    delete=False, suffix=".post-deploy.sh"
+                ) as tmp:
+                    # write to temp file such that conda can open it
+                    tmp.write(self.content_deploy)
+                    deploy_file = tmp.name
+                    tmp_deploy_file = tmp.name
         else:
             env_file = env_file.get_path_or_uri()
+            if Path(env_file).with_suffix(".post-deploy.sh").exists():
+                deploy_file = Path(env_file).with_suffix(".post-deploy.sh")
 
         env_hash = self.hash
         env_path = self.path
@@ -374,6 +423,13 @@ def create(self, dryrun=False):
                             "Cleaning up conda package tarballs and package cache."
                         )
                         shell.check_output("conda clean -y --tarballs --packages")
+
+                # Execute post-deplay script if present
+                if deploy_file:
+                    target_deploy_file = env_path + ".post-deploy.sh"
+                    shutil.copy(deploy_file, target_deploy_file)
+                    self.execute_deployment_script(env_file, target_deploy_file)
+
                 # Touch "done" flag file
                 with open(os.path.join(env_path, "env_setup_done"), "a") as f:
                     pass
@@ -392,9 +448,11 @@ def create(self, dryrun=False):
                     + e.output
                 )
 
-        if tmp_file:
+        if tmp_env_file:
             # temporary file was created
-            os.remove(tmp_file)
+            os.remove(tmp_env_file)
+        if tmp_deploy_file:
+            os.remove(tmp_deploy_file)
 
         return env_path
 

tests/test_deploy_script/Snakefile

@@ -0,0 +1,17 @@
+shell.executable("bash")
+
+rule all:
+    input:
+        "test.out"
+
+rule a:
+    output:
+        "test.out"
+    conda:
+        "test-env.yaml"
+    shell:
+        """
+        if [ -f $CONDA_PREFIX/test.txt ] ;then
+            touch {output}
+        fi
+        """

tests/test_deploy_script/test-env.post-deploy.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+touch $CONDA_PREFIX/test.txt

tests/test_deploy_script/test-env.yaml

@@ -0,0 +1,5 @@
+channels:
+  - bioconda
+  - conda-forge
+dependencies:
+  - python <3.10

tests/tests.py

@@ -429,6 +429,11 @@ def test_upstream_conda():
     run(dpath("test_conda"), use_conda=True, conda_frontend="conda")
 
 
+@skip_on_windows
+def test_deploy_script():
+    run(dpath("test_deploy_script"), use_conda=True)
+
+
 def test_conda_custom_prefix():
     run(
         dpath("test_conda_custom_prefix"),