feat!: template rendering integration (yte and jinja2), require Pytho…

…n 3.6 as minimum version (f-string support) (#1410) * feat: add template rendering integration (for now via jinja2 and yte) * finalized integration and added testcase * update copyright * Add documentation * fmt * always run template engine jobs in thread * fmt * do not install cwltool on windows
snakemake · Feb 17, 2022 · e1cbde5 · e1cbde5
1 parent b992cd1
commit e1cbde5
Show file tree

Hide file tree

Showing 68 changed files with 265 additions and 77 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -175,7 +175,7 @@ jobs:
         shell: python
         run: |
           import fileinput
-          excluded_on_win = ["environment-modules"]
+          excluded_on_win = ["environment-modules", "cwltool"]
           for line in fileinput.input("test-environment.yml", inplace=True):
               if all(pkg not in line for pkg in excluded_on_win):
                   print(line)

diff --git a/docs/snakefiles/rules.rst b/docs/snakefiles/rules.rst
@@ -1882,3 +1882,42 @@ This can be achieved by accessing their path via the ``workflow.get_source``, wh
             json=workflow.source_path("../resources/test.json")
         shell:
             "somecommand {params.json} > {output}"
+
+
+.. _snakefiles-template-integration:
+
+Template rendering integration
+------------------------------
+
+Sometimes, data analyses entail the dynamic rendering of internal configuration files that are required for certain steps.
+From Snakemake 7 on, such template rendering is directly integrated such that it can happen with minimal code and maximum performance.
+Consider the following example:
+
+.. code-block:: python
+
+    rule render_jinja2_template:
+        input:
+            "some-jinja2-template.txt"
+        output:
+            "results/{sample}.rendered-version.txt"
+        template_engine:
+            "jinja2"
+
+Here, Snakemake will automatically use the specified template engine `Jinja2 <https://jinja.palletsprojects.com/>` to render the template given as input file into the given output file.
+Template rendering rules may only have a single input and output file.
+The template_engine instruction has to be specified at the end of the rule.
+
+Apart from Jinja2, Snakemake supports YTE (YAML template engine), which is particularly designed to support templating of the ubiquitious YAML file format:
+
+.. code-block:: python
+
+    rule render_jinja2_template:
+        input:
+            "some-yte-template.yaml"
+        output:
+            "results/{sample}.rendered-version.yaml"
+        template_engine:
+            "yte"
+
+
+Template rendering rules are always executed locally, without submission to cluster or cloud processes (since templating is usually not resource intensive).
diff --git a/setup.py b/setup.py
@@ -3,16 +3,16 @@
 from __future__ import print_function
 
 __author__ = "Johannes Köster"
-__copyright__ = "Copyright 2015, Johannes Köster"
+__copyright__ = "Copyright 2022, Johannes Köster"
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 
 import sys
 import versioneer
 
 
-if sys.version_info < (3, 5):
-    print("At least Python 3.5 is required for Snakemake.\n", file=sys.stderr)
+if sys.version_info < (3, 6):
+    print("At least Python 3.6 is required for Snakemake.\n", file=sys.stderr)
     exit(1)
 
 
@@ -47,7 +47,8 @@
         "snakemake.linting",
         "snakemake.executors",
         "snakemake.unit_tests",
-        "snakemake.unit_tests.templates"
+        "snakemake.unit_tests.templates",
+        "snakemake.template_rendering",
     ],
     entry_points={
         "console_scripts": [
@@ -76,6 +77,8 @@
         "filelock",
         "stopit",
         "tabulate",
+        "yte",
+        "jinja2",
     ],
     extras_require={
         "reports": ["jinja2", "networkx", "pygments", "pygraphviz"],

diff --git a/snakemake/__init__.py b/snakemake/__init__.py
@@ -1,5 +1,5 @@
 __author__ = "Johannes Köster"
-__copyright__ = "Copyright 2021, Johannes Köster"
+__copyright__ = "Copyright 2022, Johannes Köster"
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 

diff --git a/snakemake/benchmark.py b/snakemake/benchmark.py
@@ -1,5 +1,5 @@
 __author__ = "Manuel Holtgrewe"
-__copyright__ = "Copyright 2017, Manuel Holtgrewe"
+__copyright__ = "Copyright 2022, Manuel Holtgrewe"
 __email__ = "manuel.holtgrewe@bihealth.de"
 __license__ = "MIT"
 

diff --git a/snakemake/caching/__init__.py b/snakemake/caching/__init__.py
@@ -1,5 +1,5 @@
 __authors__ = "Johannes Köster, Sven Nahnsen"
-__copyright__ = "Copyright 2021, Johannes Köster, Sven Nahnsen"
+__copyright__ = "Copyright 2022, Johannes Köster, Sven Nahnsen"
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 

diff --git a/snakemake/caching/hash.py b/snakemake/caching/hash.py
@@ -1,5 +1,5 @@
 __authors__ = "Johannes Köster, Sven Nahnsen"
-__copyright__ = "Copyright 2021, Johannes Köster, Sven Nahnsen"
+__copyright__ = "Copyright 2022, Johannes Köster, Sven Nahnsen"
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 

diff --git a/snakemake/caching/local.py b/snakemake/caching/local.py
@@ -1,5 +1,5 @@
 __authors__ = "Johannes Köster, Sven Nahnsen"
-__copyright__ = "Copyright 2021, Johannes Köster, Sven Nahnsen"
+__copyright__ = "Copyright 2022, Johannes Köster, Sven Nahnsen"
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 

diff --git a/snakemake/caching/remote.py b/snakemake/caching/remote.py
@@ -1,5 +1,5 @@
 __authors__ = "Johannes Köster, Sven Nahnsen"
-__copyright__ = "Copyright 2021, Johannes Köster, Sven Nahnsen"
+__copyright__ = "Copyright 2022, Johannes Köster, Sven Nahnsen"
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 

diff --git a/snakemake/common/__init__.py b/snakemake/common/__init__.py
@@ -1,5 +1,5 @@
 __author__ = "Johannes Köster"
-__copyright__ = "Copyright 2021, Johannes Köster"
+__copyright__ = "Copyright 2022, Johannes Köster"
 __email__ = "johannes.koester@protonmail.com"
 __license__ = "MIT"
 

diff --git a/snakemake/cwl.py b/snakemake/cwl.py
@@ -1,5 +1,5 @@
 __author__ = "Johannes Köster"
-__copyright__ = "Copyright 2021, Johannes Köster"
+__copyright__ = "Copyright 2022, Johannes Köster"
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 

diff --git a/snakemake/dag.py b/snakemake/dag.py
@@ -1,5 +1,5 @@
 __author__ = "Johannes Köster"
-__copyright__ = "Copyright 2021, Johannes Köster"
+__copyright__ = "Copyright 2022, Johannes Köster"
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 

diff --git a/snakemake/decorators.py b/snakemake/decorators.py
@@ -1,5 +1,5 @@
 __author__ = "Christopher Tomkins-Tinch"
-__copyright__ = "Copyright 2021, Christopher Tomkins-Tinch"
+__copyright__ = "Copyright 2022, Christopher Tomkins-Tinch"
 __email__ = "tomkinsc@broadinstitute.org"
 __license__ = "MIT"
 

diff --git a/snakemake/deployment/conda.py b/snakemake/deployment/conda.py
@@ -1,5 +1,5 @@
 __author__ = "Johannes Köster"
-__copyright__ = "Copyright 2021, Johannes Köster"
+__copyright__ = "Copyright 2022, Johannes Köster"
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 

diff --git a/snakemake/deployment/env_modules.py b/snakemake/deployment/env_modules.py
@@ -1,5 +1,5 @@
 __author__ = "Johannes Köster"
-__copyright__ = "Copyright 2021, Johannes Köster"
+__copyright__ = "Copyright 2022, Johannes Köster"
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 

diff --git a/snakemake/deployment/singularity.py b/snakemake/deployment/singularity.py
@@ -1,5 +1,5 @@
 __author__ = "Johannes Köster"
-__copyright__ = "Copyright 2021, Johannes Köster"
+__copyright__ = "Copyright 2022, Johannes Köster"
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 

diff --git a/snakemake/exceptions.py b/snakemake/exceptions.py
@@ -1,5 +1,5 @@
 __author__ = "Johannes Köster"
-__copyright__ = "Copyright 2021, Johannes Köster"
+__copyright__ = "Copyright 2022, Johannes Köster"
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 

diff --git a/snakemake/executors/__init__.py b/snakemake/executors/__init__.py
@@ -1,5 +1,5 @@
 __author__ = "Johannes Köster"
-__copyright__ = "Copyright 2021, Johannes Köster"
+__copyright__ = "Copyright 2022, Johannes Köster"
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 
@@ -526,7 +526,11 @@ def job_args_and_prepare(self, job):
         )
 
     def run_single_job(self, job):
-        if self.use_threads or (not job.is_shadow and not job.is_run):
+        if (
+            self.use_threads
+            or (not job.is_shadow and not job.is_run)
+            or job.is_template_engine
+        ):
             future = self.pool.submit(
                 self.cached_or_run, job, run_wrapper, *self.job_args_and_prepare(job)
             )

diff --git a/snakemake/executors/ga4gh_tes.py b/snakemake/executors/ga4gh_tes.py
@@ -1,5 +1,5 @@
 __author__ = "Sven Twardziok, Alex Kanitz, Johannes Köster"
-__copyright__ = "Copyright 2021, Johannes Köster"
+__copyright__ = "Copyright 2022, Johannes Köster"
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 

diff --git a/snakemake/executors/google_lifesciences.py b/snakemake/executors/google_lifesciences.py
@@ -1,5 +1,5 @@
 __author__ = "Johannes Köster"
-__copyright__ = "Copyright 2021, Johannes Köster"
+__copyright__ = "Copyright 2022, Johannes Köster"
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 

diff --git a/snakemake/gui.py b/snakemake/gui.py
@@ -1,5 +1,5 @@
 __author__ = "Johannes Köster"
-__copyright__ = "Copyright 2021, Johannes Köster"
+__copyright__ = "Copyright 2022, Johannes Köster"
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 

diff --git a/snakemake/io.py b/snakemake/io.py
@@ -1,5 +1,5 @@
 __author__ = "Johannes Köster"
-__copyright__ = "Copyright 2021, Johannes Köster"
+__copyright__ = "Copyright 2022, Johannes Köster"
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 

diff --git a/snakemake/jobs.py b/snakemake/jobs.py
@@ -1,5 +1,5 @@
 __author__ = "Johannes Köster"
-__copyright__ = "Copyright 2021, Johannes Köster"
+__copyright__ = "Copyright 2022, Johannes Köster"
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 
@@ -427,27 +427,31 @@ def shellcmd(self):
 
     @property
     def is_shell(self):
-        return self.rule.shellcmd is not None
+        return self.rule.is_shell
 
     @property
     def is_norun(self):
         return self.rule.norun
 
     @property
     def is_script(self):
-        return self.rule.script is not None
+        return self.rule.is_script
 
     @property
     def is_notebook(self):
-        return self.rule.notebook is not None
+        return self.rule.is_notebook
 
     @property
     def is_wrapper(self):
-        return self.rule.wrapper is not None
+        return self.rule.is_wrapper
 
     @property
     def is_cwl(self):
-        return self.rule.cwl is not None
+        return self.rule.is_cwl
+
+    @property
+    def is_template_engine(self):
+        return self.rule.is_template_engine
 
     @property
     def is_run(self):

diff --git a/snakemake/logging.py b/snakemake/logging.py
@@ -1,5 +1,5 @@
 __author__ = "Johannes Köster"
-__copyright__ = "Copyright 2021, Johannes Köster"
+__copyright__ = "Copyright 2022, Johannes Köster"
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 

diff --git a/snakemake/modules.py b/snakemake/modules.py
@@ -1,5 +1,5 @@
 __authors__ = "Johannes Köster"
-__copyright__ = "Copyright 2021, Johannes Köster"
+__copyright__ = "Copyright 2022, Johannes Köster"
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 

diff --git a/snakemake/output_index.py b/snakemake/output_index.py
@@ -1,5 +1,5 @@
 __author__ = "Johannes Köster"
-__copyright__ = "Copyright 2021, Johannes Köster"
+__copyright__ = "Copyright 2022, Johannes Köster"
 __email__ = "johannes.koester@protonmail.com"
 __license__ = "MIT"
 

diff --git a/snakemake/parser.py b/snakemake/parser.py
@@ -1,8 +1,9 @@
 __author__ = "Johannes Köster"
-__copyright__ = "Copyright 2021, Johannes Köster"
+__copyright__ = "Copyright 2022, Johannes Köster"
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 
+from tempfile import TemporaryFile
 import tokenize
 import textwrap
 import os
@@ -648,6 +649,14 @@ def args(self):
         )
 
 
+class TemplateEngine(Script):
+    start_func = "@workflow.template_engine"
+    end_func = "render_template"
+
+    def args(self):
+        yield (", input, output, params, wildcards, config")
+
+
 class CWL(Script):
     start_func = "@workflow.cwl"
     end_func = "cwl"
@@ -692,6 +701,7 @@ class Rule(GlobalKeywordState):
         script=Script,
         notebook=Notebook,
         wrapper=Wrapper,
+        template_engine=TemplateEngine,
         cwl=CWL,
         **rule_property_subautomata,
     )
@@ -749,6 +759,8 @@ def block_content(self, token):
                     or token.string == "shell"
                     or token.string == "script"
                     or token.string == "wrapper"
+                    or token.string == "notebook"
+                    or token.string == "template_engine"
                     or token.string == "cwl"
                 ):
                     if self.run:
@@ -762,7 +774,7 @@ def block_content(self, token):
                 elif self.run:
                     raise self.error(
                         "No rule keywords allowed after "
-                        "run/shell/script/wrapper/cwl in "
+                        "run/shell/script/notebook/wrapper/template_engine/cwl in "
                         "rule {}.".format(self.rulename),
                         token,
                     )

diff --git a/snakemake/path_modifier.py b/snakemake/path_modifier.py
@@ -1,5 +1,5 @@
 __authors__ = "Johannes Köster"
-__copyright__ = "Copyright 2021, Johannes Köster"
+__copyright__ = "Copyright 2022, Johannes Köster"
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 

diff --git a/snakemake/persistence.py b/snakemake/persistence.py
@@ -1,5 +1,5 @@
 __author__ = "Johannes Köster"
-__copyright__ = "Copyright 2021, Johannes Köster"
+__copyright__ = "Copyright 2022, Johannes Köster"
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 

diff --git a/snakemake/remote/AzBlob.py b/snakemake/remote/AzBlob.py
@@ -2,7 +2,7 @@
 """
 
 __author__ = "Sebastian Kurscheid"
-__copyright__ = "Copyright 2021, Sebastian Kurscheid"
+__copyright__ = "Copyright 2022, Sebastian Kurscheid"
 __email__ = "sebastian.kurscheid@anu.edu.au"
 __license__ = "MIT"
 

diff --git a/snakemake/remote/EGA.py b/snakemake/remote/EGA.py
@@ -1,5 +1,5 @@
 __author__ = "Johannes Köster"
-__copyright__ = "Copyright 2021, Johannes Köster"
+__copyright__ = "Copyright 2022, Johannes Köster"
 __email__ = "johannes.koester@tu-dortmund.de"
 __license__ = "MIT"
 

diff --git a/snakemake/remote/FTP.py b/snakemake/remote/FTP.py
@@ -1,5 +1,5 @@
 __author__ = "Christopher Tomkins-Tinch"
-__copyright__ = "Copyright 2015, Christopher Tomkins-Tinch"
+__copyright__ = "Copyright 2022, Christopher Tomkins-Tinch"
 __email__ = "tomkinsc@broadinstitute.org"
 __license__ = "MIT"
 

diff --git a/snakemake/remote/GS.py b/snakemake/remote/GS.py
@@ -1,5 +1,5 @@
 __author__ = "Johannes Köster"
-__copyright__ = "Copyright 2021, Johannes Köster"
+__copyright__ = "Copyright 2022, Johannes Köster"
 __email__ = "johannes.koester@tu-dortmund.de"
 __license__ = "MIT"