From e1cbde5a378a29e3e7c7c16c73e08b35afa47a56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Thu, 17 Feb 2022 17:10:48 +0100 Subject: [PATCH] feat!: template rendering integration (yte and jinja2), require Python 3.6 as minimum version (f-string support) (#1410) * feat: add template rendering integration (for now via jinja2 and yte) * finalized integration and added testcase * update copyright * Add documentation * fmt * always run template engine jobs in thread * fmt * do not install cwltool on windows --- .github/workflows/main.yml | 2 +- docs/snakefiles/rules.rst | 39 ++++++++++++++++++ setup.py | 11 +++-- snakemake/__init__.py | 2 +- snakemake/benchmark.py | 2 +- snakemake/caching/__init__.py | 2 +- snakemake/caching/hash.py | 2 +- snakemake/caching/local.py | 2 +- snakemake/caching/remote.py | 2 +- snakemake/common/__init__.py | 2 +- snakemake/cwl.py | 2 +- snakemake/dag.py | 2 +- snakemake/decorators.py | 2 +- snakemake/deployment/conda.py | 2 +- snakemake/deployment/env_modules.py | 2 +- snakemake/deployment/singularity.py | 2 +- snakemake/exceptions.py | 2 +- snakemake/executors/__init__.py | 8 +++- snakemake/executors/ga4gh_tes.py | 2 +- snakemake/executors/google_lifesciences.py | 2 +- snakemake/gui.py | 2 +- snakemake/io.py | 2 +- snakemake/jobs.py | 16 +++++--- snakemake/logging.py | 2 +- snakemake/modules.py | 2 +- snakemake/output_index.py | 2 +- snakemake/parser.py | 16 +++++++- snakemake/path_modifier.py | 2 +- snakemake/persistence.py | 2 +- snakemake/remote/AzBlob.py | 2 +- snakemake/remote/EGA.py | 2 +- snakemake/remote/FTP.py | 2 +- snakemake/remote/GS.py | 2 +- snakemake/remote/HTTP.py | 2 +- snakemake/remote/NCBI.py | 2 +- snakemake/remote/S3.py | 2 +- snakemake/remote/S3Mocked.py | 2 +- snakemake/remote/SFTP.py | 2 +- snakemake/remote/XRootD.py | 2 +- snakemake/remote/__init__.py | 2 +- snakemake/remote/dropbox.py | 2 +- snakemake/remote/gfal.py | 2 +- snakemake/remote/gridftp.py | 2 +- snakemake/remote/iRODS.py | 2 +- snakemake/remote/webdav.py | 2 +- snakemake/report/__init__.py | 2 +- snakemake/ruleinfo.py | 3 +- snakemake/rules.py | 8 +++- snakemake/scheduler.py | 2 +- snakemake/script.py | 2 +- snakemake/shell.py | 2 +- snakemake/sourcecache.py | 2 +- snakemake/stats.py | 2 +- snakemake/template_rendering/__init__.py | 41 +++++++++++++++++++ snakemake/template_rendering/jinja2.py | 15 +++++++ snakemake/template_rendering/yte.py | 15 +++++++ snakemake/utils.py | 2 +- snakemake/workflow.py | 21 +++++++--- snakemake/wrapper.py | 2 +- test-environment.yml | 11 ++--- tests/common.py | 2 +- .../S3MockedForStaticTest.py | 2 +- tests/test_template_engine/Snakefile | 21 ++++++++++ .../expected-results/rendered.jinja2 | 3 ++ .../expected-results/rendered.yte | 1 + .../test_template_engine/template.jinja2.txt | 3 ++ tests/test_template_engine/template.yte.yaml | 4 ++ tests/tests.py | 6 ++- 68 files changed, 265 insertions(+), 77 deletions(-) create mode 100644 snakemake/template_rendering/__init__.py create mode 100644 snakemake/template_rendering/jinja2.py create mode 100644 snakemake/template_rendering/yte.py create mode 100644 tests/test_template_engine/Snakefile create mode 100644 tests/test_template_engine/expected-results/rendered.jinja2 create mode 100644 tests/test_template_engine/expected-results/rendered.yte create mode 100644 tests/test_template_engine/template.jinja2.txt create mode 100644 tests/test_template_engine/template.yte.yaml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index be9e8078e..3067b2e15 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -175,7 +175,7 @@ jobs: shell: python run: | import fileinput - excluded_on_win = ["environment-modules"] + excluded_on_win = ["environment-modules", "cwltool"] for line in fileinput.input("test-environment.yml", inplace=True): if all(pkg not in line for pkg in excluded_on_win): print(line) diff --git a/docs/snakefiles/rules.rst b/docs/snakefiles/rules.rst index 9a4f4a469..744aead28 100644 --- a/docs/snakefiles/rules.rst +++ b/docs/snakefiles/rules.rst @@ -1882,3 +1882,42 @@ This can be achieved by accessing their path via the ``workflow.get_source``, wh json=workflow.source_path("../resources/test.json") shell: "somecommand {params.json} > {output}" + + +.. _snakefiles-template-integration: + +Template rendering integration +------------------------------ + +Sometimes, data analyses entail the dynamic rendering of internal configuration files that are required for certain steps. +From Snakemake 7 on, such template rendering is directly integrated such that it can happen with minimal code and maximum performance. +Consider the following example: + +.. code-block:: python + + rule render_jinja2_template: + input: + "some-jinja2-template.txt" + output: + "results/{sample}.rendered-version.txt" + template_engine: + "jinja2" + +Here, Snakemake will automatically use the specified template engine `Jinja2 ` to render the template given as input file into the given output file. +Template rendering rules may only have a single input and output file. +The template_engine instruction has to be specified at the end of the rule. + +Apart from Jinja2, Snakemake supports YTE (YAML template engine), which is particularly designed to support templating of the ubiquitious YAML file format: + +.. code-block:: python + + rule render_jinja2_template: + input: + "some-yte-template.yaml" + output: + "results/{sample}.rendered-version.yaml" + template_engine: + "yte" + + +Template rendering rules are always executed locally, without submission to cluster or cloud processes (since templating is usually not resource intensive). diff --git a/setup.py b/setup.py index ec701db5b..dc1fd87f1 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from __future__ import print_function __author__ = "Johannes Köster" -__copyright__ = "Copyright 2015, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" @@ -11,8 +11,8 @@ import versioneer -if sys.version_info < (3, 5): - print("At least Python 3.5 is required for Snakemake.\n", file=sys.stderr) +if sys.version_info < (3, 6): + print("At least Python 3.6 is required for Snakemake.\n", file=sys.stderr) exit(1) @@ -47,7 +47,8 @@ "snakemake.linting", "snakemake.executors", "snakemake.unit_tests", - "snakemake.unit_tests.templates" + "snakemake.unit_tests.templates", + "snakemake.template_rendering", ], entry_points={ "console_scripts": [ @@ -76,6 +77,8 @@ "filelock", "stopit", "tabulate", + "yte", + "jinja2", ], extras_require={ "reports": ["jinja2", "networkx", "pygments", "pygraphviz"], diff --git a/snakemake/__init__.py b/snakemake/__init__.py index 2c90a0e38..95d16e43a 100644 --- a/snakemake/__init__.py +++ b/snakemake/__init__.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/benchmark.py b/snakemake/benchmark.py index d53df8e71..c3a127957 100644 --- a/snakemake/benchmark.py +++ b/snakemake/benchmark.py @@ -1,5 +1,5 @@ __author__ = "Manuel Holtgrewe" -__copyright__ = "Copyright 2017, Manuel Holtgrewe" +__copyright__ = "Copyright 2022, Manuel Holtgrewe" __email__ = "manuel.holtgrewe@bihealth.de" __license__ = "MIT" diff --git a/snakemake/caching/__init__.py b/snakemake/caching/__init__.py index b7d3049a0..6f6b2d4df 100644 --- a/snakemake/caching/__init__.py +++ b/snakemake/caching/__init__.py @@ -1,5 +1,5 @@ __authors__ = "Johannes Köster, Sven Nahnsen" -__copyright__ = "Copyright 2021, Johannes Köster, Sven Nahnsen" +__copyright__ = "Copyright 2022, Johannes Köster, Sven Nahnsen" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/caching/hash.py b/snakemake/caching/hash.py index 7f02a123e..ca260f447 100644 --- a/snakemake/caching/hash.py +++ b/snakemake/caching/hash.py @@ -1,5 +1,5 @@ __authors__ = "Johannes Köster, Sven Nahnsen" -__copyright__ = "Copyright 2021, Johannes Köster, Sven Nahnsen" +__copyright__ = "Copyright 2022, Johannes Köster, Sven Nahnsen" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/caching/local.py b/snakemake/caching/local.py index ad701c36b..33c21cf7e 100644 --- a/snakemake/caching/local.py +++ b/snakemake/caching/local.py @@ -1,5 +1,5 @@ __authors__ = "Johannes Köster, Sven Nahnsen" -__copyright__ = "Copyright 2021, Johannes Köster, Sven Nahnsen" +__copyright__ = "Copyright 2022, Johannes Köster, Sven Nahnsen" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/caching/remote.py b/snakemake/caching/remote.py index db8be9a5c..cbec6b30f 100644 --- a/snakemake/caching/remote.py +++ b/snakemake/caching/remote.py @@ -1,5 +1,5 @@ __authors__ = "Johannes Köster, Sven Nahnsen" -__copyright__ = "Copyright 2021, Johannes Köster, Sven Nahnsen" +__copyright__ = "Copyright 2022, Johannes Köster, Sven Nahnsen" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/common/__init__.py b/snakemake/common/__init__.py index 9f85c5a40..2cde9d810 100644 --- a/snakemake/common/__init__.py +++ b/snakemake/common/__init__.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@protonmail.com" __license__ = "MIT" diff --git a/snakemake/cwl.py b/snakemake/cwl.py index e39f12b44..d1d9ca849 100644 --- a/snakemake/cwl.py +++ b/snakemake/cwl.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/dag.py b/snakemake/dag.py index 20d50bb7b..ee45228e0 100755 --- a/snakemake/dag.py +++ b/snakemake/dag.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/decorators.py b/snakemake/decorators.py index 6af530a03..41afe0a06 100644 --- a/snakemake/decorators.py +++ b/snakemake/decorators.py @@ -1,5 +1,5 @@ __author__ = "Christopher Tomkins-Tinch" -__copyright__ = "Copyright 2021, Christopher Tomkins-Tinch" +__copyright__ = "Copyright 2022, Christopher Tomkins-Tinch" __email__ = "tomkinsc@broadinstitute.org" __license__ = "MIT" diff --git a/snakemake/deployment/conda.py b/snakemake/deployment/conda.py index e19cb86d9..c2becbfbd 100644 --- a/snakemake/deployment/conda.py +++ b/snakemake/deployment/conda.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/deployment/env_modules.py b/snakemake/deployment/env_modules.py index ad8b0ec56..464c6d348 100644 --- a/snakemake/deployment/env_modules.py +++ b/snakemake/deployment/env_modules.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/deployment/singularity.py b/snakemake/deployment/singularity.py index 7c23f124a..5ebef81fc 100644 --- a/snakemake/deployment/singularity.py +++ b/snakemake/deployment/singularity.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/exceptions.py b/snakemake/exceptions.py index 4f33f2c52..c753d4556 100644 --- a/snakemake/exceptions.py +++ b/snakemake/exceptions.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/executors/__init__.py b/snakemake/executors/__init__.py index 06f2394f1..d8628d2fa 100644 --- a/snakemake/executors/__init__.py +++ b/snakemake/executors/__init__.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" @@ -526,7 +526,11 @@ def job_args_and_prepare(self, job): ) def run_single_job(self, job): - if self.use_threads or (not job.is_shadow and not job.is_run): + if ( + self.use_threads + or (not job.is_shadow and not job.is_run) + or job.is_template_engine + ): future = self.pool.submit( self.cached_or_run, job, run_wrapper, *self.job_args_and_prepare(job) ) diff --git a/snakemake/executors/ga4gh_tes.py b/snakemake/executors/ga4gh_tes.py index 09c445969..6e0defb61 100644 --- a/snakemake/executors/ga4gh_tes.py +++ b/snakemake/executors/ga4gh_tes.py @@ -1,5 +1,5 @@ __author__ = "Sven Twardziok, Alex Kanitz, Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/executors/google_lifesciences.py b/snakemake/executors/google_lifesciences.py index 5259af3cd..f4009dd80 100644 --- a/snakemake/executors/google_lifesciences.py +++ b/snakemake/executors/google_lifesciences.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/gui.py b/snakemake/gui.py index dddba4e9b..f0071fd82 100644 --- a/snakemake/gui.py +++ b/snakemake/gui.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/io.py b/snakemake/io.py index ac8cde812..f898834e2 100755 --- a/snakemake/io.py +++ b/snakemake/io.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/jobs.py b/snakemake/jobs.py index e7b0014d4..12f721d24 100644 --- a/snakemake/jobs.py +++ b/snakemake/jobs.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" @@ -427,7 +427,7 @@ def shellcmd(self): @property def is_shell(self): - return self.rule.shellcmd is not None + return self.rule.is_shell @property def is_norun(self): @@ -435,19 +435,23 @@ def is_norun(self): @property def is_script(self): - return self.rule.script is not None + return self.rule.is_script @property def is_notebook(self): - return self.rule.notebook is not None + return self.rule.is_notebook @property def is_wrapper(self): - return self.rule.wrapper is not None + return self.rule.is_wrapper @property def is_cwl(self): - return self.rule.cwl is not None + return self.rule.is_cwl + + @property + def is_template_engine(self): + return self.rule.is_template_engine @property def is_run(self): diff --git a/snakemake/logging.py b/snakemake/logging.py index 4602326ac..509f0b4b6 100644 --- a/snakemake/logging.py +++ b/snakemake/logging.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/modules.py b/snakemake/modules.py index b23b00452..b3d84afa3 100644 --- a/snakemake/modules.py +++ b/snakemake/modules.py @@ -1,5 +1,5 @@ __authors__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/output_index.py b/snakemake/output_index.py index a5f15a80b..144cc1ea4 100644 --- a/snakemake/output_index.py +++ b/snakemake/output_index.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@protonmail.com" __license__ = "MIT" diff --git a/snakemake/parser.py b/snakemake/parser.py index 51ad77c1f..e82da47a4 100644 --- a/snakemake/parser.py +++ b/snakemake/parser.py @@ -1,8 +1,9 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" +from tempfile import TemporaryFile import tokenize import textwrap import os @@ -648,6 +649,14 @@ def args(self): ) +class TemplateEngine(Script): + start_func = "@workflow.template_engine" + end_func = "render_template" + + def args(self): + yield (", input, output, params, wildcards, config") + + class CWL(Script): start_func = "@workflow.cwl" end_func = "cwl" @@ -692,6 +701,7 @@ class Rule(GlobalKeywordState): script=Script, notebook=Notebook, wrapper=Wrapper, + template_engine=TemplateEngine, cwl=CWL, **rule_property_subautomata, ) @@ -749,6 +759,8 @@ def block_content(self, token): or token.string == "shell" or token.string == "script" or token.string == "wrapper" + or token.string == "notebook" + or token.string == "template_engine" or token.string == "cwl" ): if self.run: @@ -762,7 +774,7 @@ def block_content(self, token): elif self.run: raise self.error( "No rule keywords allowed after " - "run/shell/script/wrapper/cwl in " + "run/shell/script/notebook/wrapper/template_engine/cwl in " "rule {}.".format(self.rulename), token, ) diff --git a/snakemake/path_modifier.py b/snakemake/path_modifier.py index 59604cf19..2ba697b9e 100644 --- a/snakemake/path_modifier.py +++ b/snakemake/path_modifier.py @@ -1,5 +1,5 @@ __authors__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/persistence.py b/snakemake/persistence.py index 63ce0e65c..4f5338946 100755 --- a/snakemake/persistence.py +++ b/snakemake/persistence.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/remote/AzBlob.py b/snakemake/remote/AzBlob.py index fccc6145f..c54565274 100644 --- a/snakemake/remote/AzBlob.py +++ b/snakemake/remote/AzBlob.py @@ -2,7 +2,7 @@ """ __author__ = "Sebastian Kurscheid" -__copyright__ = "Copyright 2021, Sebastian Kurscheid" +__copyright__ = "Copyright 2022, Sebastian Kurscheid" __email__ = "sebastian.kurscheid@anu.edu.au" __license__ = "MIT" diff --git a/snakemake/remote/EGA.py b/snakemake/remote/EGA.py index 7e70ccd99..43e334305 100644 --- a/snakemake/remote/EGA.py +++ b/snakemake/remote/EGA.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@tu-dortmund.de" __license__ = "MIT" diff --git a/snakemake/remote/FTP.py b/snakemake/remote/FTP.py index c82170ee2..cf92a517b 100644 --- a/snakemake/remote/FTP.py +++ b/snakemake/remote/FTP.py @@ -1,5 +1,5 @@ __author__ = "Christopher Tomkins-Tinch" -__copyright__ = "Copyright 2015, Christopher Tomkins-Tinch" +__copyright__ = "Copyright 2022, Christopher Tomkins-Tinch" __email__ = "tomkinsc@broadinstitute.org" __license__ = "MIT" diff --git a/snakemake/remote/GS.py b/snakemake/remote/GS.py index 18d129b1f..16213777e 100644 --- a/snakemake/remote/GS.py +++ b/snakemake/remote/GS.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@tu-dortmund.de" __license__ = "MIT" diff --git a/snakemake/remote/HTTP.py b/snakemake/remote/HTTP.py index cc9766f0b..5b398d058 100644 --- a/snakemake/remote/HTTP.py +++ b/snakemake/remote/HTTP.py @@ -1,5 +1,5 @@ __author__ = "Christopher Tomkins-Tinch" -__copyright__ = "Copyright 2015, Christopher Tomkins-Tinch" +__copyright__ = "Copyright 2022, Christopher Tomkins-Tinch" __email__ = "tomkinsc@broadinstitute.org" __license__ = "MIT" diff --git a/snakemake/remote/NCBI.py b/snakemake/remote/NCBI.py index 31392fbac..bbc5f0c22 100644 --- a/snakemake/remote/NCBI.py +++ b/snakemake/remote/NCBI.py @@ -1,5 +1,5 @@ __author__ = "Christopher Tomkins-Tinch" -__copyright__ = "Copyright 2017, Christopher Tomkins-Tinch" +__copyright__ = "Copyright 2022, Christopher Tomkins-Tinch" __email__ = "tomkinsc@broadinstitute.org" __license__ = "MIT" diff --git a/snakemake/remote/S3.py b/snakemake/remote/S3.py index 192e95603..5c32c9753 100644 --- a/snakemake/remote/S3.py +++ b/snakemake/remote/S3.py @@ -1,5 +1,5 @@ __author__ = "Christopher Tomkins-Tinch" -__copyright__ = "Copyright 2015, Christopher Tomkins-Tinch" +__copyright__ = "Copyright 2022, Christopher Tomkins-Tinch" __email__ = "tomkinsc@broadinstitute.org" __license__ = "MIT" diff --git a/snakemake/remote/S3Mocked.py b/snakemake/remote/S3Mocked.py index 18cd8f1c3..228b97e92 100644 --- a/snakemake/remote/S3Mocked.py +++ b/snakemake/remote/S3Mocked.py @@ -1,5 +1,5 @@ __author__ = "Christopher Tomkins-Tinch" -__copyright__ = "Copyright 2015, Christopher Tomkins-Tinch" +__copyright__ = "Copyright 2022, Christopher Tomkins-Tinch" __email__ = "tomkinsc@broadinstitute.org" __license__ = "MIT" diff --git a/snakemake/remote/SFTP.py b/snakemake/remote/SFTP.py index 224495405..ed4462ced 100644 --- a/snakemake/remote/SFTP.py +++ b/snakemake/remote/SFTP.py @@ -1,5 +1,5 @@ __author__ = "Christopher Tomkins-Tinch" -__copyright__ = "Copyright 2015, Christopher Tomkins-Tinch" +__copyright__ = "Copyright 2022, Christopher Tomkins-Tinch" __email__ = "tomkinsc@broadinstitute.org" __license__ = "MIT" diff --git a/snakemake/remote/XRootD.py b/snakemake/remote/XRootD.py index 6a6670a66..7b430058b 100644 --- a/snakemake/remote/XRootD.py +++ b/snakemake/remote/XRootD.py @@ -1,5 +1,5 @@ __author__ = "Chris Burr" -__copyright__ = "Copyright 2017, Chris Burr" +__copyright__ = "Copyright 2022, Chris Burr" __email__ = "christopher.burr@cern.ch" __license__ = "MIT" diff --git a/snakemake/remote/__init__.py b/snakemake/remote/__init__.py index 0d95ff23f..f3d2d0c1b 100644 --- a/snakemake/remote/__init__.py +++ b/snakemake/remote/__init__.py @@ -1,5 +1,5 @@ __author__ = "Christopher Tomkins-Tinch" -__copyright__ = "Copyright 2015, Christopher Tomkins-Tinch" +__copyright__ = "Copyright 2022, Christopher Tomkins-Tinch" __email__ = "tomkinsc@broadinstitute.org" __license__ = "MIT" diff --git a/snakemake/remote/dropbox.py b/snakemake/remote/dropbox.py index 84897cf54..3468ae67d 100644 --- a/snakemake/remote/dropbox.py +++ b/snakemake/remote/dropbox.py @@ -1,5 +1,5 @@ __author__ = "Christopher Tomkins-Tinch" -__copyright__ = "Copyright 2015, Christopher Tomkins-Tinch" +__copyright__ = "Copyright 2022, Christopher Tomkins-Tinch" __email__ = "tomkinsc@broadinstitute.org" __license__ = "MIT" diff --git a/snakemake/remote/gfal.py b/snakemake/remote/gfal.py index cbc279fd3..bfa3be924 100644 --- a/snakemake/remote/gfal.py +++ b/snakemake/remote/gfal.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@tu-dortmund.de" __license__ = "MIT" diff --git a/snakemake/remote/gridftp.py b/snakemake/remote/gridftp.py index 0ebed3706..e45168788 100644 --- a/snakemake/remote/gridftp.py +++ b/snakemake/remote/gridftp.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@tu-dortmund.de" __license__ = "MIT" diff --git a/snakemake/remote/iRODS.py b/snakemake/remote/iRODS.py index a021ca4c7..e433321ae 100644 --- a/snakemake/remote/iRODS.py +++ b/snakemake/remote/iRODS.py @@ -1,5 +1,5 @@ __author__ = "Oliver Stolpe" -__copyright__ = "Copyright 2017, BIH Core Unit Bioinformatics" +__copyright__ = "Copyright 2022, BIH Core Unit Bioinformatics" __email__ = "oliver.stolpe@bihealth.org" __license__ = "MIT" diff --git a/snakemake/remote/webdav.py b/snakemake/remote/webdav.py index b211a219f..060c96dae 100644 --- a/snakemake/remote/webdav.py +++ b/snakemake/remote/webdav.py @@ -1,5 +1,5 @@ __author__ = "Christopher Tomkins-Tinch" -__copyright__ = "Copyright 2017, Christopher Tomkins-Tinch" +__copyright__ = "Copyright 2022, Christopher Tomkins-Tinch" __email__ = "tomkinsc@broadinstitute.org" __license__ = "MIT" diff --git a/snakemake/report/__init__.py b/snakemake/report/__init__.py index 2d32bbf82..ba4c4edc4 100644 --- a/snakemake/report/__init__.py +++ b/snakemake/report/__init__.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/ruleinfo.py b/snakemake/ruleinfo.py index f9305b1bb..0edd3f8dc 100644 --- a/snakemake/ruleinfo.py +++ b/snakemake/ruleinfo.py @@ -1,5 +1,5 @@ __authors__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" @@ -33,6 +33,7 @@ def __init__(self, func=None): self.script = None self.notebook = None self.wrapper = None + self.template_engine = None self.cwl = None self.cache = False self.path_modifier = None diff --git a/snakemake/rules.py b/snakemake/rules.py index fcd95abe7..21138c129 100644 --- a/snakemake/rules.py +++ b/snakemake/rules.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" @@ -103,6 +103,7 @@ def __init__(self, *args, lineno=None, snakefile=None, restart_times=0): self.script = None self.notebook = None self.wrapper = None + self.template_engine = None self.cwl = None self.norun = False self.is_handover = False @@ -152,6 +153,7 @@ def __init__(self, *args, lineno=None, snakefile=None, restart_times=0): self.script = other.script self.notebook = other.notebook self.wrapper = other.wrapper + self.template_engine = other.template_engine self.cwl = other.cwl self.norun = other.norun self.is_handover = other.is_handover @@ -267,6 +269,10 @@ def is_notebook(self): def is_wrapper(self): return self.wrapper is not None + @property + def is_template_engine(self): + return self.template_engine is not None + @property def is_cwl(self): return self.cwl is not None diff --git a/snakemake/scheduler.py b/snakemake/scheduler.py index c42fb8fa7..e27a6f234 100644 --- a/snakemake/scheduler.py +++ b/snakemake/scheduler.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/script.py b/snakemake/script.py index babf874d6..cb90f5472 100644 --- a/snakemake/script.py +++ b/snakemake/script.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/shell.py b/snakemake/shell.py index 860fecc07..ec0989151 100644 --- a/snakemake/shell.py +++ b/snakemake/shell.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/sourcecache.py b/snakemake/sourcecache.py index 2566ea7d5..36e7a9949 100644 --- a/snakemake/sourcecache.py +++ b/snakemake/sourcecache.py @@ -1,5 +1,5 @@ __authors__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/stats.py b/snakemake/stats.py index 34fbe0a8c..138f544c6 100644 --- a/snakemake/stats.py +++ b/snakemake/stats.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/template_rendering/__init__.py b/snakemake/template_rendering/__init__.py new file mode 100644 index 000000000..a16672edc --- /dev/null +++ b/snakemake/template_rendering/__init__.py @@ -0,0 +1,41 @@ +from abc import ABC, abstractmethod + + +class TemplateRenderer(ABC): + def __init__(self, input, output, params, wildcards, config): + assert len(input) == 1 + assert len(output) == 1 + + self.input_file = input[0] + self.output_file = output[0] + self.params = params + self.wildcards = wildcards + self.config = config + + @property + def variables(self): + return { + "params": self.params, + "wildcards": self.wildcards, + "config": self.config, + } + + @abstractmethod + def render(self): + ... + + +def render_template(engine, input, output, params, wildcards, config): + if engine == "yte": + from snakemake.template_rendering.yte import YteRenderer + + return YteRenderer(input, output, params, wildcards, config).render() + elif engine == "jinja2": + from snakemake.template_rendering.jinja2 import Jinja2Renderer + + return Jinja2Renderer(input, output, params, wildcards, config).render() + else: + raise WorkflowError( + f"Unsupported template engine {engine}. " + "So far, only yte and jinja2 are supported." + ) diff --git a/snakemake/template_rendering/jinja2.py b/snakemake/template_rendering/jinja2.py new file mode 100644 index 000000000..016425145 --- /dev/null +++ b/snakemake/template_rendering/jinja2.py @@ -0,0 +1,15 @@ +from snakemake.exceptions import WorkflowError +from snakemake.template_rendering import TemplateRenderer + + +class Jinja2Renderer(TemplateRenderer): + def render(self): + import jinja2 + + try: + with open(self.input_file, "r") as infile: + template = jinja2.Template(infile.read()) + with open(self.output_file, "w") as outfile: + outfile.write(template.render(**self.variables)) + except Exception as e: + raise WorkflowError("Failed to render jinja2 template.", e) diff --git a/snakemake/template_rendering/yte.py b/snakemake/template_rendering/yte.py new file mode 100644 index 000000000..6bea0220e --- /dev/null +++ b/snakemake/template_rendering/yte.py @@ -0,0 +1,15 @@ +from snakemake.exceptions import WorkflowError +from snakemake.template_rendering import TemplateRenderer + + +class YteRenderer(TemplateRenderer): + def render(self): + import yte + + try: + with open(self.output_file, "w") as outfile, open( + self.input_file, "r" + ) as infile: + yte.process_yaml(infile, outfile=outfile, variables=self.variables) + except Exception as e: + raise WorkflowError("Failed to render yte template.", e) diff --git a/snakemake/utils.py b/snakemake/utils.py index a8b7bf0f5..c833e13ed 100644 --- a/snakemake/utils.py +++ b/snakemake/utils.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/snakemake/workflow.py b/snakemake/workflow.py index fa7a645aa..5834e458f 100644 --- a/snakemake/workflow.py +++ b/snakemake/workflow.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" @@ -62,6 +62,7 @@ from snakemake.notebook import notebook from snakemake.wrapper import wrapper from snakemake.cwl import cwl +from snakemake.template_rendering import render_template import snakemake.wrapper from snakemake.common import ( Mode, @@ -509,7 +510,9 @@ def list_resources(self): logger.info(resource) def is_local(self, rule): - return rule.group is None and (rule.name in self._localrules or rule.norun) + return rule.group is None and ( + rule.name in self._localrules or rule.norun or rule.is_template_engine + ) def check_localrules(self): undefined = self._localrules - set(rule.name for rule in self.rules) @@ -1467,7 +1470,7 @@ def decorate(ruleinfo): if invalid_rule: raise RuleException( "envmodules directive is only allowed with " - "shell, script, notebook, or wrapper directives (not with run)", + "shell, script, notebook, or wrapper directives (not with run or template_engine)", rule=rule, ) from snakemake.deployment.env_modules import EnvModules @@ -1484,7 +1487,7 @@ def decorate(ruleinfo): raise RuleException( "Conda environments are only allowed " "with shell, script, notebook, or wrapper directives " - "(not with run).", + "(not with run or template_engine).", rule=rule, ) @@ -1513,7 +1516,7 @@ def decorate(ruleinfo): raise RuleException( "Singularity directive is only allowed " "with shell, script, notebook or wrapper directives " - "(not with run).", + "(not with run or template_engine).", rule=rule, ) rule.container_img = ruleinfo.container_img @@ -1533,6 +1536,7 @@ def decorate(ruleinfo): rule.script = ruleinfo.script rule.notebook = ruleinfo.notebook rule.wrapper = ruleinfo.wrapper + rule.template_engine = ruleinfo.template_engine rule.cwl = ruleinfo.cwl rule.restart_times = self.restart_times rule.basedir = self.current_basedir @@ -1791,6 +1795,13 @@ def decorate(ruleinfo): return decorate + def template_engine(self, template_engine): + def decorate(ruleinfo): + ruleinfo.template_engine = template_engine + return ruleinfo + + return decorate + def cwl(self, cwl): def decorate(ruleinfo): ruleinfo.cwl = cwl diff --git a/snakemake/wrapper.py b/snakemake/wrapper.py index 91b1a04fd..f3c5ef9dd 100644 --- a/snakemake/wrapper.py +++ b/snakemake/wrapper.py @@ -1,5 +1,5 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/test-environment.yml b/test-environment.yml index 0474ee941..df82929e8 100644 --- a/test-environment.yml +++ b/test-environment.yml @@ -2,7 +2,8 @@ channels: - conda-forge - bioconda dependencies: - - python >=3.5.3 + - python + - yte - stopit - datrie - boto3 @@ -11,19 +12,19 @@ dependencies: - httpretty - wrapt - pyyaml - - pytest ==6.1.2 + - pytest - pytest-cov - ftputil - pysftp - requests - - responses <0.12.1 # responses 0.12.1 delivers a "TypeError: cannot unpack non-iterable CallbackResponse object" + - responses - dropbox - numpy - appdirs - pytools - docutils - pygments - - pandoc <2.0 # pandoc has changed the CLI API so that it is no longer compatible with the version of r-markdown below + - pandoc - xorg-libxrender - xorg-libxext - xorg-libxau @@ -38,7 +39,7 @@ dependencies: - configargparse - appdirs - python-irodsclient - - cwltool <=3.0.20201203173111 # TODO remove once schema-salad has been updated in conda-forge + - cwltool - jsonschema - pandas - networkx diff --git a/tests/common.py b/tests/common.py index 7731cf405..c5f159fa8 100644 --- a/tests/common.py +++ b/tests/common.py @@ -1,5 +1,5 @@ __authors__ = ["Tobias Marschall", "Marcel Martin", "Johannes Köster"] -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" diff --git a/tests/test_static_remote/S3MockedForStaticTest.py b/tests/test_static_remote/S3MockedForStaticTest.py index 051825671..f76396fc7 100644 --- a/tests/test_static_remote/S3MockedForStaticTest.py +++ b/tests/test_static_remote/S3MockedForStaticTest.py @@ -1,5 +1,5 @@ __author__ = "Christopher Tomkins-Tinch" -__copyright__ = "Copyright 2015, Christopher Tomkins-Tinch" +__copyright__ = "Copyright 2022, Christopher Tomkins-Tinch" __email__ = "tomkinsc@broadinstitute.org" __license__ = "MIT" diff --git a/tests/test_template_engine/Snakefile b/tests/test_template_engine/Snakefile new file mode 100644 index 000000000..95a09828c --- /dev/null +++ b/tests/test_template_engine/Snakefile @@ -0,0 +1,21 @@ +rule all: + input: + expand("rendered.{engine}", engine=["yte", "jinja2"]) + + +rule render_yte_template: + input: + "template.yte.yaml" + output: + "rendered.yte" + template_engine: + "yte" + + +rule render_jinja2_template: + input: + "template.jinja2.txt" + output: + "rendered.jinja2" + template_engine: + "jinja2" \ No newline at end of file diff --git a/tests/test_template_engine/expected-results/rendered.jinja2 b/tests/test_template_engine/expected-results/rendered.jinja2 new file mode 100644 index 000000000..b198e6a2d --- /dev/null +++ b/tests/test_template_engine/expected-results/rendered.jinja2 @@ -0,0 +1,3 @@ + + +foo: 1 \ No newline at end of file diff --git a/tests/test_template_engine/expected-results/rendered.yte b/tests/test_template_engine/expected-results/rendered.yte new file mode 100644 index 000000000..790bc26c9 --- /dev/null +++ b/tests/test_template_engine/expected-results/rendered.yte @@ -0,0 +1 @@ +foo: 1 diff --git a/tests/test_template_engine/template.jinja2.txt b/tests/test_template_engine/template.jinja2.txt new file mode 100644 index 000000000..e7de4b170 --- /dev/null +++ b/tests/test_template_engine/template.jinja2.txt @@ -0,0 +1,3 @@ +{% set foo = 1 %} + +foo: {{ foo }} \ No newline at end of file diff --git a/tests/test_template_engine/template.yte.yaml b/tests/test_template_engine/template.yte.yaml new file mode 100644 index 000000000..e34e82ca7 --- /dev/null +++ b/tests/test_template_engine/template.yte.yaml @@ -0,0 +1,4 @@ +?if True: + foo: 1 +?else: + bar: 2 \ No newline at end of file diff --git a/tests/tests.py b/tests/tests.py index aae2fa80a..3eb5139e5 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -1,5 +1,5 @@ __authors__ = ["Tobias Marschall", "Marcel Martin", "Johannes Köster"] -__copyright__ = "Copyright 2021, Johannes Köster" +__copyright__ = "Copyright 2022, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" @@ -1479,3 +1479,7 @@ def test_github_issue1384(): @skip_on_windows def test_peppy(): run(dpath("test_peppy")) + + +def test_template_engine(): + run(dpath("test_template_engine"))