diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 030d76a1b..be9e8078e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -95,7 +95,7 @@ jobs: # activate conda env export PATH="/usr/share/miniconda/bin:$PATH" source activate snakemake - + pytest -v -x tests/test_expand.py tests/test_io.py tests/test_schema.py tests/test_linting.py tests/tests.py - name: Build and publish docker image diff --git a/CHANGELOG.md b/CHANGELOG.md index e8ba17da0..1719b6428 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## [6.11.0](https://www.github.com/snakemake/snakemake/compare/v6.10.0...v6.11.0) (2021-11-25) + + +### Features + +* fail with an error if snakemake cannot write job metadata. ([#1273](https://www.github.com/snakemake/snakemake/issues/1273)) ([cd968cd](https://www.github.com/snakemake/snakemake/commit/cd968cd03437ad6db1d791f5d7ae5295b9754137)) + + +### Bug Fixes + +* Adds fixes for the first two MREs in [#823](https://www.github.com/snakemake/snakemake/issues/823) ([#1215](https://www.github.com/snakemake/snakemake/issues/1215)) ([cfd2f89](https://www.github.com/snakemake/snakemake/commit/cfd2f890a0af57628f7b9278d8d43f59b7006825)) +* env file usage after changes to source file handling (inspired by [#1233](https://www.github.com/snakemake/snakemake/issues/1233) and [#1211](https://www.github.com/snakemake/snakemake/issues/1211)). ([#1236](https://www.github.com/snakemake/snakemake/issues/1236)) ([3ac8e85](https://www.github.com/snakemake/snakemake/commit/3ac8e858a7b908326922c8f68cae512b1250e906)) +* fixed code change detection when using modules ([#1264](https://www.github.com/snakemake/snakemake/issues/1264)) ([b571e09](https://www.github.com/snakemake/snakemake/commit/b571e09ce452f6a1a95395e1c3c8b9e3f83867ad)) +* handle config file extension/overwriting more explicitly ([#1251](https://www.github.com/snakemake/snakemake/issues/1251)) ([d0a7bf2](https://www.github.com/snakemake/snakemake/commit/d0a7bf243c5df204136fa1f14706aab793793c68)) +* Issue [#1253](https://www.github.com/snakemake/snakemake/issues/1253) (problems editing Jupyter Notebooks) ([#1255](https://www.github.com/snakemake/snakemake/issues/1255)) ([3398ddf](https://www.github.com/snakemake/snakemake/commit/3398ddffd1f68182af768ef4ea519e9a9ad4efaf)) +* more informative nothing to be done message ([#1234](https://www.github.com/snakemake/snakemake/issues/1234)) ([368d265](https://www.github.com/snakemake/snakemake/commit/368d265ff3da984bd3a53b319dcb882d6916975b)) +* only consider context of shell command for technical switches if called from snakemake rules. ([#1213](https://www.github.com/snakemake/snakemake/issues/1213)) ([4816a58](https://www.github.com/snakemake/snakemake/commit/4816a58653e466ca94b1482a1d947a856f5381b3)) +* R encoding of pathlib.Path objects ([#1201](https://www.github.com/snakemake/snakemake/issues/1201)) ([bd516e9](https://www.github.com/snakemake/snakemake/commit/bd516e958af22e57c18cacf0cb22552c2a237bd8)) +* Use 'snakemake.utils.update_config' instead of 'dict.update' ([#1126](https://www.github.com/snakemake/snakemake/issues/1126)) ([2658027](https://www.github.com/snakemake/snakemake/commit/2658027458dde4c10b3d6e1af7671564d175f9cb)) + ## [6.10.0](https://www.github.com/snakemake/snakemake/compare/v6.9.1...v6.10.0) (2021-10-21) diff --git a/snakemake/__init__.py b/snakemake/__init__.py index aee6066fa..7c08e4d98 100644 --- a/snakemake/__init__.py +++ b/snakemake/__init__.py @@ -497,13 +497,13 @@ def snakemake( configfiles = [] for f in configfiles: # get values to override. Later configfiles override earlier ones. - overwrite_config.update(load_configfile(f)) + update_config(overwrite_config, load_configfile(f)) # convert provided paths to absolute paths configfiles = list(map(os.path.abspath, configfiles)) # directly specified elements override any configfiles if config: - overwrite_config.update(config) + update_config(overwrite_config, config) if config_args is None: config_args = unparse_config(config) @@ -947,7 +947,7 @@ def parse_config(args): except: pass assert v is not None - config[key] = v + update_config(config, {key: v}) return config diff --git a/snakemake/workflow.py b/snakemake/workflow.py index fb4a23159..f7e591b2f 100644 --- a/snakemake/workflow.py +++ b/snakemake/workflow.py @@ -204,7 +204,7 @@ def __init__( self.attempt = attempt self.default_remote_provider = default_remote_provider self.default_remote_prefix = default_remote_prefix - self.configfiles = overwrite_configfiles or [] + self.configfiles = list(overwrite_configfiles) or [] self.run_local = run_local self.report_text = None self.conda_cleanup_pkgs = conda_cleanup_pkgs @@ -682,7 +682,8 @@ def files(items): rules = self.rules if allowed_rules: - rules = [rule for rule in rules if rule.name in set(allowed_rules)] + allowed_rules = set(allowed_rules) + rules = [rule for rule in rules if rule.name in allowed_rules] if wait_for_files is not None: try: diff --git a/tests/test_checkpoint_allowed_rules/Snakefile b/tests/test_checkpoint_allowed_rules/Snakefile new file mode 100644 index 000000000..9dee10825 --- /dev/null +++ b/tests/test_checkpoint_allowed_rules/Snakefile @@ -0,0 +1,13 @@ +checkpoint a: + output: touch("a.txt") + +rule b: + output: touch("b.txt") + +def get_input(wildcards): + checkpoints.a.get() + return "b.txt" + +rule c: + input: get_input + shell: "echo {input}" \ No newline at end of file diff --git a/tests/test_checkpoint_allowed_rules/expected-results/b.txt b/tests/test_checkpoint_allowed_rules/expected-results/b.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_checkpoint_allowed_rules/qsub b/tests/test_checkpoint_allowed_rules/qsub new file mode 100755 index 000000000..0bc8aabba --- /dev/null +++ b/tests/test_checkpoint_allowed_rules/qsub @@ -0,0 +1,7 @@ +#!/bin/bash +echo `date` >> qsub.log +tail -n1 $1 >> qsub.log +# simulate printing of job id by a random number +echo $RANDOM +cat $1 >> qsub.log +sh $1 diff --git a/tests/test_config_merging/Snakefile b/tests/test_config_merging/Snakefile new file mode 100644 index 000000000..1faef6651 --- /dev/null +++ b/tests/test_config_merging/Snakefile @@ -0,0 +1,11 @@ +configfile: "config_snakefile.yaml" + + +rule dump_config: + output: + "test.out", + run: + import json + + with open(output[0], "w") as fd: + json.dump(config, fd, sort_keys=True) diff --git a/tests/test_config_merging/config_cmdline_01.yaml b/tests/test_config_merging/config_cmdline_01.yaml new file mode 100644 index 000000000..fc731e9b5 --- /dev/null +++ b/tests/test_config_merging/config_cmdline_01.yaml @@ -0,0 +1,4 @@ +block: + foo: cfg01_foo + bar: cfg01_bar + baz: cfg01_baz diff --git a/tests/test_config_merging/config_cmdline_02.yaml b/tests/test_config_merging/config_cmdline_02.yaml new file mode 100644 index 000000000..c772fe7fb --- /dev/null +++ b/tests/test_config_merging/config_cmdline_02.yaml @@ -0,0 +1,4 @@ +block: + foo: cfg02_foo + qux: cfg02_qux + bowser: cfg02_bowser diff --git a/tests/test_config_merging/config_snakefile.yaml b/tests/test_config_merging/config_snakefile.yaml new file mode 100644 index 000000000..c5fbe0a86 --- /dev/null +++ b/tests/test_config_merging/config_snakefile.yaml @@ -0,0 +1,3 @@ +block: + bar: snake_bar + fubar: snake_fubar diff --git a/tests/test_config_merging/expected-results/test.out b/tests/test_config_merging/expected-results/test.out new file mode 100644 index 000000000..967372364 --- /dev/null +++ b/tests/test_config_merging/expected-results/test.out @@ -0,0 +1 @@ +{"block": {"bar": "cfg01_bar", "baz": "cfg01_baz", "bowser": "cmdline_bowser", "foo": "cfg02_foo", "fubar": "snake_fubar", "qux": "cfg02_qux", "toad": "cmdline_toad"}} \ No newline at end of file diff --git a/tests/tests.py b/tests/tests.py index a86f06ce6..174ebc445 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -264,6 +264,13 @@ def test_update_config(): run(dpath("test_update_config")) +def test_config_merging(): + run( + dpath("test_config_merging"), + shellcmd='snakemake -j 1 --configfile config_cmdline_01.yaml config_cmdline_02.yaml --config "block={bowser: cmdline_bowser}" "block={toad: cmdline_toad}"', + ) + + def test_wildcard_keyword(): run(dpath("test_wildcard_keyword")) @@ -1346,3 +1353,8 @@ def test_converting_path_for_r_script(): def test_ancient_dag(): run(dpath("test_ancient_dag")) + + +@skip_on_windows +def test_checkpoint_allowed_rules(): + run(dpath("test_checkpoint_allowed_rules"), targets=["c"], cluster="./qsub")