From 2658027458dde4c10b3d6e1af7671564d175f9cb Mon Sep 17 00:00:00 2001 From: Kim Date: Thu, 25 Nov 2021 13:56:16 +0100 Subject: [PATCH] fix: Use 'snakemake.utils.update_config' instead of 'dict.update' (#1126) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Use 'snakemake.utils.update_config' instead of 'dict.update' to update configs * Add config merging test * Improve testcase * Add '--config' to test by using 'shellcmd' parameter * Store config as JSON in testcase * Fix and test parsing of multiple '--config' entries * add debug statement * fmt * dbg * fix test on windows * dbg * cleanup and test case fixes * dbg * fmt * dbg * dbg * dbg * dbg * fmt * fix: make a copy of overwrite_configfiles Co-authored-by: Johannes Köster Co-authored-by: Johannes Köster --- .github/workflows/main.yml | 2 +- snakemake/__init__.py | 6 +++--- snakemake/workflow.py | 2 +- tests/test_config_merging/Snakefile | 11 +++++++++++ tests/test_config_merging/config_cmdline_01.yaml | 4 ++++ tests/test_config_merging/config_cmdline_02.yaml | 4 ++++ tests/test_config_merging/config_snakefile.yaml | 3 +++ tests/test_config_merging/expected-results/test.out | 1 + tests/tests.py | 7 +++++++ 9 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 tests/test_config_merging/Snakefile create mode 100644 tests/test_config_merging/config_cmdline_01.yaml create mode 100644 tests/test_config_merging/config_cmdline_02.yaml create mode 100644 tests/test_config_merging/config_snakefile.yaml create mode 100644 tests/test_config_merging/expected-results/test.out diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 030d76a1b..be9e8078e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -95,7 +95,7 @@ jobs: # activate conda env export PATH="/usr/share/miniconda/bin:$PATH" source activate snakemake - + pytest -v -x tests/test_expand.py tests/test_io.py tests/test_schema.py tests/test_linting.py tests/tests.py - name: Build and publish docker image diff --git a/snakemake/__init__.py b/snakemake/__init__.py index aee6066fa..7c08e4d98 100644 --- a/snakemake/__init__.py +++ b/snakemake/__init__.py @@ -497,13 +497,13 @@ def snakemake( configfiles = [] for f in configfiles: # get values to override. Later configfiles override earlier ones. - overwrite_config.update(load_configfile(f)) + update_config(overwrite_config, load_configfile(f)) # convert provided paths to absolute paths configfiles = list(map(os.path.abspath, configfiles)) # directly specified elements override any configfiles if config: - overwrite_config.update(config) + update_config(overwrite_config, config) if config_args is None: config_args = unparse_config(config) @@ -947,7 +947,7 @@ def parse_config(args): except: pass assert v is not None - config[key] = v + update_config(config, {key: v}) return config diff --git a/snakemake/workflow.py b/snakemake/workflow.py index fb4a23159..db4c69306 100644 --- a/snakemake/workflow.py +++ b/snakemake/workflow.py @@ -204,7 +204,7 @@ def __init__( self.attempt = attempt self.default_remote_provider = default_remote_provider self.default_remote_prefix = default_remote_prefix - self.configfiles = overwrite_configfiles or [] + self.configfiles = list(overwrite_configfiles) or [] self.run_local = run_local self.report_text = None self.conda_cleanup_pkgs = conda_cleanup_pkgs diff --git a/tests/test_config_merging/Snakefile b/tests/test_config_merging/Snakefile new file mode 100644 index 000000000..1faef6651 --- /dev/null +++ b/tests/test_config_merging/Snakefile @@ -0,0 +1,11 @@ +configfile: "config_snakefile.yaml" + + +rule dump_config: + output: + "test.out", + run: + import json + + with open(output[0], "w") as fd: + json.dump(config, fd, sort_keys=True) diff --git a/tests/test_config_merging/config_cmdline_01.yaml b/tests/test_config_merging/config_cmdline_01.yaml new file mode 100644 index 000000000..fc731e9b5 --- /dev/null +++ b/tests/test_config_merging/config_cmdline_01.yaml @@ -0,0 +1,4 @@ +block: + foo: cfg01_foo + bar: cfg01_bar + baz: cfg01_baz diff --git a/tests/test_config_merging/config_cmdline_02.yaml b/tests/test_config_merging/config_cmdline_02.yaml new file mode 100644 index 000000000..c772fe7fb --- /dev/null +++ b/tests/test_config_merging/config_cmdline_02.yaml @@ -0,0 +1,4 @@ +block: + foo: cfg02_foo + qux: cfg02_qux + bowser: cfg02_bowser diff --git a/tests/test_config_merging/config_snakefile.yaml b/tests/test_config_merging/config_snakefile.yaml new file mode 100644 index 000000000..c5fbe0a86 --- /dev/null +++ b/tests/test_config_merging/config_snakefile.yaml @@ -0,0 +1,3 @@ +block: + bar: snake_bar + fubar: snake_fubar diff --git a/tests/test_config_merging/expected-results/test.out b/tests/test_config_merging/expected-results/test.out new file mode 100644 index 000000000..967372364 --- /dev/null +++ b/tests/test_config_merging/expected-results/test.out @@ -0,0 +1 @@ +{"block": {"bar": "cfg01_bar", "baz": "cfg01_baz", "bowser": "cmdline_bowser", "foo": "cfg02_foo", "fubar": "snake_fubar", "qux": "cfg02_qux", "toad": "cmdline_toad"}} \ No newline at end of file diff --git a/tests/tests.py b/tests/tests.py index a86f06ce6..21284d9d7 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -264,6 +264,13 @@ def test_update_config(): run(dpath("test_update_config")) +def test_config_merging(): + run( + dpath("test_config_merging"), + shellcmd='snakemake -j 1 --configfile config_cmdline_01.yaml config_cmdline_02.yaml --config "block={bowser: cmdline_bowser}" "block={toad: cmdline_toad}"', + ) + + def test_wildcard_keyword(): run(dpath("test_wildcard_keyword"))