fix: Use 'snakemake.utils.update_config' instead of 'dict.update' (#1126

)

* Use 'snakemake.utils.update_config' instead of 'dict.update' to update configs

* Add config merging test

* Improve testcase

* Add '--config' to test by using 'shellcmd' parameter

* Store config as JSON in testcase

* Fix and test parsing of multiple '--config' entries

* add debug statement

* fmt

* dbg

* fix test on windows

* dbg

* cleanup and test case fixes

* dbg

* fmt

* dbg

* dbg

* dbg

* dbg

* fmt

* fix: make a copy of overwrite_configfiles

Co-authored-by: Johannes Köster <johannes.koester@uni-due.de>
Co-authored-by: Johannes Köster <johannes.koester@tu-dortmund.de>

.github/workflows/main.yml

@@ -95,7 +95,7 @@ jobs:
           # activate conda env
           export PATH="/usr/share/miniconda/bin:$PATH"
           source activate snakemake
-
+          
           pytest -v -x tests/test_expand.py tests/test_io.py tests/test_schema.py tests/test_linting.py tests/tests.py
 
       - name: Build and publish docker image

snakemake/__init__.py

@@ -497,13 +497,13 @@ def snakemake(
         configfiles = []
     for f in configfiles:
         # get values to override. Later configfiles override earlier ones.
-        overwrite_config.update(load_configfile(f))
+        update_config(overwrite_config, load_configfile(f))
     # convert provided paths to absolute paths
     configfiles = list(map(os.path.abspath, configfiles))
 
     # directly specified elements override any configfiles
     if config:
-        overwrite_config.update(config)
+        update_config(overwrite_config, config)
         if config_args is None:
             config_args = unparse_config(config)
 
@@ -947,7 +947,7 @@ def parse_config(args):
                 except:
                     pass
             assert v is not None
-            config[key] = v
+            update_config(config, {key: v})
     return config
 
 

snakemake/workflow.py

@@ -204,7 +204,7 @@ def __init__(
         self.attempt = attempt
         self.default_remote_provider = default_remote_provider
         self.default_remote_prefix = default_remote_prefix
-        self.configfiles = overwrite_configfiles or []
+        self.configfiles = list(overwrite_configfiles) or []
         self.run_local = run_local
         self.report_text = None
         self.conda_cleanup_pkgs = conda_cleanup_pkgs

tests/test_config_merging/Snakefile

@@ -0,0 +1,11 @@
+configfile: "config_snakefile.yaml"
+
+
+rule dump_config:
+    output:
+        "test.out",
+    run:
+        import json
+
+        with open(output[0], "w") as fd:
+            json.dump(config, fd, sort_keys=True)

tests/test_config_merging/config_cmdline_01.yaml

@@ -0,0 +1,4 @@
+block:
+    foo: cfg01_foo
+    bar: cfg01_bar
+    baz: cfg01_baz

tests/test_config_merging/config_cmdline_02.yaml

@@ -0,0 +1,4 @@
+block:
+    foo: cfg02_foo
+    qux: cfg02_qux
+    bowser: cfg02_bowser

tests/test_config_merging/config_snakefile.yaml

@@ -0,0 +1,3 @@
+block:
+    bar: snake_bar
+    fubar: snake_fubar

tests/test_config_merging/expected-results/test.out

@@ -0,0 +1 @@
+{"block": {"bar": "cfg01_bar", "baz": "cfg01_baz", "bowser": "cmdline_bowser", "foo": "cfg02_foo", "fubar": "snake_fubar", "qux": "cfg02_qux", "toad": "cmdline_toad"}}

tests/tests.py

@@ -264,6 +264,13 @@ def test_update_config():
     run(dpath("test_update_config"))
 
 
+def test_config_merging():
+    run(
+        dpath("test_config_merging"),
+        shellcmd='snakemake -j 1 --configfile config_cmdline_01.yaml config_cmdline_02.yaml --config "block={bowser: cmdline_bowser}" "block={toad: cmdline_toad}"',
+    )
+
+
 def test_wildcard_keyword():
     run(dpath("test_wildcard_keyword"))