Merge branch 'main' of github.com:snakemake/snakemake

.github/workflows/main.yml

@@ -95,7 +95,7 @@ jobs:
           # activate conda env
           export PATH="/usr/share/miniconda/bin:$PATH"
           source activate snakemake
-
+          
           pytest -v -x tests/test_expand.py tests/test_io.py tests/test_schema.py tests/test_linting.py tests/tests.py
 
       - name: Build and publish docker image

CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## [6.11.0](https://www.github.com/snakemake/snakemake/compare/v6.10.0...v6.11.0) (2021-11-25)
+
+
+### Features
+
+* fail with an error if snakemake cannot write job metadata. ([#1273](https://www.github.com/snakemake/snakemake/issues/1273)) ([cd968cd](https://www.github.com/snakemake/snakemake/commit/cd968cd03437ad6db1d791f5d7ae5295b9754137))
+
+
+### Bug Fixes
+
+* Adds fixes for the first two MREs in [#823](https://www.github.com/snakemake/snakemake/issues/823) ([#1215](https://www.github.com/snakemake/snakemake/issues/1215)) ([cfd2f89](https://www.github.com/snakemake/snakemake/commit/cfd2f890a0af57628f7b9278d8d43f59b7006825))
+* env file usage after changes to source file handling (inspired by [#1233](https://www.github.com/snakemake/snakemake/issues/1233) and [#1211](https://www.github.com/snakemake/snakemake/issues/1211)). ([#1236](https://www.github.com/snakemake/snakemake/issues/1236)) ([3ac8e85](https://www.github.com/snakemake/snakemake/commit/3ac8e858a7b908326922c8f68cae512b1250e906))
+* fixed code change detection when using modules ([#1264](https://www.github.com/snakemake/snakemake/issues/1264)) ([b571e09](https://www.github.com/snakemake/snakemake/commit/b571e09ce452f6a1a95395e1c3c8b9e3f83867ad))
+* handle config file extension/overwriting more explicitly ([#1251](https://www.github.com/snakemake/snakemake/issues/1251)) ([d0a7bf2](https://www.github.com/snakemake/snakemake/commit/d0a7bf243c5df204136fa1f14706aab793793c68))
+* Issue [#1253](https://www.github.com/snakemake/snakemake/issues/1253) (problems editing Jupyter Notebooks) ([#1255](https://www.github.com/snakemake/snakemake/issues/1255)) ([3398ddf](https://www.github.com/snakemake/snakemake/commit/3398ddffd1f68182af768ef4ea519e9a9ad4efaf))
+* more informative nothing to be done message ([#1234](https://www.github.com/snakemake/snakemake/issues/1234)) ([368d265](https://www.github.com/snakemake/snakemake/commit/368d265ff3da984bd3a53b319dcb882d6916975b))
+* only consider context of shell command for technical switches if called from snakemake rules. ([#1213](https://www.github.com/snakemake/snakemake/issues/1213)) ([4816a58](https://www.github.com/snakemake/snakemake/commit/4816a58653e466ca94b1482a1d947a856f5381b3))
+* R encoding of pathlib.Path objects ([#1201](https://www.github.com/snakemake/snakemake/issues/1201)) ([bd516e9](https://www.github.com/snakemake/snakemake/commit/bd516e958af22e57c18cacf0cb22552c2a237bd8))
+* Use 'snakemake.utils.update_config' instead of 'dict.update' ([#1126](https://www.github.com/snakemake/snakemake/issues/1126)) ([2658027](https://www.github.com/snakemake/snakemake/commit/2658027458dde4c10b3d6e1af7671564d175f9cb))
+
 ## [6.10.0](https://www.github.com/snakemake/snakemake/compare/v6.9.1...v6.10.0) (2021-10-21)
 
 

snakemake/__init__.py

@@ -497,13 +497,13 @@ def snakemake(
         configfiles = []
     for f in configfiles:
         # get values to override. Later configfiles override earlier ones.
-        overwrite_config.update(load_configfile(f))
+        update_config(overwrite_config, load_configfile(f))
     # convert provided paths to absolute paths
     configfiles = list(map(os.path.abspath, configfiles))
 
     # directly specified elements override any configfiles
     if config:
-        overwrite_config.update(config)
+        update_config(overwrite_config, config)
         if config_args is None:
             config_args = unparse_config(config)
 
@@ -947,7 +947,7 @@ def parse_config(args):
                 except:
                     pass
             assert v is not None
-            config[key] = v
+            update_config(config, {key: v})
     return config
 
 

snakemake/workflow.py

@@ -204,7 +204,7 @@ def __init__(
         self.attempt = attempt
         self.default_remote_provider = default_remote_provider
         self.default_remote_prefix = default_remote_prefix
-        self.configfiles = overwrite_configfiles or []
+        self.configfiles = list(overwrite_configfiles) or []
         self.run_local = run_local
         self.report_text = None
         self.conda_cleanup_pkgs = conda_cleanup_pkgs
@@ -682,7 +682,8 @@ def files(items):
 
         rules = self.rules
         if allowed_rules:
-            rules = [rule for rule in rules if rule.name in set(allowed_rules)]
+            allowed_rules = set(allowed_rules)
+            rules = [rule for rule in rules if rule.name in allowed_rules]
 
         if wait_for_files is not None:
             try:

tests/test_checkpoint_allowed_rules/Snakefile

@@ -0,0 +1,13 @@
+checkpoint a:
+    output: touch("a.txt")
+
+rule b:
+    output: touch("b.txt")
+
+def get_input(wildcards):
+    checkpoints.a.get()
+    return "b.txt"
+
+rule c:
+    input: get_input
+    shell: "echo {input}"

tests/test_checkpoint_allowed_rules/qsub

@@ -0,0 +1,7 @@
+#!/bin/bash
+echo `date` >> qsub.log
+tail -n1 $1 >> qsub.log
+# simulate printing of job id by a random number
+echo $RANDOM
+cat $1 >> qsub.log
+sh $1

tests/test_config_merging/Snakefile

@@ -0,0 +1,11 @@
+configfile: "config_snakefile.yaml"
+
+
+rule dump_config:
+    output:
+        "test.out",
+    run:
+        import json
+
+        with open(output[0], "w") as fd:
+            json.dump(config, fd, sort_keys=True)

tests/test_config_merging/config_cmdline_01.yaml

@@ -0,0 +1,4 @@
+block:
+    foo: cfg01_foo
+    bar: cfg01_bar
+    baz: cfg01_baz

tests/test_config_merging/config_cmdline_02.yaml

@@ -0,0 +1,4 @@
+block:
+    foo: cfg02_foo
+    qux: cfg02_qux
+    bowser: cfg02_bowser

tests/test_config_merging/config_snakefile.yaml

@@ -0,0 +1,3 @@
+block:
+    bar: snake_bar
+    fubar: snake_fubar

tests/test_config_merging/expected-results/test.out

@@ -0,0 +1 @@
+{"block": {"bar": "cfg01_bar", "baz": "cfg01_baz", "bowser": "cmdline_bowser", "foo": "cfg02_foo", "fubar": "snake_fubar", "qux": "cfg02_qux", "toad": "cmdline_toad"}}

tests/tests.py

@@ -264,6 +264,13 @@ def test_update_config():
     run(dpath("test_update_config"))
 
 
+def test_config_merging():
+    run(
+        dpath("test_config_merging"),
+        shellcmd='snakemake -j 1 --configfile config_cmdline_01.yaml config_cmdline_02.yaml --config "block={bowser: cmdline_bowser}" "block={toad: cmdline_toad}"',
+    )
+
+
 def test_wildcard_keyword():
     run(dpath("test_wildcard_keyword"))
 
@@ -1346,3 +1353,8 @@ def test_converting_path_for_r_script():
 
 def test_ancient_dag():
     run(dpath("test_ancient_dag"))
+
+
+@skip_on_windows
+def test_checkpoint_allowed_rules():
+    run(dpath("test_checkpoint_allowed_rules"), targets=["c"], cluster="./qsub")