Merge branch 'main' of github.com:snakemake/snakemake

snakemake/rules.py

@@ -787,8 +787,6 @@ def _apply_wildcards(
                     groupid=groupid,
                     **aux_params
                 )
-                if apply_path_modifier and not incomplete:
-                    item = self.apply_path_modifier(item, property=property)
 
             if is_unpack and not incomplete:
                 if not allow_unpack:
@@ -808,14 +806,14 @@ def _apply_wildcards(
                     )
                 # Allow streamlined code with/without unpack
                 if isinstance(item, list):
-                    pairs = zip([None] * len(item), item)
+                    pairs = zip([None] * len(item), item, [_is_callable] * len(item))
                 else:
                     assert isinstance(item, dict)
-                    pairs = item.items()
+                    pairs = [(name, item, _is_callable) for name, item in item.items()]
             else:
-                pairs = [(name, item)]
+                pairs = [(name, item, _is_callable)]
 
-            for name, item in pairs:
+            for name, item, from_callable in pairs:
                 is_iterable = True
                 if not_iterable(item) or no_flattening:
                     item = [item]
@@ -829,6 +827,10 @@ def _apply_wildcards(
                         raise WorkflowError(
                             "Function did not return str or list " "of str.", rule=self
                         )
+
+                    if from_callable and apply_path_modifier and not incomplete:
+                        item_ = self.apply_path_modifier(item_, property=property)
+
                     concrete = concretize(item_, wildcards, _is_callable)
                     newitems.append(concrete)
                     if mapping is not None:

snakemake/sourcecache.py

@@ -11,6 +11,7 @@
 import tempfile
 import io
 from abc import ABC, abstractmethod
+from datetime import datetime
 
 
 from snakemake.common import (
@@ -60,6 +61,10 @@ def join(self, path):
             path = path.get_path_or_uri()
         return self.__class__(smart_join(self.get_path_or_uri(), path))
 
+    def mtime(self):
+        """If possible, return mtime of the file. Otherwise, return None."""
+        return None
+
     def __hash__(self):
         return self.get_path_or_uri().__hash__()
 
@@ -111,6 +116,9 @@ def isabs(self):
     def simplify_path(self):
         return utils.simplify_path(self.path)
 
+    def mtime(self):
+        return os.stat(self.path).st_mtime
+
     def __fspath__(self):
         return self.path
 
@@ -225,6 +233,18 @@ class GithubFile(HostingProviderFile):
     def get_path_or_uri(self):
         return "https://github.com/{}/raw/{}/{}".format(self.repo, self.ref, self.path)
 
+    def mtime(self):
+        import requests
+
+        url = f"https://api.github.com/repos/{self.repo}/commits?path={self.path}&page=1&per_page=1"
+        mtime = requests.get(url).json()[0]["commit"]["committer"]["date"]
+        assert mtime.endswith(
+            "Z"
+        ), "bug: expected suffix Z on Github provided time stamp"
+        # assume UTC and make it understandable to fromisoformat
+        mtime = mtime[:-1] + "+00:00"
+        return datetime.fromisoformat(mtime).timestamp()
+
 
 class GitlabFile(HostingProviderFile):
     def __init__(
@@ -345,6 +365,14 @@ def _do_cache(self, source_file, cache_entry):
         ) as cache_source:
             cache_source.write(source.read())
 
+        mtime = source_file.mtime()
+        if mtime is not None:
+            # Set to mtime of original file
+            # In case we don't have that mtime, it is fine
+            # to just keep the time at the time of caching
+            # as mtime.
+            os.utime(cache_entry, times=(mtime, mtime))
+
     def _open(self, path_or_uri, mode):
         from smart_open import open
 

tests/test_github_issue1396/Snakefile

@@ -0,0 +1,16 @@
+def get_files(wildcards):
+    files_1 = expand("file_{i}", i=list(range(1, 5)))
+    files_2 = expand("file_{i}", i=list(range(5, 9)))
+    return {"files_1": files_1, "files_2": files_2}
+
+
+rule all:
+    input:
+        unpack(get_files),
+
+
+rule make_files:
+    output:
+        expand("file_{i}", i=list(range(1, 9))),
+    shell:
+        "touch {output}"

tests/test_google_lifesciences.py

@@ -109,3 +109,20 @@ def test_cloud_checkpoints_issue574():
         )
     finally:
         cleanup_google_storage(storage_prefix, bucket_name)
+
+
+def test_github_issue1396():
+    bucket_name = "snakemake-testing-%s" % next(tempfile._get_candidate_names())
+    create_google_storage(bucket_name)
+    storage_prefix = "test_github_issue1396"
+    workdir = dpath("test_github_issue1396")
+    try:
+        run(
+            workdir,
+            default_remote_prefix="%s/%s" % (bucket_name, storage_prefix),
+            google_lifesciences=True,
+            google_lifesciences_cache=False,
+            dryrun=True,
+        )
+    finally:
+        cleanup_google_storage(storage_prefix, bucket_name)