pushing test fix

the issue is that the blob object held by a GS remote object can go sort
of stale, returning a False for blob.exists() when it clearly exists! To fix
we need to do an additional self.update_blob() and then returning the exists
check again. I am not sure if this can be made more efficient by only checking
under certain conditions, but since it seems likely we cannot perfectly know when
the blob has gone stale the sure way is to always update.

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

snakemake/io.py

@@ -96,6 +96,7 @@ def lutime(f, times):
     def lchmod(f, mode):
         os.chmod(f, mode, follow_symlinks=False)
 
+
 else:
 
     def lchmod(f, mode):

snakemake/jobs.py

@@ -1410,6 +1410,8 @@ def cleanup(self):
             job.cleanup()
 
     def postprocess(self, error=False, **kwargs):
+        # Intermediate jobs that are remove we can leave on remote
+        # force_stay_on_remote=False
         for job in self.jobs:
             job.postprocess(error=error, **kwargs)
         # remove all pipe and service outputs since all jobs of this group are done and the

snakemake/remote/GS.py

@@ -206,8 +206,10 @@ def exists(self):
             return True
         elif any(self.directory_entries()):
             return True
-        else:
-            return False
+
+        # The blob object can get out of sync, one last try!
+        self.update_blob()
+        return self.blob.exists()
 
     @retry.Retry(predicate=google_cloud_retry_predicate)
     def mtime(self):

snakemake/scheduler.py

@@ -517,6 +517,7 @@ def schedule(self):
     def _finish_jobs(self):
         # must be called from within lock
         # clear the global tofinish such that parallel calls do not interfere
+
         for job in self._tofinish:
             if self.handle_job_success:
                 try:

tests/conftest.py

@@ -9,7 +9,9 @@
 
 skip_on_windows = pytest.mark.skipif(ON_WINDOWS, reason="Unix stuff")
 only_on_windows = pytest.mark.skipif(not ON_WINDOWS, reason="Windows stuff")
-needs_strace = pytest.mark.xfail(os.system("strace -o /dev/null true") != 0, reason="Missing strace")
+needs_strace = pytest.mark.xfail(
+    os.system("strace -o /dev/null true") != 0, reason="Missing strace"
+)
 
 
 @pytest.fixture(autouse=True)

tests/test_google_lifesciences.py

@@ -28,6 +28,7 @@ def cleanup_google_storage(prefix, bucket_name="snakemake-testing", restrict_to=
     Arguments:
       prefix (str) : the "subfolder" or prefix for some files in the buckets
       bucket_name (str) : the name of the bucket, default snakemake-testing
+      restrict_to (list) : only delete files in these paths (None deletes all)
     """
     client = storage.Client()
     bucket = client.get_bucket(bucket_name)
@@ -134,6 +135,9 @@ def test_github_issue1396():
 
 
 def test_github_issue1460():
+    import IPython
+
+    IPython.embed()
     bucket_name = "snakemake-testing-%s" % next(tempfile._get_candidate_names())
     create_google_storage(bucket_name)
     storage_prefix = "test_github_issue1460"

tests/test_io.py

@@ -104,8 +104,11 @@ def __repr__(self):
     ) == sorted(["a: aa + b: b", "a: aa + b: bb", "c: c", "c: cc"])
 
     # expand on pathlib.Path objects
-    assert expand(
-        PosixPath() / "{x}" / "{y}",
-        x="Hello",
-        y="world",
-    ) == ["Hello/world"]
+    assert (
+        expand(
+            PosixPath() / "{x}" / "{y}",
+            x="Hello",
+            y="world",
+        )
+        == ["Hello/world"]
+    )

tests/testapi.py

@@ -72,4 +72,11 @@ def test_dicts_in_config():
                 ),
                 file=f,
             )
-        snakemake(path, workdir=tmpdir, config={"this_option": "does_not_break", "test": {'this_dict':'shoult_not_either'}})
+        snakemake(
+            path,
+            workdir=tmpdir,
+            config={
+                "this_option": "does_not_break",
+                "test": {"this_dict": "shoult_not_either"},
+            },
+        )