fix: issue #1257 (missing logfile failure when using shadow directory) (

#1258)

* Test for issue #1257 fails for missing logfile with shadow directory

* If rule did not create logfile, create it in its shadow directory

This addresses gh #1257. If the rule did not create a logfile, an empty
one would be created in its final destination. If the shadow directive
was used, however, Snakemake would try to move the nonexistent logfile
from the shadow directory to its final location. This caused the
observed failures. This commit has Snakemake create the empty file where
Snakemake would have expected the program to create it (i.e. in the
shadow directory). This enables the unshadow operation that takes place
later to succeed.

* move remote handling of log files into the general remote output handling; fix existence check

Co-authored-by: Johannes Köster <johannes.koester@tu-dortmund.de>
Co-authored-by: Johannes Köster <johannes.koester@uni-due.de>

snakemake/dag.py

@@ -638,17 +638,10 @@ def unneeded_files():
 
     def handle_log(self, job, upload_remote=True):
         for f in job.log:
+            f = job.shadowed_path(f)
             if not f.exists_local:
                 # If log file was not created during job, create an empty one.
                 f.touch_or_create()
-            if upload_remote and f.is_remote and not f.should_stay_on_remote:
-                f.upload_to_remote()
-                if not f.exists_remote:
-                    raise RemoteFileException(
-                        "The file upload was attempted, but it does not "
-                        "exist on remote. Check that your credentials have "
-                        "read AND write permissions."
-                    )
 
     def handle_remote(self, job, upload=True):
         """Remove local files if they are no longer needed and upload."""
@@ -657,6 +650,8 @@ def handle_remote(self, job, upload=True):
             files = job.expanded_output
             if job.benchmark:
                 files = chain(job.expanded_output, (job.benchmark,))
+            if job.log:
+                files = chain(files, job.log)
             for f in files:
                 if f.is_remote and not f.should_stay_on_remote:
                     f.upload_to_remote()

tests/test_shadowed_log/Snakefile

@@ -0,0 +1,4 @@
+rule all:
+    shadow: "minimal"
+    output: touch("all.out")
+    log: "all.log"

tests/tests.py

@@ -357,6 +357,11 @@ def test_shadow_prefix_qsub():
     run(dpath("test_shadow_prefix"), shadow_prefix="shadowdir", cluster="./qsub")
 
 
+@skip_on_windows
+def test_shadowed_log():
+    run(dpath("test_shadowed_log"))
+
+
 def test_until():
     run(
         dpath("test_until"),