From 426d92fd9610b61b414b7f0152d777c463c939a2 Mon Sep 17 00:00:00 2001 From: Joseph K Aicher <4666753+jaicher@users.noreply.github.com> Date: Sat, 27 Nov 2021 02:24:46 -0500 Subject: [PATCH] fix: issue #1257 (missing logfile failure when using shadow directory) (#1258) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Test for issue #1257 fails for missing logfile with shadow directory * If rule did not create logfile, create it in its shadow directory This addresses gh #1257. If the rule did not create a logfile, an empty one would be created in its final destination. If the shadow directive was used, however, Snakemake would try to move the nonexistent logfile from the shadow directory to its final location. This caused the observed failures. This commit has Snakemake create the empty file where Snakemake would have expected the program to create it (i.e. in the shadow directory). This enables the unshadow operation that takes place later to succeed. * move remote handling of log files into the general remote output handling; fix existence check Co-authored-by: Johannes Köster Co-authored-by: Johannes Köster --- snakemake/dag.py | 11 +++-------- tests/test_shadowed_log/Snakefile | 4 ++++ tests/test_shadowed_log/expected-results/all.log | 0 tests/test_shadowed_log/expected-results/all.out | 0 tests/tests.py | 5 +++++ 5 files changed, 12 insertions(+), 8 deletions(-) create mode 100644 tests/test_shadowed_log/Snakefile create mode 100644 tests/test_shadowed_log/expected-results/all.log create mode 100644 tests/test_shadowed_log/expected-results/all.out diff --git a/snakemake/dag.py b/snakemake/dag.py index a8f5e8578..655148e13 100755 --- a/snakemake/dag.py +++ b/snakemake/dag.py @@ -638,17 +638,10 @@ def unneeded_files(): def handle_log(self, job, upload_remote=True): for f in job.log: + f = job.shadowed_path(f) if not f.exists_local: # If log file was not created during job, create an empty one. f.touch_or_create() - if upload_remote and f.is_remote and not f.should_stay_on_remote: - f.upload_to_remote() - if not f.exists_remote: - raise RemoteFileException( - "The file upload was attempted, but it does not " - "exist on remote. Check that your credentials have " - "read AND write permissions." - ) def handle_remote(self, job, upload=True): """Remove local files if they are no longer needed and upload.""" @@ -657,6 +650,8 @@ def handle_remote(self, job, upload=True): files = job.expanded_output if job.benchmark: files = chain(job.expanded_output, (job.benchmark,)) + if job.log: + files = chain(files, job.log) for f in files: if f.is_remote and not f.should_stay_on_remote: f.upload_to_remote() diff --git a/tests/test_shadowed_log/Snakefile b/tests/test_shadowed_log/Snakefile new file mode 100644 index 000000000..780f8c444 --- /dev/null +++ b/tests/test_shadowed_log/Snakefile @@ -0,0 +1,4 @@ +rule all: + shadow: "minimal" + output: touch("all.out") + log: "all.log" diff --git a/tests/test_shadowed_log/expected-results/all.log b/tests/test_shadowed_log/expected-results/all.log new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_shadowed_log/expected-results/all.out b/tests/test_shadowed_log/expected-results/all.out new file mode 100644 index 000000000..e69de29bb diff --git a/tests/tests.py b/tests/tests.py index 174ebc445..f11dcd806 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -357,6 +357,11 @@ def test_shadow_prefix_qsub(): run(dpath("test_shadow_prefix"), shadow_prefix="shadowdir", cluster="./qsub") +@skip_on_windows +def test_shadowed_log(): + run(dpath("test_shadowed_log")) + + def test_until(): run( dpath("test_until"),