Skip to content

Commit

Permalink
Merge branch 'main' of github.com:snakemake/snakemake
Browse files Browse the repository at this point in the history
  • Loading branch information
johanneskoester committed Mar 31, 2022
2 parents e6826b6 + ce9a064 commit 8628a3e
Show file tree
Hide file tree
Showing 9 changed files with 63 additions and 9 deletions.
17 changes: 17 additions & 0 deletions CHANGELOG.md
@@ -1,5 +1,22 @@
# Changelog

### [7.3.4](https://www.github.com/snakemake/snakemake/compare/v7.3.3...v7.3.4) (2022-03-30)


### Bug Fixes

* better error messages in case of missing files after latency period ([#1528](https://www.github.com/snakemake/snakemake/issues/1528)) ([5b394c0](https://www.github.com/snakemake/snakemake/commit/5b394c0319cfc5f8000b616d0f3c911a9091d05b))
* correct handling of exceptions in input functions that are generators ([#1536](https://www.github.com/snakemake/snakemake/issues/1536)) ([d9a56aa](https://www.github.com/snakemake/snakemake/commit/d9a56aaf75c5f70ba0217d9d461d839fa3013f2e))
* obtaining conda prefix when using in combination with singularity ([#1535](https://www.github.com/snakemake/snakemake/issues/1535)) ([99b22d3](https://www.github.com/snakemake/snakemake/commit/99b22d33f2100a7e4cf2d080a2272959a712d055))
* proper error message in case of missing git when checking for source files ([#1534](https://www.github.com/snakemake/snakemake/issues/1534)) ([92887a3](https://www.github.com/snakemake/snakemake/commit/92887a33dc3674f942bd0355edfbba53b810f18f))
* throw error message in case of target rule that depends on a pipe. ([#1532](https://www.github.com/snakemake/snakemake/issues/1532)) ([b9e9a7e](https://www.github.com/snakemake/snakemake/commit/b9e9a7eff4b6e3349dde6b90eec9f5a37ef69ce7))


### Documentation

* display rust-script env. ([950d8ba](https://www.github.com/snakemake/snakemake/commit/950d8ba785a384fa47fcda3d6fb948799a259e0e))
* zenodo example ([76159ae](https://www.github.com/snakemake/snakemake/commit/76159ae22539e38923712e487371a5f32d7cb3cf))

### [7.3.3](https://www.github.com/snakemake/snakemake/compare/v7.3.2...v7.3.3) (2022-03-28)


Expand Down
5 changes: 1 addition & 4 deletions snakemake/dag.py
Expand Up @@ -502,10 +502,7 @@ def check_and_touch_output(
)
except IOError as e:
raise MissingOutputException(
str(e) + "\nThis might be due to "
"filesystem latency. If that is the case, consider to increase the "
"wait time with --latency-wait."
+ "\nJob id: {jobid}".format(jobid=job.jobid),
str(e),
rule=job.rule,
jobid=self.jobid(job),
)
Expand Down
2 changes: 1 addition & 1 deletion snakemake/exceptions.py
Expand Up @@ -510,7 +510,7 @@ class CheckSumMismatchException(WorkflowError):
class IncompleteCheckpointException(Exception):
def __init__(self, rule, targetfile):
super().__init__(
"The requested checkpoint output is not yet created."
"The requested checkpoint output is not yet created. "
"If you see this error, you have likely tried to use "
"checkpoint output outside of an input function, or "
"you have tried to call an input function directly "
Expand Down
4 changes: 3 additions & 1 deletion snakemake/executors/__init__.py
Expand Up @@ -465,6 +465,9 @@ def get_python_executable(self):
def get_envvar_declarations(self):
return ""

def get_job_args(self, job, **kwargs):
return f"{super().get_job_args(job, **kwargs)} --quiet"

def run(self, job, callback=None, submit_callback=None, error_callback=None):
super()._run(job)

Expand Down Expand Up @@ -581,7 +584,6 @@ def run_group_job(self, job):

def spawn_job(self, job):
cmd = self.format_job_exec(job)
print(cmd)
try:
subprocess.check_call(cmd, shell=True)
except subprocess.CalledProcessError as e:
Expand Down
8 changes: 5 additions & 3 deletions snakemake/io.py
Expand Up @@ -824,10 +824,12 @@ def get_missing():
if not get_missing():
return
time.sleep(1)
missing = "\n".join(get_missing())
raise IOError(
"Missing files after {} seconds:\n{}".format(
latency_wait, "\n".join(get_missing())
)
f"Missing files after {latency_wait} seconds. This might be due to "
"filesystem latency. If that is the case, consider to increase the "
"wait time with --latency-wait:\n"
f"{missing}"
)


Expand Down
6 changes: 6 additions & 0 deletions snakemake/rules.py
Expand Up @@ -5,6 +5,7 @@

import os
import re
import types
from snakemake.path_modifier import PATH_MODIFIER_FLAG
import sys
import inspect
Expand Down Expand Up @@ -753,6 +754,11 @@ def apply_input_function(

try:
value = func(Wildcards(fromdict=wildcards), **_aux_params)
if isinstance(value, types.GeneratorType):
# generators should be immediately collected here,
# otherwise we would miss any exceptions and
# would have to capture them again later.
value = list(value)
except IncompleteCheckpointException as e:
value = incomplete_checkpoint_func(e)
incomplete = True
Expand Down
25 changes: 25 additions & 0 deletions tests/test_github_issue261/Snakefile
@@ -0,0 +1,25 @@
import random

checkpoint random:
output:
target = 'test1/{target}/{config}.txt'

run:
with open(output.target, 'w') as ftg:
for i in range(2, 10):
ftg.write(f'test1/{wildcards.target}/{wildcards.config}/v{i}\n')

rule process:
output:
touch('test1/{target}/{config}/v{index}')

def genetate_inputs(wildcards):
with checkpoints.random.get(**wildcards).output[0].open() as fran:
for line in fran.readlines():
yield line.rstrip()

rule generate:
input:
genetate_inputs
output:
touch('test1/{target}/{config}.done')
Empty file.
5 changes: 5 additions & 0 deletions tests/tests.py
Expand Up @@ -1567,6 +1567,11 @@ def test_incomplete_params():
run(dpath("test_incomplete_params"), dryrun=True, printshellcmds=True)


@skip_on_windows
def test_github_issue261():
run(dpath("test_github_issue261"), targets=["test1/target1/config1.done"])


@skip_on_windows # no pipe support on windows
def test_pipe_depend():
run(dpath("test_pipe_depend"), shouldfail=True)
Expand Down

0 comments on commit 8628a3e

Please sign in to comment.