From d243c22ff494b63bd5e07b7c5bf1f6ff32539cde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Susana=20Posada=20C=C3=A9spedes?= Date: Thu, 21 Oct 2021 13:39:02 +0200 Subject: [PATCH] fix: enable usage of job grouping in GLS (#1054) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Allow using grouped rules on Google cloud using Google Life Science API * Explain new behaviour in the docs * Modify behaviour to request all rules in a group to be set as preemptible instances * Fix typo and add comment about documentation Co-authored-by: Johannes Köster --- docs/snakefiles/rules.rst | 2 ++ snakemake/executors/google_lifesciences.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/snakefiles/rules.rst b/docs/snakefiles/rules.rst index 6be2b2eda..0086a694a 100644 --- a/docs/snakefiles/rules.rst +++ b/docs/snakefiles/rules.rst @@ -1492,6 +1492,8 @@ However, if we would add ``group: "mygroup"`` to rule ``c``, all jobs would end Alternatively, groups can be defined via the command line interface. This enables to almost arbitrarily partition the DAG, e.g. in order to safe network traffic, see :ref:`here `. +For execution on the cloud using Google Life Science API and preemptible instances, we expect all rules in the group to be homogenously set as preemptible instances (e.g., with command-line option ``--preemptible-rules``), such that a preemptible VM is requested for the execution of the group job. + Piped output ------------ diff --git a/snakemake/executors/google_lifesciences.py b/snakemake/executors/google_lifesciences.py index f7d27e24e..5259af3cd 100644 --- a/snakemake/executors/google_lifesciences.py +++ b/snakemake/executors/google_lifesciences.py @@ -511,12 +511,23 @@ def _generate_job_resources(self, job): "Selected machine type {}:{}".format(smallest, selected["description"]) ) + if job.is_group(): + preemptible = all(rule in self.preemptible_rules for rule in job.rules) + if not preemptible and any( + rule in self.preemptible_rules for rule in job.rules + ): + raise WorkflowError( + "All grouped rules should be homogenously set as preemptible rules" + "(see Defining groups for execution in snakemake documentation)" + ) + else: + preemptible = job.rule.name in self.preemptible_rules # We add the size for the image itself (10 GB) to bootDiskSizeGb virtual_machine = { "machineType": smallest, "labels": {"app": "snakemake"}, "bootDiskSizeGb": disk_gb + 10, - "preemptible": job.rule.name in self.preemptible_rules, + "preemptible": preemptible, } # If the user wants gpus, add accelerators here