From 434c4de90b44c2da6011a3a07d27da370787e813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Mon, 30 Jan 2023 12:04:34 +0100 Subject: [PATCH 1/5] fix: properly handle proxy setting in vep cache wrapper --- bio/vep/cache/wrapper.py | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/bio/vep/cache/wrapper.py b/bio/vep/cache/wrapper.py index bcd7ccfa8d..a5a72a0f89 100644 --- a/bio/vep/cache/wrapper.py +++ b/bio/vep/cache/wrapper.py @@ -1,21 +1,35 @@ __author__ = "Johannes Köster" -__copyright__ = "Copyright 2020, Johannes Köster" +__copyright__ = "Copyright 2023, Johannes Köster" __email__ = "johannes.koester@uni-due.de" __license__ = "MIT" +import tempfile from pathlib import Path from snakemake.shell import shell + extra = snakemake.params.get("extra", "") -log = snakemake.log_fmt_shell(stdout=True, stderr=True) -shell( - "vep_install --AUTO cf " - "--SPECIES {snakemake.params.species} " - "--ASSEMBLY {snakemake.params.build} " - "--VERSION {snakemake.params.release} " - "--CACHEDIR {snakemake.output} " - "--CONVERT " - "--NO_UPDATE " - "{extra} {log}" -) +with tempfile.TemporaryDirectory() as tmpdir: + # We download the cache tarball manually because vep_install does not consider proxy settings (in contrast to curl). + # See https://github.com/bcbio/bcbio-nextgen/issues/1080 + cache_tarball = f"{snakemake.params.species}_vep_{snakemake.params.release}_{snakemake.params.build}.tar.gz" + log = snakemake.log_fmt_shell(stdout=True, stderr=True) + shell( + "curl -L ftp://ftp.ensembl.org/pub/release-{snakemake.params.release}/" + "variation/VEP/{cache_tarball} " + "-o {tmpdir.name}/{cache_tarball} {log}" + ) + + log = snakemake.log_fmt_shell(stdout=True, stderr=True, append=True) + shell( + "vep_install --AUTO cf " + "--SPECIES {snakemake.params.species} " + "--ASSEMBLY {snakemake.params.build} " + "--VERSION {snakemake.params.release} " + "--CACHEURL {tmpdir.name} " + "--CACHEDIR {snakemake.output} " + "--CONVERT " + "--NO_UPDATE " + "{extra} {log}" + ) From 05a2764eb29bed254b31a013be6ec1f479f1c92b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Mon, 30 Jan 2023 12:10:48 +0100 Subject: [PATCH 2/5] fix usage --- bio/vep/cache/wrapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bio/vep/cache/wrapper.py b/bio/vep/cache/wrapper.py index a5a72a0f89..1f73123eec 100644 --- a/bio/vep/cache/wrapper.py +++ b/bio/vep/cache/wrapper.py @@ -18,7 +18,7 @@ shell( "curl -L ftp://ftp.ensembl.org/pub/release-{snakemake.params.release}/" "variation/VEP/{cache_tarball} " - "-o {tmpdir.name}/{cache_tarball} {log}" + "-o {tmpdir}/{cache_tarball} {log}" ) log = snakemake.log_fmt_shell(stdout=True, stderr=True, append=True) @@ -27,7 +27,7 @@ "--SPECIES {snakemake.params.species} " "--ASSEMBLY {snakemake.params.build} " "--VERSION {snakemake.params.release} " - "--CACHEURL {tmpdir.name} " + "--CACHEURL {tmpdir} " "--CACHEDIR {snakemake.output} " "--CONVERT " "--NO_UPDATE " From f7a466d88216c763e16ca5c9b5e26ffa77e72f11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Mon, 30 Jan 2023 12:52:49 +0100 Subject: [PATCH 3/5] adapt to ftp server folder structure evolution --- bio/vep/cache/wrapper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bio/vep/cache/wrapper.py b/bio/vep/cache/wrapper.py index 1f73123eec..78b73c8e5c 100644 --- a/bio/vep/cache/wrapper.py +++ b/bio/vep/cache/wrapper.py @@ -13,11 +13,12 @@ with tempfile.TemporaryDirectory() as tmpdir: # We download the cache tarball manually because vep_install does not consider proxy settings (in contrast to curl). # See https://github.com/bcbio/bcbio-nextgen/issues/1080 + vep_dir = "vep" if snakemake.params.release >= 97 else "VEP" cache_tarball = f"{snakemake.params.species}_vep_{snakemake.params.release}_{snakemake.params.build}.tar.gz" log = snakemake.log_fmt_shell(stdout=True, stderr=True) shell( "curl -L ftp://ftp.ensembl.org/pub/release-{snakemake.params.release}/" - "variation/VEP/{cache_tarball} " + "variation/{vep_dir}/{cache_tarball} " "-o {tmpdir}/{cache_tarball} {log}" ) From fa4e5edc62ad3b3e2b3e8e7be81521bbe50f2653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Mon, 30 Jan 2023 13:09:16 +0100 Subject: [PATCH 4/5] fix --- bio/vep/cache/wrapper.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/bio/vep/cache/wrapper.py b/bio/vep/cache/wrapper.py index 78b73c8e5c..bdde2dd968 100644 --- a/bio/vep/cache/wrapper.py +++ b/bio/vep/cache/wrapper.py @@ -10,11 +10,16 @@ extra = snakemake.params.get("extra", "") +try: + release = int(snakemake.params.release) +except ValueError: + raise ValueError("The parameter release is supposed to be an integer.") + with tempfile.TemporaryDirectory() as tmpdir: # We download the cache tarball manually because vep_install does not consider proxy settings (in contrast to curl). # See https://github.com/bcbio/bcbio-nextgen/issues/1080 - vep_dir = "vep" if snakemake.params.release >= 97 else "VEP" - cache_tarball = f"{snakemake.params.species}_vep_{snakemake.params.release}_{snakemake.params.build}.tar.gz" + vep_dir = "vep" if release >= 97 else "VEP" + cache_tarball = f"{snakemake.params.species}_vep_{release}_{snakemake.params.build}.tar.gz" log = snakemake.log_fmt_shell(stdout=True, stderr=True) shell( "curl -L ftp://ftp.ensembl.org/pub/release-{snakemake.params.release}/" @@ -27,7 +32,7 @@ "vep_install --AUTO cf " "--SPECIES {snakemake.params.species} " "--ASSEMBLY {snakemake.params.build} " - "--VERSION {snakemake.params.release} " + "--VERSION {release} " "--CACHEURL {tmpdir} " "--CACHEDIR {snakemake.output} " "--CONVERT " From f07eba59029c0984cbf364302e9d4299de709cae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Mon, 30 Jan 2023 14:26:23 +0100 Subject: [PATCH 5/5] fmt --- bio/vep/cache/wrapper.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bio/vep/cache/wrapper.py b/bio/vep/cache/wrapper.py index bdde2dd968..b5dafe5f9a 100644 --- a/bio/vep/cache/wrapper.py +++ b/bio/vep/cache/wrapper.py @@ -19,7 +19,9 @@ # We download the cache tarball manually because vep_install does not consider proxy settings (in contrast to curl). # See https://github.com/bcbio/bcbio-nextgen/issues/1080 vep_dir = "vep" if release >= 97 else "VEP" - cache_tarball = f"{snakemake.params.species}_vep_{release}_{snakemake.params.build}.tar.gz" + cache_tarball = ( + f"{snakemake.params.species}_vep_{release}_{snakemake.params.build}.tar.gz" + ) log = snakemake.log_fmt_shell(stdout=True, stderr=True) shell( "curl -L ftp://ftp.ensembl.org/pub/release-{snakemake.params.release}/"