feat: added wrapper-utils, and tempdirs (#467)

* Added wrapper-utils, and tempdirs

* Added inference of index format

* Update wrapper utils

Co-authored-by: Johannes Köster <johannes.koester@uni-due.de>

bio/bcftools/call/environment.yaml

@@ -3,4 +3,5 @@ channels:
   - bioconda
   - nodefaults
 dependencies:
-  - bcftools =1.11
+  - bcftools =1.14
+  - snakemake-wrapper-utils =0.4

bio/bcftools/call/meta.yaml

@@ -1,5 +1,10 @@
 name: bcftools call
-description: Call variants with `bcftools call <https://samtools.github.io/bcftools/bcftools.html#call>`_.
+description: Call variants with `bcftools call`.
+url: http://www.htslib.org/doc/bcftools.html#call
 authors:
   - Johannes Köster
   - Michael Hall
+  - Filipe G. Vieira
+notes: |
+  * The `uncompressed_bcf` param allows to specify that a BCF output should be uncompressed (ignored otherwise).
+  * The `extra` param allows for additional program arguments (not `--threads`, `-o/--output`, or `-O/--output-type`).

bio/bcftools/call/test/Snakefile

@@ -4,8 +4,9 @@ rule bcftools_call:
     output:
         calls="{sample}.calls.bcf",
     params:
-        caller="-m", # valid options include -c/--consensus-caller or -m/--multiallelic-caller
-        options="--ploidy 1 --prior 0.001",
+        uncompressed_bcf=False,
+        caller="-m",  # valid options include -c/--consensus-caller or -m/--multiallelic-caller
+        extra="--ploidy 1 --prior 0.001",
     log:
         "logs/bcftools_call/{sample}.log",
     wrapper:

bio/bcftools/call/wrapper.py

@@ -5,7 +5,11 @@
 
 
 from snakemake.shell import shell
+from snakemake_wrapper_utils.bcftools import get_bcftools_opts
 
+
+bcftools_opts = get_bcftools_opts(snakemake, parse_ref=False, parse_memory=False)
+extra = snakemake.params.get("extra", "")
 log = snakemake.log_fmt_shell(stdout=True, stderr=True)
 
 
@@ -22,10 +26,12 @@ class CallerOptionError(Exception):
         "-c/--consensus-caller as caller option."
     )
 
-options = snakemake.params.get("options", "")
 
 shell(
-    "bcftools call {options} {caller_opt} --threads {snakemake.threads} "
-    "-o {snakemake.output.calls} {snakemake.input.pileup} "
-    "{log}"
+    "bcftools call"
+    " {bcftools_opts}"
+    " {caller_opt}"
+    " {extra}"
+    " {snakemake.input[0]}"
+    " {log}"
 )

bio/bcftools/concat/environment.yaml

@@ -3,5 +3,5 @@ channels:
   - bioconda
   - nodefaults
 dependencies:
-  - bcftools =1.12
-  - snakemake-wrapper-utils ==0.2.0
+  - bcftools =1.14
+  - snakemake-wrapper-utils =0.4

bio/bcftools/concat/meta.yaml

@@ -1,5 +1,6 @@
 name: bcftools concat
-description: Concatenate vcf/bcf files with bcftools. For more information see `BCFtools documentation <https://www.htslib.org/doc/bcftools.html#concat>`_.
+description: Concatenate vcf/bcf files with bcftools.
+url: http://www.htslib.org/doc/bcftools.html#concat
 authors:
   - Johannes Köster
   - Filipe G. Vieira
@@ -9,5 +10,4 @@ output:
   - Concatenated VCF/BCF file
 notes: |
   * The `uncompressed_bcf` param allows to specify that a BCF output should be uncompressed (ignored otherwise).
-  * The `extra` param alllows for additional program arguments (not `--threads, `-O/--output-type`, `-m/--max-mem`, or `-T/--temp-dir`).
-  * For more information see, https://samtools.github.io/bcftools/bcftools.html
+  * The `extra` param alllows for additional program arguments (not `--threads`, `-o/--output`, or `-O/--output-type`).

bio/bcftools/concat/wrapper.py

@@ -4,17 +4,13 @@
 __license__ = "MIT"
 
 
-from os import path
 from snakemake.shell import shell
 from snakemake_wrapper_utils.bcftools import get_bcftools_opts
 
 
-bcftools_opts = get_bcftools_opts(snakemake, parse_memory=False)
+bcftools_opts = get_bcftools_opts(snakemake, parse_ref=False, parse_memory=False)
+extra = snakemake.params.get("extra", "")
 log = snakemake.log_fmt_shell(stdout=True, stderr=True)
 
 
-shell(
-    "bcftools concat {snakemake.params.extra} {bcftools_opts} -o {snakemake.output[0]} "
-    "{snakemake.input.calls} "
-    "{log}"
-)
+shell("bcftools concat {bcftools_opts} {extra} {snakemake.input[0]} {log}")

bio/bcftools/filter/environment.yaml

@@ -3,5 +3,5 @@ channels:
   - bioconda
   - nodefaults
 dependencies:
-  - bcftools ==1.12
-  - snakemake-wrapper-utils ==0.2
+  - bcftools =1.14
+  - snakemake-wrapper-utils =0.4

bio/bcftools/filter/meta.yaml

@@ -1,14 +1,14 @@
 name: bcftools filter
 description: filter vcf/bcf file.
+url: http://www.htslib.org/doc/bcftools.html#filter
 authors:
   - Patrik Smeds
   - Nikos Tsardakas Renhuldt
+  - Filipe G. Vieira
 input:
   - VCF/BCF file
 output:
   - Filtered VCF/BCF file
 notes: |
   * The `uncompressed_bcf` param allows to specify that a BCF output should be uncompressed (ignored otherwise).
-  * The `bcftools_use_mem` param controls whether to pass the resources.mem_mb to bcftools
-  * The `extra` param allows for additional program arguments (not `--threads, `-O/--output-type`, `-m/--max-mem`, or `-T/--temp-dir`).
-  * For more information see, https://samtools.github.io/bcftools/bcftools.html
+  * The `extra` param allows for additional program arguments (not `--threads`, `-o/--output`, or `-O/--output-type`).

bio/bcftools/filter/wrapper.py

@@ -7,18 +7,22 @@
 from snakemake.shell import shell
 from snakemake_wrapper_utils.bcftools import get_bcftools_opts
 
-bcftools_opts = get_bcftools_opts(snakemake, parse_memory=False)
+
+bcftools_opts = get_bcftools_opts(snakemake, parse_ref=False, parse_memory=False)
+extra = snakemake.params.get("extra", "")
 log = snakemake.log_fmt_shell(stdout=False, stderr=True)
+filter = snakemake.params.get("filter", "")
+
 
 if len(snakemake.output) > 1:
     raise Exception("Only one output file expected, got: " + str(len(snakemake.output)))
 
-filter = snakemake.params.get("filter", "")
-extra = snakemake.params.get("extra", "")
 
 shell(
-    "bcftools filter {filter} {extra} {snakemake.input[0]} "
-    "{bcftools_opts} "
-    "-o {snakemake.output[0]} "
-    "{log}"
+    "bcftools filter"
+    " {bcftools_opts}"
+    " {filter}"
+    " {extra}"
+    " {snakemake.input[0]}"
+    " {log}"
 )

bio/bcftools/index/environment.yaml

@@ -3,4 +3,5 @@ channels:
   - bioconda
   - nodefaults
 dependencies:
-  - bcftools =1.11
+  - bcftools =1.14
+  - snakemake-wrapper-utils =0.4

bio/bcftools/index/meta.yaml

@@ -1,4 +1,8 @@
 name: bcftools index
-description: Index vcf/bcf file. For more information see `BCFtools documentation <https://www.htslib.org/doc/bcftools.html#index>`_.
+description: Index vcf/bcf file.
+url: http://www.htslib.org/doc/bcftools.html#index
 authors:
   - Jan Forster
+  - Filipe G. Vieira
+notes: |
+  * The `extra` param allows for additional program arguments (not `--threads`, `-o/--output`).

bio/bcftools/index/test/Snakefile

@@ -1,9 +1,11 @@
 rule bcftools_index:
     input:
-        "a.bcf"
+        "a.bcf",
     output:
-        "a.bcf.csi"
+        "a.bcf.csi",
+    log:
+        "index/a.log",
     params:
-        extra=""  # optional parameters for bcftools index
+        extra="",  # optional parameters for bcftools index
     wrapper:
         "master/bio/bcftools/index"

bio/bcftools/index/wrapper.py

@@ -5,10 +5,27 @@
 
 
 from snakemake.shell import shell
+from snakemake_wrapper_utils.bcftools import get_bcftools_opts
 
-log = snakemake.log_fmt_shell(stdout=True, stderr=True)
 
-## Extract arguments
+bcftools_opts = get_bcftools_opts(
+    snakemake, parse_ref=False, parse_output_format=False, parse_memory=False
+)
 extra = snakemake.params.get("extra", "")
+log = snakemake.log_fmt_shell(stdout=True, stderr=True)
+
+
+if "--tbi" in extra or "--csi" in extra:
+    raise ValueError(
+        "You have specified index format (`--tbi/--csi`) in `params.extra`; this is automatically infered from the first output file."
+    )
+
+if snakemake.output[0].endswith(".tbi"):
+    extra += " --tbi"
+elif snakemake.output[0].endswith(".csi"):
+    extra += " --csi"
+else:
+    raise ValueError("invalid index file format ('.tbi', '.csi').")
+
 
-shell("bcftools index {extra} {snakemake.input[0]} {log}")
+shell("bcftools index {bcftools_opts} {extra} {snakemake.input[0]} {log}")

bio/bcftools/merge/environment.yaml

@@ -3,4 +3,5 @@ channels:
   - bioconda
   - nodefaults
 dependencies:
-  - bcftools =1.11
+  - bcftools =1.14
+  - snakemake-wrapper-utils =0.4

bio/bcftools/merge/meta.yaml

@@ -1,4 +1,9 @@
 name: bcftools merge
-description: Merge vcf/bcf files with bcftools. For more information see `BCFtools documentation <https://www.htslib.org/doc/bcftools.html#merge>`_.
+description: Merge vcf/bcf files with bcftools.
+url: http://www.htslib.org/doc/bcftools.html#merge
 authors:
   - Patrik Smeds
+  - Filipe G. Vieira
+notes: |
+  * The `uncompressed_bcf` param allows to specify that a BCF output should be uncompressed (ignored otherwise).
+  * The `extra` param allows for additional program arguments (not `--threads`, `-o/--output`, or `-O/--output-type`).

bio/bcftools/merge/test/Snakefile

@@ -1,9 +1,12 @@
 rule bcftools_merge:
     input:
-        calls=["a.bcf", "b.bcf"]
+        calls=["a.bcf", "b.bcf"],
     output:
-        "all.bcf"
+        "all.bcf",
+    log:
+        "all.log"
     params:
-        ""  # optional parameters for bcftools concat (except -o)
+        uncompressed_bcf=False,
+        extra="",  # optional parameters for bcftools concat (except -o)
     wrapper:
         "master/bio/bcftools/merge"

bio/bcftools/merge/wrapper.py

@@ -5,11 +5,11 @@
 
 
 from snakemake.shell import shell
+from snakemake_wrapper_utils.bcftools import get_bcftools_opts
 
+
+bcftools_opts = get_bcftools_opts(snakemake, parse_ref=False, parse_memory=False)
+extra = snakemake.params.get("extra", "")
 log = snakemake.log_fmt_shell(stdout=True, stderr=True)
 
-shell(
-    "bcftools merge {snakemake.params} -o {snakemake.output[0]} "
-    "{snakemake.input.calls} "
-    "{log}"
-)
+shell("bcftools merge {bcftools_opts} {extra} {snakemake.input} {log}")

bio/bcftools/mpileup/environment.yaml

@@ -3,4 +3,5 @@ channels:
   - bioconda
   - nodefaults
 dependencies:
-  - bcftools =1.11
+  - bcftools =1.14
+  - snakemake-wrapper-utils =0.4

bio/bcftools/mpileup/meta.yaml

@@ -1,4 +1,9 @@
 name: bcftools mpileup
-description: Generate VCF or BCF containing genotype likelihoods for one or multiple alignment (BAM or CRAM) files with `bcftools mpileup <https://samtools.github.io/bcftools/bcftools.html#mpileup>`_.
+description: Generate VCF or BCF containing genotype likelihoods for one or multiple alignment (BAM or CRAM) files.
+url: http://www.htslib.org/doc/bcftools.html#mpileup
 authors:
   - Michael Hall
+  - Filipe G. Vieira
+notes: |
+  * The `uncompressed_bcf` param allows to specify that a BCF output should be uncompressed (ignored otherwise).
+  * The `extra` param allows for additional program arguments (not `--threads`, `-f/--fasta-ref`, `-o/--output`, or `-O/--output-type`).

bio/bcftools/mpileup/test/Snakefile

@@ -1,12 +1,13 @@
 rule bcftools_mpileup:
     input:
-        index="genome.fasta.fai",
-        ref="genome.fasta", # this can be left out if --no-reference is in options
         alignments="mapped/{sample}.bam",
+        ref="genome.fasta",  # this can be left out if --no-reference is in options
+        index="genome.fasta.fai",
     output:
         pileup="pileups/{sample}.pileup.bcf",
     params:
-        options="--max-depth 100 --min-BQ 15",
+        uncompressed_bcf=False,
+        extra="--max-depth 100 --min-BQ 15",
     log:
         "logs/bcftools_mpileup/{sample}.log",
     wrapper:

bio/bcftools/mpileup/wrapper.py

@@ -5,29 +5,18 @@
 
 
 from snakemake.shell import shell
+from snakemake_wrapper_utils.bcftools import get_bcftools_opts
 
+
+extra = snakemake.params.get("extra", "")
+bcftools_opts = get_bcftools_opts(
+    snakemake, parse_ref=("--no-reference" not in extra), parse_memory=False
+)
 log = snakemake.log_fmt_shell(stdout=True, stderr=True)
 
 
 class MissingReferenceError(Exception):
     pass
 
 
-options = snakemake.params.get("options", "")
-
-# determine if a fasta reference is provided or not and add to options
-if "--no-reference" not in options:
-    ref = snakemake.input.get("ref", "")
-    if not ref:
-        raise MissingReferenceError(
-            "The --no-reference option was not given, but no fasta reference was "
-            "provided."
-        )
-    options += " --fasta-ref {}".format(ref)
-
-shell(
-    "bcftools mpileup {options} --threads {snakemake.threads} "
-    "--output {snakemake.output.pileup} "
-    "{snakemake.input.alignments} "
-    "{log}"
-)
+shell("bcftools mpileup {bcftools_opts} {extra} {snakemake.input[0]} {log}")

bio/bcftools/norm/environment.yaml

@@ -3,5 +3,5 @@ channels:
   - bioconda
   - nodefaults
 dependencies:
-  - bcftools =1.11
-  - snakemake-wrapper-utils =0.2
+  - bcftools =1.14
+  - snakemake-wrapper-utils =0.4

bio/bcftools/norm/meta.yaml

@@ -1,5 +1,9 @@
 name: bcftools norm
-description: Left-align and normalize indels, check if REF alleles match the reference, split multiallelic sites into multiple rows; recover multiallelics from multiple rows. For more information see `BCFtools documentation <https://www.htslib.org/doc/bcftools.html#norm>`_.
+description: Left-align and normalize indels, check if REF alleles match the reference, split multiallelic sites into multiple rows; recover multiallelics from multiple rows.
+url: http://www.htslib.org/doc/bcftools.html#norm
 authors:
   - Dayne Filer
   - Filipe G. Vieira
+notes: |
+  * The `uncompressed_bcf` param allows to specify that a BCF output should be uncompressed (ignored otherwise).
+  * The `extra` param allows for additional program arguments (not `--threads`, `-f/--fasta-ref`, `-o/--output`, or `-O/--output-type`).

bio/bcftools/norm/test/Snakefile

@@ -6,6 +6,7 @@ rule norm_vcf:
     log:
         "{prefix}.norm.log",
     params:
+        #        uncompressed_bcf=False,
         extra="--rm-dup none",  # optional
     wrapper:
         "master/bio/bcftools/norm"

bio/bcftools/norm/wrapper.py

@@ -7,11 +7,9 @@
 from snakemake.shell import shell
 from snakemake_wrapper_utils.bcftools import get_bcftools_opts
 
-bcftools_opts = get_bcftools_opts(snakemake, parse_memory=False, parse_temp_dir=False)
+bcftools_opts = get_bcftools_opts(snakemake, parse_memory=False)
 extra = snakemake.params.get("extra", "")
 log = snakemake.log_fmt_shell(stdout=True, stderr=True)
 
 
-shell(
-    "bcftools norm {bcftools_opts} {extra} {snakemake.input[0]} -o {snakemake.output[0]} {log}"
-)
+shell("bcftools norm {bcftools_opts} {extra} {snakemake.input[0]} {log}")

bio/bcftools/reheader/environment.yaml

@@ -3,4 +3,5 @@ channels:
   - bioconda
   - nodefaults
 dependencies:
-  - bcftools =1.11
+  - bcftools =1.14
+  - snakemake-wrapper-utils =0.4