feat: updated versions and fixes for samtools wrappers. (#431)

* Added proper tempdir and updated version

* Reformat

* Added log files and missing import

* Fixed tempfile

* Fix tmpfil

* Yet another tmpfile fix

* Changed wrapper name since it also changed in samtools

* Changed params name

* Removed extension

* chore: exclude deleted files from those that are linted

Co-authored-by: Johannes Köster <johannes.koester@tu-dortmund.de>

.github/workflows/qc.yml

@@ -46,7 +46,8 @@ jobs:
           export PATH="/usr/share/miniconda/bin:$PATH"
           source activate snakemake
           declare -i ERRORS=0
-          for f in $(git diff origin/master --name-only | grep Snakefile)
+          # get all modified and added files, not those that are deleted
+          for f in $(git diff origin/master --name-only --diff-filter=d | grep Snakefile)
           do 
             snakemake -s $f --lint
             ERRORS+=$?

bio/samtools/bam2fq/interleaved/environment.yaml → bio/samtools/fastq/interleaved/environment.yaml

@@ -2,4 +2,4 @@ channels:
   - conda-forge
   - bioconda
 dependencies:
-  - samtools ==1.10
+  - samtools =1.14

bio/samtools/bam2fq/interleaved/meta.yaml → bio/samtools/fastq/interleaved/meta.yaml

@@ -1,7 +1,10 @@
-name: samtools bam2fq interleaved
+name: samtools fastq interleaved
 description:
   Convert a bam file back to unaligned reads in a single fastq file with
   samtools. For paired end reads, this results in an unsorted interleaved file.
 authors:
   - David Laehnemann
   - Victoria Sack
+  - Filipe G. Vieira
+notes: |
+  * For more information see, http://www.htslib.org/doc/samtools-fasta.html

bio/samtools/fastq/interleaved/test/Snakefile

@@ -0,0 +1,12 @@
+rule samtools_fastq_interleaved:
+    input:
+        "mapped/{sample}.bam",
+    output:
+        "reads/{sample}.fq",
+    log:
+        "{sample}.interleaved.log",
+    params:
+        " ",
+    threads: 3
+    wrapper:
+        "master/bio/samtools/fastq/interleaved"

bio/samtools/bam2fq/interleaved/wrapper.py → bio/samtools/fastq/interleaved/wrapper.py

@@ -12,7 +12,7 @@
 prefix = os.path.splitext(snakemake.output[0])[0]
 
 shell(
-    "samtools bam2fq {snakemake.params} "
+    "samtools fastq {snakemake.params} "
     " -@ {snakemake.threads} "
     " {snakemake.input[0]}"
     " > {snakemake.output[0]} "

bio/samtools/bam2fq/separate/environment.yaml → bio/samtools/fastq/separate/environment.yaml

@@ -2,4 +2,4 @@ channels:
   - conda-forge
   - bioconda
 dependencies:
-  - samtools ==1.10
+  - samtools =1.14

bio/samtools/bam2fq/separate/meta.yaml → bio/samtools/fastq/separate/meta.yaml

@@ -1,11 +1,13 @@
-name: samtools bam2fq separate
+name: samtools fastq separate
 description:
   Convert a bam file with paired end reads back to unaligned reads in a two
-  separate fastq files with samtools. Reads that are not properly paired are
-  discarded (READ_OTHER and singleton reads in samtools bam2fq documentation),
+  separate fastq files with `samtools`. Reads that are not properly paired are
+  discarded (READ_OTHER and singleton reads in `samtools fastq` documentation),
   as are secondary (0x100) and supplementary reads (0x800).
 authors:
   - David Laehnemann
   - Victoria Sack
+  - Filipe G. Vieira
 notes: |
   * Samtools -@/--threads takes one integer as input. This is the number of additional threads and not raw threads.
+  * For more information see, http://www.htslib.org/doc/samtools-fasta.html

bio/samtools/fastq/separate/test/Snakefile

@@ -0,0 +1,15 @@
+rule samtools_fastq_separate:
+    input:
+        "mapped/{sample}.bam",
+    output:
+        "reads/{sample}.1.fq",
+        "reads/{sample}.2.fq",
+    log:
+        "{sample}.separate.log",
+    params:
+        sort="-m 4G",
+        fastq="-n",
+    # Remember, this is the number of samtools' additional threads. At least 2 threads have to be requested on cluster sumbission. This value - 2 will be sent to samtools sort -@ argument.
+    threads: 3
+    wrapper:
+        "master/bio/samtools/fastq/separate"

bio/samtools/bam2fq/separate/wrapper.py → bio/samtools/fastq/separate/wrapper.py

@@ -5,34 +5,36 @@
 
 
 import os
+import tempfile
 from snakemake.shell import shell
 
 params_sort = snakemake.params.get("sort", "")
-params_bam2fq = snakemake.params.get("bam2fq", "")
+params_fastq = snakemake.params.get("fastq", "")
 log = snakemake.log_fmt_shell(stdout=True, stderr=True)
 
 prefix = os.path.splitext(snakemake.output[0])[0]
 
 # Samtools takes additional threads through its option -@
 # One thread is used bu Samtools sort
-# One thread is used by Samtools bam2fq
+# One thread is used by Samtools fastq
 # So snakemake.threads has to take them into account
 # before allowing additional threads through samtools sort -@
 threads = "" if snakemake.threads <= 2 else " -@ {} ".format(snakemake.threads - 2)
 
-shell(
-    "(samtools sort -n "
-    " {threads} "
-    " -T {prefix} "
-    " {params_sort} "
-    " {snakemake.input[0]} | "
-    "samtools bam2fq "
-    " {params_bam2fq} "
-    " -1 {snakemake.output[0]} "
-    " -2 {snakemake.output[1]} "
-    " -0 /dev/null "
-    " -s /dev/null "
-    " -F 0x900 "
-    " - "
-    ") {log}"
-)
+with tempfile.NamedTemporaryFile() as tmpfile:
+    shell(
+        "(samtools sort -n "
+        " {threads} "
+        " -T {tmpfile.name} "
+        " {params_sort} "
+        " {snakemake.input[0]} | "
+        "samtools fastq "
+        " {params_fastq} "
+        " -1 {snakemake.output[0]} "
+        " -2 {snakemake.output[1]} "
+        " -0 /dev/null "
+        " -s /dev/null "
+        " -F 0x900 "
+        " - "
+        ") {log}"
+    )

test.py

@@ -2480,17 +2480,17 @@ def test_samtools_idxstats():
 
 
 @skip_if_not_modified
-def test_samtools_bam2fq_interleaved():
+def test_samtools_fastq_interleaved():
     run(
-        "bio/samtools/bam2fq/interleaved",
+        "bio/samtools/fastq/interleaved",
         ["snakemake", "--cores", "1", "reads/a.fq", "--use-conda", "-F"],
     )
 
 
 @skip_if_not_modified
-def test_samtools_bam2fq_separate():
+def test_samtools_fastq_separate():
     run(
-        "bio/samtools/bam2fq/separate",
+        "bio/samtools/fastq/separate",
         ["snakemake", "--cores", "1", "reads/a.1.fq", "--use-conda", "-F"],
     )