feat!: Pin wrapper versions in meta-wrappers; add alignoth_report meta-wrapper

.github/workflows/autobump.yml

@@ -33,4 +33,5 @@ jobs:
         with:
           subcommand: update-conda-envs
           args: "*/${{ matrix.prefix }}*/environment.yaml */${{ matrix.prefix }}*/*/environment.yaml --create-prs --warn-on-error --entity-regex '(?P<entity>.+)/environment.yaml' --pr-add-label --pin-envs"
-
+
+      # TODO add snakedeploy update-wrappers command applied to all meta-wrappers once it has the --create-prs option

.github/workflows/qc.yml

@@ -63,11 +63,23 @@ jobs:
         run: |
           declare -i ERRORS=0
           # get all modified and added files, not those that are deleted
-          for f in $(git diff origin/master --name-only --diff-filter=d | grep Snakefile)
+          for f in $(git diff origin/master --name-only --diff-filter=d | grep -P "(test/Snakefile|meta_wrapper.smk)")
           do
-            echo "Linting $f"
-            (cd $(dirname $f) && snakemake --lint)
-            ERRORS+=$?
+            # lint meta_wrapper.smk and test/Snakefile of normal wrappers
+            if [[ $f == meta/**/meta_wrapper.smk ]]
+            then
+              echo "Linting $f"
+              # ignore errors in meta_wrapper.smk files for now
+              # TODO We have to rather lint test/Snakefile because it defines the right pathvars
+              # or we disable the pathvar checking when linting
+              (cd $(dirname $f) && (snakemake --lint --snakefile $(basename $f)) || true)
+              ERRORS+=$?
+            elif [[ $f != meta/**/test/Snakefile ]]
+            then
+              echo "Linting $f"
+              (cd $(dirname $f) && snakemake --lint --snakefile $(basename $f))
+              ERRORS+=$?
+            fi
           done
           if (( $ERRORS > 0 ))
           then

meta/bio/alignoth_report/meta.yaml

@@ -0,0 +1,5 @@
+name: alignoth_report
+description: Export variants via vembrane and create alignment plots with alignoth for a Datavzrd report.
+authors:
+  - Felix Wiegand
+  - Felix Mölder

meta/bio/alignoth_report/meta_wrapper.smk

@@ -0,0 +1,59 @@
+def get_alignoth_tables(wildcards, results_dir):
+    count = count_variants(wildcards)
+    return [f"{results_dir}/alignoth/{{sample}}/{i}/" for i in range(count)]
+
+
+def count_variants(wildcards):
+    return sum(1 for _ in open(checkpoints.vembrane_table.get(sample=wildcards.sample).output[0], "r")) - 1
+
+
+checkpoint vembrane_table:
+    input:
+        "<results>/{sample}.bcf",
+    output:
+        "<results>/tables/{sample}.tsv"
+    log:
+        "<logs>/vembrane_table/{sample}.log"
+    params:
+        expression="INDEX, CHROM, POS, REF, ALT",
+        extra=""
+    wrapper:
+        "v7.6.1/bio/vembrane/table"
+
+
+rule alignoth:
+    input:
+        bam="<results>/mapped/{sample}.bam",
+        bam_idx="<results>/mapped/{sample}.bam.bai",
+        reference="resources/genome.fa",
+        reference_idx="resources/genome.fa.fai",
+        vcf="<results>/{sample}.bcf",
+        vcf_idx="<results>/{sample}.bcf.csi",
+        overview="<results>/tables/{sample}.tsv"
+    output:
+        directory("<results>/alignoth/{sample}/{index}/")
+    params:
+        extra=lambda wc, input: f"--around-vcf-record {wc.index} -f tsv"
+    log:
+        "<logs>/alignoth/{sample}_{index}.log"
+    wrapper:
+        "v7.9.1/bio/alignoth"
+
+
+rule datavzrd:
+    input:
+        config=workflow.source_path("resources/template.datavzrd.yaml"),
+        overview="<results>/tables/{sample}.tsv",
+        plot_tables=lambda wc: get_alignoth_tables(wc, "<results>")
+    output:
+        report(
+            directory("<results>/datavzrd-report/{sample}"),
+            htmlindex="index.html",
+        ),
+        # config = "resources/datavzrd/{sample}.rendered_config.yaml"
+    params:
+        max_index=lambda wc: count_variants(wc)
+    log:
+        "<logs>/datavzrd/{sample}.log",
+    wrapper:
+        "v7.9.1/utils/datavzrd"

meta/bio/alignoth_report/resources/template.datavzrd.yaml

@@ -0,0 +1,56 @@
+__use_yte__: true
+
+default-view: variants
+
+datasets:
+  variants:
+    path: ?input.overview
+    separator: "\t"
+    links:
+        alignment plot:
+          column: INDEX
+          view: "{value}"
+  ?for i in range(params.max_index):
+    ?f"{i}.coverage":
+      path: ?f"{input.plot_tables[i]}/{wildcards.sample}.coverage.tsv"
+      separator: "\t"
+    ?f"{i}.reference":
+      path: ?f"{input.plot_tables[i]}/{wildcards.sample}.reference.tsv"
+      separator: "\t"
+    ?f"{i}.reads":
+      path: ?f"{input.plot_tables[i]}/{wildcards.sample}.reads.tsv"
+      separator: "\t"
+    ?f"{i}.highlight":
+      path: ?f"{input.plot_tables[i]}/{wildcards.sample}.highlight.tsv"
+      separator: "\t"
+views:
+  variants:
+    dataset: variants
+    render-table:
+      columns:
+        SAMPLE:
+          label: sample
+        CHROM:
+          label: chromosome
+        POS:
+          label: position
+        INDEX:
+          display-mode: hidden
+        REF:
+          label: reference allele
+          spell:
+            url: v1.5.0/med/alleles
+        ALT:
+          label: alternative allele
+          spell:
+            url: v1.5.0/med/alleles
+  ?for i in range(params.max_index):
+    ?i:
+        hidden: true
+        datasets:
+            reads: ?f"{i}.reads"
+            reference: ?f"{i}.reference"
+            coverage: ?f"{i}.coverage"
+            highlight: ?f"{i}.highlight"
+        render-plot:
+            spec-path: ?f"{input.plot_tables[i]}/{wildcards.sample}.vl.json"

meta/bio/alignoth_report/test/Snakefile

@@ -0,0 +1,8 @@
+from snakemake.utils import min_version
+
+min_version("9.13.1")
+
+module alignoth_report:
+    meta_wrapper: "file:.."
+
+use rule * from alignoth_report

meta/bio/alignoth_report/test/resources/genome.fa

@@ -0,0 +1,2 @@
+>1
+TGTGCACCCAGGCTAATTAATGTTCATGGGAACTAATTAGAGCTGACATGTACAGCCAGCCTTTTGCTTACTCTGCGGGATCTTGTTCTGTGTGATTTCTCTAGGGGGAGCCTGCTCACTTTGAGTATTTGATGTATCCATTGCACTCAGCACCCATCACCGGTCTAGCTACCTGCATCCGCAAACCCCTTATAGCCACCTGTTCTCTGGATCGATCCATCCGCCTTTGGAATTATGAAACAAAGTAAGGAATGAAAGGCTTGCCTACTTTATTATGCAAGACCCCAGTTGTGAATTTATGTGAATTATTTTAATTACAGTGTTCCAAACTTTAATTATTTAGAAGCTTTCAGTTGGAAAAGATTTTTAAAAAATCAAGTCAGTCAACCTACCTGTTTCTCACTCTTGCGTGGGCTAGAAAACTACTTTGTGGTTTATTTAATGCTTACCACATGCTAGACACTGGGGACATGAAGAGATGAAGACAACACCTCAGTCTCTCACCTTCTTACATAAACTCCTGGGTCCCTCCAGCTCTCCTATCCCCTTGTTTCCCACAGGAACTGCTCTTTGCTTCAGGGAGCTCTCCAGACCCTTTCTTCCTCATCTTTCTCCCAGGACATCACTATCTCAACCACCCCCAACTCCCTCCTGACTTTACCTTCCAGCACAGCTTCCCTGGGAAGCTCCAACCCTAGGTGGACCCCCACCCCCTACTGTTTTCTTTGTTCTTATGTACAACTTCTGGGGCTTCTGGGGAAAAAATACTTGGACAATACTCATGAATTTTGGAATATAAACAGAGCTATGTCTTCAGCATCGCTAGACCACACCTTGACCAGTCTCCCTATCTACCATGCTCCACGCTGGACATTATCCCCTAATAACCAAGATCTAGAAAAGAGAGGAACCAACAATTGCAAACAGTAGTGTGTCTCAAATACGATGATAACAATATATATCAGATACAGTAGATGACTGGAAGGATTTAGTGAGCTAAGCTTACAATTCAGTAATTGATCTTTTTCACAGCACCCTGGAACTATTTAAGGAATACCAAGAAGAGGCATATTCCATCAGCCTTCATCCATCTGGACACTTCATTGTAGTAGGGTTTGCTGACAAACTACGCCTCATGAATCTACTCATTGATGATATACGTTCTTTCAAAGAATACTCTGTTAGAGGATGCGGAGAGGTAAAAAAAAAACTGCTGAAGACAAAAGTCCAGTTTCTTAGAAAGCCACGGCTGAAATATCAGGAACTAACATGATCTGGGGAGGTGGGACCAAAATCTGTCTTGGAGAACTACAGTGGCTTAAAAAAAGGGGGAAGGAGGAAATCCAGTTATTTTCCCAGCAACCTTATTGAGTATATGGTTCCATCCCCATATTCTTATGTCCACCAGACTATTTCATTGAACCTAAAACCTTGTGAGATCCCCGCTATGATAAGTCCTGTCTGTAATTGTGGAATATTAACAAATATTTTAAAAGAAAAGTAATCTCAGAATCAGAGTCACAGCTAAGAAGGTGAGGTTGGAGACGGAGGAGCCAGGGAAAAATAAGCATCAATGGTGAGAGGTAATAAAGAGCGCATCAGCTTTCCCTGAGGTGTCTTGCAGAAACAAGGCAATGATTTCAGGAATTACTTCCCCGGCTCTGAAACATGCTTCTACAGGCAAAATGATTGCTTGTGGTCCCAACGGCTTAATGATTTGTGGGCCTGACAGTGTCAGCATCTGGCCACTATGAGTTACAGGAATAATGAAAGTGTCTGAAAATGTCAGCCACTCCTCAGTCTCATTTGCACCTTCCCCCCACTCCCACCCTCACACGTAGTTTCAGTCTGAACTCGTTGGGAAAAGCACCTCTTGACTGTAACCTCTAATTAGACTGCTCACTTCCTGCTTGCTTGCTCTCTTGCTGTCTTTCCCTATAGTGTTCCTTTAGCAATGGAGGTCACCTGTTTGCTGCAGTCAATGGAAATGTGATTCACGTTTACACCACCACGAGCCTAGAGAACATCTCAAGCCTGAAAGGACACACAGGGAAGGTAAGTGAGTGAACAGTCTCTGGGGAAACAAGGGGCACGGAGCCAAGTATGCCGCCAGCCAGTGGGATACAGGTGAACAAAAGAGAGATGGTTCCTTTCCTCATGGGTTCACTGTCTACTTGGGGAAAGAGACAATAAAACAAATGCACAGACAAATGTAGTTACTAATTGCAGTTCCACTGAAAGAAAAACATGGTGTTGTTGTGAGAGAAATAATGAAGGGGGCGGCATAATTTAGGTGGCTGTATTTGCAAAGGCTTGGTCCTATAAGAGTGAGTAGGAGTTAATCAGACACAACTGGTGGGGATAATCACTGCAGATAGGGATCCAAAGGCCAGAGATGGGAGACTCATAGG

meta/bio/alignoth_report/test/resources/genome.fa.fai

@@ -0,0 +1 @@
+1	2411	3	2411	2412

meta/bio/alignoth_report/used_wrappers.yaml

@@ -0,0 +1,4 @@
+wrappers:
+  - bio/vembrane/table
+  - bio/alignoth
+  - utils/datavzrd

meta/bio/bowtie2_sambamba/meta_wrapper.smk

@@ -17,7 +17,7 @@ rule bowtie2_build:
         extra="",
     threads: 8
     wrapper:
-        "master/bio/bowtie2/build"
+        "v3.11.0/bio/bowtie2/build"
 
 
 rule bowtie2_alignment:
@@ -43,7 +43,7 @@ rule bowtie2_alignment:
         ),
     threads: 8
     wrapper:
-        "master/bio/bowtie2/align"
+        "v7.6.0/bio/bowtie2/align"
 
 
 rule sambamba_sort:
@@ -57,7 +57,7 @@ rule sambamba_sort:
         "<logs>/sambamba-sort/{sample}.log",
     threads: 8
     wrapper:
-        "master/bio/sambamba/sort"
+        "v3.11.0/bio/sambamba/sort"
 
 
 rule sambamba_view:
@@ -74,7 +74,7 @@ rule sambamba_view:
         "logs/sambamba-view/{sample}.log",
     threads: 8
     wrapper:
-        "master/bio/sambamba/view"
+        "v6.1.0/bio/sambamba/view"
 
 
 rule sambamba_markdup:
@@ -88,7 +88,7 @@ rule sambamba_markdup:
         "<logs>/sambamba-markdup/{sample}.log",
     threads: 8
     wrapper:
-        "master/bio/sambamba/markdup"
+        "v6.1.0/bio/sambamba/markdup"
 
 
 rule sambamba_index:
@@ -102,4 +102,4 @@ rule sambamba_index:
         "<logs>/sambamba-index/{sample}.log",
     threads: 8
     wrapper:
-        "master/bio/sambamba/index"
+        "v6.1.0/bio/sambamba/index"

meta/bio/bowtie2_sambamba/test/Snakefile

@@ -5,7 +5,7 @@ min_version("9.13.1")
 configfile: "config.yaml"
 
 module bowtie2_sambamba:
-    meta_wrapper: "master/meta/bio/bowtie2_sambamba"
+    meta_wrapper: "file:.."
     config: config
 
 use rule * from bowtie2_sambamba

meta/bio/bwa_mapping/meta_wrapper.smk

@@ -6,7 +6,7 @@ rule bwa_index:
     log:
         "<logs>/bwa_index.log",
     wrapper:
-        "master/bio/bwa/index"
+        "v5.10.0/bio/bwa/index"
 
 
 rule bwa_mem:
@@ -24,7 +24,7 @@ rule bwa_mem:
         sort_extra=""             # Extra args for samtools/picard.
     threads: 8
     wrapper:
-        "master/bio/bwa/mem"
+        "v7.6.0/bio/bwa/mem"
 
 
 rule samtools_index:
@@ -37,4 +37,4 @@ rule samtools_index:
     params:
         "" # optional params string
     wrapper:
-        "master/bio/samtools/index"
+        "v7.3.0/bio/samtools/index"

meta/bio/bwa_mapping/test/Snakefile

@@ -5,7 +5,7 @@ min_version("9.13.1")
 configfile: "config.yaml"
 
 module bwa_mapping:
-    meta_wrapper: "master/meta/bio/bwa_mapping"
+    meta_wrapper: "file:.."
     config: config
 
 use rule * from bwa_mapping

meta/bio/calc_consensus_reads/meta_wrapper.smk

@@ -17,7 +17,7 @@ rule calc_consensus_reads:
     log:
         "<logs>/consensus/{sample}.log",
     wrapper:
-        "master/bio/rbt/collapse_reads_to_fragments-bam"
+        "v3.9.0/bio/rbt/collapse_reads_to_fragments-bam"
 
 
 rule bwa_index:
@@ -28,7 +28,7 @@ rule bwa_index:
     log:
         "<logs>/bwa_index.log",
     wrapper:
-        "master/bio/bwa/index"
+        "v5.10.0/bio/bwa/index"
 
 
 rule map_consensus_reads:
@@ -52,7 +52,7 @@ rule map_consensus_reads:
         "<logs>/bwa_mem/{sample}.{read_type}.consensus.log",
     threads: 8
     wrapper:
-        "master/bio/bwa/mem"
+        "v7.6.0/bio/bwa/mem"
 
 
 rule sort_skipped_reads:
@@ -68,7 +68,7 @@ rule sort_skipped_reads:
     # Samtools takes additional threads through its option -@
     threads: 8  # This value - 1 will be sent to -@.
     wrapper:
-        "master/bio/samtools/sort"
+        "v7.6.0/bio/samtools/sort"
 
 
 rule mark_duplicates_skipped:
@@ -88,7 +88,7 @@ rule mark_duplicates_skipped:
     resources:
         mem_mb=1024,
     wrapper:
-        "master/bio/picard/markduplicates"
+        "v7.6.0/bio/picard/markduplicates"
 
 
 rule merge_consensus_reads:
@@ -102,4 +102,4 @@ rule merge_consensus_reads:
         "<logs>/samtools_merge/{sample}.log",
     threads: 8
     wrapper:
-        "master/bio/samtools/merge"
+        "v7.6.0/bio/samtools/merge"

meta/bio/calc_consensus_reads/test/Snakefile

@@ -5,7 +5,7 @@ min_version("9.13.1")
 configfile: "config.yaml"
 
 module calc_consensus_reads:
-    meta_wrapper: "master/meta/bio/calc_consensus_reads"
+    meta_wrapper: "file:.."
     config: config
 
 use rule * from calc_consensus_reads