Merge pull request #819 from nf-core/release300

Sync dev with fixes in release 300 branch

Author: ramprasadn

CHANGELOG.md

@@ -28,9 +28,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add peddy --sites hg38 argument when running with GRCh38 [#799](https://github.com/nf-core/raredisease/pull/799)
 - Saltshaker for downstream processing of mitochondrial SV calls from MitoSAlt [#775](https://github.com/nf-core/raredisease/pull/775)
 - Env variable NXF_SINGULARITY_NEW_PID_NAMESPACE = false to accommodate hisat2 running with latest Nextflow and Singularity [#775](https://github.com/nf-core/raredisease/pull/775)
+- Parameter `exclude_alt` to filter alignments to alt/unplaced contigs after alignment using samtools view, retaining only primary chromosomes (GRCh37: 1-22,X,Y,MT / GRCh38: chr1-chr22,chrX,chrY,chrM). Note that enabling this will restrict variant calling to these chromosomes [#803](https://github.com/nf-core/raredisease/pull/803)]
+- Parameters `save_all_mapped_as_cram` and `save_noalt_mapped_as_cram` to replace `save_mapped_as_cram`, allowing independent control over publishing unfiltered and alt-filtered alignment files as CRAM [#807](https://github.com/nf-core/raredisease/pull/807)
 
 ### `Changed`
 
+- Sort parameters of `CALL_STRUCTURAL_VARIANTS` and `CALL_SV_MANTA` alphabetically [[#](https://github.com/nf-core/raredisease/pull/)]
 - Use distinct output filenames for bcfools (in call_mobile_elements subworkflow) and svdb (in call_sv_tiddit subworkflow) [#716](https://github.com/nf-core/raredisease/pull/716)
 - Use nf-core's most severe consequence & pli scripts instead of local ones [#732](https://github.com/nf-core/raredisease/pull/732)
 - Use nf-core's VCF_FILTER_BCFTOOLS_ENSEMBLVEP subworkflow to generate clinical set instead of a local subworkflow [#727](https://github.com/nf-core/raredisease/pull/727)
@@ -56,6 +59,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Run MitoSAlt.pl from bin rather than within container [#775](https://github.com/nf-core/raredisease/pull/775)
 - Include mitochonrdial SV calls in combined SV vcf, change call_sv output directory structure to remove mitochondria/ and genome/ [#775](https://github.com/nf-core/raredisease/pull/775)
 - Remove Qualimap and Haplogrep3 as they were made redundant by Picard and VerifyBamID2 [#801](https://github.com/nf-core/raredisease/pull/801)
+- Remove env variable NXF_SINGULARITY_NEW_PID_NAMESPACE from the config since this has to be set outside the subworkflow [#804](https://github.com/nf-core/raredisease/pull/804)
+- Run UPD_SITES, UPD_REGIONS, and CHROMOGRAPH for UPD only when analysis type is WGS [#806](https://github.com/nf-core/raredisease/pull/806)
+- Change saltshaker classification output from txt to html [#808](https://github.com/nf-core/raredisease/pull/808)
 
 ### `Fixed`
 
@@ -67,12 +73,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Parameters
 
-| Old parameter   | New parameter           |
-| --------------- | ----------------------- |
-|                 | sambamba_regions        |
-| bwa_as_fallback |                         |
-|                 | multiqc_samples         |
-|                 | homoplasmy_af_threshold |
+| Old parameter       | New parameter             |
+| ------------------- | ------------------------- |
+|                     | sambamba_regions          |
+| bwa_as_fallback     |                           |
+|                     | multiqc_samples           |
+|                     | homoplasmy_af_threshold   |
+|                     | exclude_alt               |
+| save_mapped_as_cram |                           |
+|                     | save_all_mapped_as_cram   |
+|                     | save_noalt_mapped_as_cram |
 
 ### Tool updates
 

conf/modules/align.config

@@ -21,8 +21,23 @@ process{
         ].join(' ').trim() }
     }
 
-    withName: '.*ALIGN:SAMTOOLS_VIEW' {
+    withName: '.*ALIGN:CONVERTTOCRAM_ALTFILTERED' {
         ext.args   = { '--output-fmt cram' }
-        ext.prefix = { "${meta.id}_sort_md" }
+        ext.prefix = { "${meta.id}_sorted_md_primary_contigs" }
+    }
+
+    withName: '.*ALIGN:CONVERTTOCRAM_UNFILTERED' {
+        ext.args   = { '--output-fmt cram' }
+        ext.prefix = { "${meta.id}_sorted_md" }
+    }
+
+    withName: '.*ALIGN:SAMTOOLS_VIEW_EXCLUDE_ALT' {
+        ext.args   = { '--fetch-pairs' }
+        ext.args2  = {
+            params.genome == 'GRCh38'
+                ? ((1..22).collect { n -> "chr${n}" } + ['chrX', 'chrY', 'chrM']).join(' ')
+                : ((1..22).collect { n -> "${n}" }   + ['X',    'Y',   'MT'   ]).join(' ')
+        }
+        ext.prefix = { "${meta.id}_sorted_md_primary_contigs" }
     }
 }

conf/modules/call_snv_deepvariant.config

@@ -19,6 +19,7 @@ process {
 
     withName: '.*CALL_SNV_DEEPVARIANT:DEEPVARIANT' {
         ext.args = { [
+            "--vcf_stats_report=true",
             "--model_type=${params.analysis_type.toUpperCase()}",
             meta.sex == 1 ? params.genome == 'GRCh37' ? '--haploid_contigs="X,Y"' : '--haploid_contigs="chrX,chrY"' : ''
         ].join(' ') }

conf/modules/generate_cytosure_files.config

@@ -42,7 +42,8 @@ process {
         ext.args = { [
             meta.sex == 1 ? '--sex male' : '--sex female',
             '--size 5000',
-            '--maxbnd 5000'
+            '--maxbnd 5000',
+            params.genome.equals("GRCh38") ? "--genome 38" : ""
             ].join(' ') }
         ext.prefix = { "${meta.custid}" ? "${meta.custid}" : "${meta.id}" }
     }

docs/output.md

@@ -60,7 +60,8 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d
     - [Mitochondrial analysis](#mitochondrial-analysis)
       - [Alignment and variant calling](#alignment-and-variant-calling)
         - [MT deletion script](#mt-deletion-script)
-        - [eKLIPse](#eklipse)
+        - [MitoSAlt](#mitosalt)
+        - [saltshaker](#saltshaker)
       - [Annotation](#annotation)
         - [vcfanno](#vcfanno-1)
         - [CADD](#cadd-1)
@@ -100,26 +101,28 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d
 
 ##### Picard's MarkDuplicates
 
-[Picard MarkDuplicates](https://broadinstitute.github.io/picard/command-line-overview.html#MarkDuplicates) is used for marking PCR duplicates that can occur during library amplification. This is essential as the presence of such duplicates results in false inflated coverages, which in turn can lead to overly-confident genotyping calls during variant calling. Only reads aligned by Bwa-mem2, bwameme and bwa are processed by this tool. By default, alignment files are published in bam format. If you would like to store cram files instead, set `--save_mapped_as_cram` to true.
+[Picard MarkDuplicates](https://broadinstitute.github.io/picard/command-line-overview.html#MarkDuplicates) is used for marking PCR duplicates that can occur during library amplification. This is essential as the presence of such duplicates results in false inflated coverages, which in turn can lead to overly-confident genotyping calls during variant calling. Only reads aligned by Bwa-mem2, bwameme and bwa are processed by this tool. By default, alignment files are published in bam format. To publish cram files instead, use `--save_all_mapped_as_cram` for the full (unfiltered) alignment, or `--save_noalt_mapped_as_cram` for the alt-filtered alignment (requires `--exclude_alt`).
 
 <details markdown="1">
 <summary>Output files from Alignment</summary>
 
 - `{outputdir}/alignment/`
-  - `*.bam|*.cram`: Alignment file in bam/cram format.
+  - `*_sorted_md.bam|*_sorted_md.cram`: Full (unfiltered) alignment file. Published as bam by default, or as cram when `--save_all_mapped_as_cram` is set.
+  - `*_sorted_md_primary_contigs.cram`: Alt-filtered alignment file in cram format. Published when `--save_noalt_mapped_as_cram` is set (requires `--exclude_alt`). Contains only primary chromosomes (GRCh37: 1-22,X,Y,MT / GRCh38: chr1-chr22,chrX,chrY,chrM).
   - `*.bai|*.crai`: Index of the corresponding bam/cram file.
   - `*.txt`: Text file containing the dedup metrics.
   </details>
 
 ##### Sentieon Dedup
 
-[Sentieon Dedup](https://support.sentieon.com/manual/DNAseq_usage/dnaseq/#remove-or-mark-duplicates) is the algorithm used by Sentieon's driver to remove duplicate reads. Only reads aligned by Sentieon's implementation of bwa are processed by this algorithm. By default, alignment files are published in bam format. If you would like to store cram files instead, set `--save_mapped_as_cram` to true.
+[Sentieon Dedup](https://support.sentieon.com/manual/DNAseq_usage/dnaseq/#remove-or-mark-duplicates) is the algorithm used by Sentieon's driver to remove duplicate reads. Only reads aligned by Sentieon's implementation of bwa are processed by this algorithm. By default, alignment files are published in bam format. To publish cram files instead, use `--save_all_mapped_as_cram` for the full (unfiltered) alignment, or `--save_noalt_mapped_as_cram` for the alt-filtered alignment (requires `--exclude_alt`).
 
 <details markdown="1">
 <summary>Output files from Alignment</summary>
 
 - `{outputdir}/alignment/`
-  - `*.bam|*.cram`: Alignment file in bam/cram format.
+  - `*_sorted_md.bam|*_sorted_md.cram`: Full (unfiltered) alignment file. Published as bam by default, or as cram when `--save_all_mapped_as_cram` is set.
+  - `*_sorted_md_primary_contigs.cram`: Alt-filtered alignment file in cram format. Published when `--save_noalt_mapped_as_cram` is set (requires `--exclude_alt`). Contains only primary chromosomes (GRCh37: 1-22,X,Y,MT / GRCh38: chr1-chr22,chrX,chrY,chrM).
   - `*.bai|*.crai`: Index of the corresponding bam/cram file.
   - `*.metrics`: Text file containing the dedup metrics.
   </details>
@@ -467,7 +470,7 @@ The pipeline for mitochondrial variant discovery, using Mutect2, uses a high sen
 [Saltshaker](https://github.com/aksenia/saltshaker) allows for downstream clustering and classification of mtDNA strucutral variants. Called variants are combined with structural variants called in the nuclear genome.
 
 - `call_sv`
-  - `<sample_id>.saltshaker_classify.txt`: report containing case-level classification of mitochondrial deletions.
+  - `<sample_id>.saltshaker_classify.html`: report containing case-level classification of mitochondrial deletions.
   - `<sample_id>.saltshaker.png`: circos plot.
 
 #### Annotation

docs/usage.md

@@ -210,6 +210,7 @@ The mandatory and optional parameters for each category are tabulated below.
 |                                | min_trimmed_length<sup>6</sup>  |
 |                                | extract_alignments              |
 |                                | restrict_to_contigs<sup>7</sup> |
+|                                | exclude_alt<sup>8</sup>         |
 
 <sup>1</sup>Default value is bwamem2. Other alternatives are bwa, bwameme and sentieon (requires valid Sentieon license ).<br />
 <sup>2</sup>Analysis set reference genome in fasta format, first 25 contigs need to be chromosome 1-22, X, Y and the mitochondria.<br />
@@ -218,6 +219,7 @@ The mandatory and optional parameters for each category are tabulated below.
 <sup>5</sup>Used only by Sentieon.<br />
 <sup>6</sup>Default value is 40. Used only by fastp.<br />
 <sup>7</sup>Used to limit your analysis to specific contigs. Can be used to remove alignments to unplaced contigs to minimize potential errors. This parameter should be used in conjunction with the `extract_alignments` parameter.<br />
+<sup>8</sup>When set to true, alignments to alt/unplaced contigs are removed after alignment using samtools view, retaining only primary chromosomes (GRCh37: 1-22,X,Y,MT / GRCh38: chr1-chr22,chrX,chrY,chrM). Note that this will affect all downstream variant calling, as variants will only be called on these primary chromosomes.<br />
 
 ##### 2. QC stats from the alignment files
 

main.nf

@@ -51,6 +51,7 @@ workflow NFCORE_RAREDISEASE {
     val_cadd_resources
     val_call_interval
     val_concatenate_snv_calls
+    val_exclude_alt
     val_extract_alignments
     val_fai
     val_fasta
@@ -109,7 +110,8 @@ workflow NFCORE_RAREDISEASE {
     val_sambamba_regions
     val_sample_id_map
     val_samtools_sort_threads
-    val_save_mapped_as_cram
+    val_save_all_mapped_as_cram
+    val_save_noalt_mapped_as_cram
     val_save_reference
     val_score_config_mt
     val_score_config_snv
@@ -250,7 +252,7 @@ workflow NFCORE_RAREDISEASE {
     // Using channelFromSamplesheet helper. Returns either an empty channel or validated channel.
     ch_me_references            = channelFromSamplesheet(val_mobile_element_references, "${projectDir}/assets/mobile_element_references_schema.json", false)
     ch_me_svdb_resources        = channelFromSamplesheet(val_mobile_element_svdb_annotations, "${projectDir}/assets/svdb_query_vcf_schema.json", false)
-    ch_sample_id_map            = channelFromSamplesheet(val_sample_id_map, "${projectDir}/assets/sample_id_map.json")
+    ch_sample_id_map            = channelFromSamplesheet(val_sample_id_map, "${projectDir}/assets/sample_id_map.json", false)
     ch_svdb_bedpedbs            = channelFromSamplesheet(val_svdb_query_bedpedbs, "${projectDir}/assets/svdb_query_bedpe_schema.json", false)
     ch_svdb_dbs                 = channelFromSamplesheet(val_svdb_query_dbs, "${projectDir}/assets/svdb_query_vcf_schema.json", false)
 
@@ -330,6 +332,13 @@ workflow NFCORE_RAREDISEASE {
     skip_sv_calling            = parseSkipList(val_skip_subworkflows, 'sv_calling')
     skip_generate_clinical_set = parseSkipList(val_skip_subworkflows, 'generate_clinical_set')
 
+    //
+    // Validate parameter combinations
+    //
+    if (val_save_noalt_mapped_as_cram && !val_exclude_alt) {
+        error("save_noalt_mapped_as_cram requires exclude_alt to be set to true")
+    }
+
     //
     // SV caller priority
     //
@@ -469,6 +478,7 @@ workflow NFCORE_RAREDISEASE {
         val_analysis_type,
         val_cadd_resources,
         val_concatenate_snv_calls,
+        val_exclude_alt,
         val_extract_alignments,
         val_genome,
         val_heavy_strand_origin_end,
@@ -502,7 +512,8 @@ workflow NFCORE_RAREDISEASE {
         val_run_rtgvcfeval,
         val_sample_id_map,
         val_samtools_sort_threads,
-        val_save_mapped_as_cram,
+        val_save_all_mapped_as_cram,
+        val_save_noalt_mapped_as_cram,
         val_svdb_query_bedpedbs,
         val_svdb_query_dbs,
         val_target_bed,
@@ -554,6 +565,7 @@ workflow {
         params.cadd_resources,
         params.call_interval,
         params.concatenate_snv_calls,
+        params.exclude_alt,
         params.extract_alignments,
         params.fai,
         params.fasta,
@@ -612,7 +624,8 @@ workflow {
         params.sambamba_regions,
         params.sample_id_map,
         params.samtools_sort_threads,
-        params.save_mapped_as_cram,
+        params.save_all_mapped_as_cram,
+        params.save_noalt_mapped_as_cram,
         params.save_reference,
         params.score_config_mt,
         params.score_config_snv,
@@ -663,7 +676,7 @@ workflow {
 
 output {
     subworkflow_results {
-        path { destination, value -> destination }
+        path { destination, _value -> destination }
     }
 }
 

modules/local/saltshaker_to_html/main.nf

@@ -0,0 +1,28 @@
+process SALTSHAKER_TO_HTML {
+    tag "$meta.id"
+    label "process_low"
+
+    input:
+    tuple val(meta), path(classify)
+
+    output:
+    tuple val(meta), path("*.html"), emit: classify_html
+
+    script:
+    """
+    python3 << 'EOF'
+    import re
+    def saltshaker_txt_to_html(txt_file):
+        with open(txt_file) as f:
+            content = f.read()
+        html_content = re.sub(r'\\n', '<br>', content)
+        return html_content
+
+    html = saltshaker_txt_to_html("${classify}")
+    with open("${classify.baseName}.html", 'w') as f:
+        f.write('<html><body>')
+        f.write(f'<pre style="padding: 15px; border-radius: 5px; overflow-x: auto;">{html}</pre>')
+        f.write('</body></html>')
+    EOF
+    """
+}

nextflow.config

@@ -29,13 +29,15 @@ params {
     analysis_type              = 'wgs'
     bait_padding               = 100
     concatenate_snv_calls      = false
+    exclude_alt                = false
     extract_alignments         = false
     mt_subsample_approach      = 'reads'
     mt_subsample_reads         = 18000
     restrict_to_contigs        = null
     run_mt_for_wes             = false
     run_rtgvcfeval             = false
-    save_mapped_as_cram        = false
+    save_all_mapped_as_cram    = false
+    save_noalt_mapped_as_cram  = false
     scatter_count              = 20
     skip_tools                 = null
     skip_subworkflows          = null
@@ -58,6 +60,7 @@ params {
     gnomad_af                       = null
     gnomad_af_idx                   = null
     hisat2                          = null
+    hisat2_build_memory             = null
     intervals_wgs                   = null
     intervals_y                     = null
     known_dbsnp                     = null
@@ -121,7 +124,7 @@ params {
     vep_cache_version          = 112
 
     // sentieon Defaults
-    ml_model                   = ''
+    ml_model                   = null
 
     // Dnascope SNV calling
     sentieon_dnascope_pcr_indel_model = 'CONSERVATIVE'
@@ -344,7 +347,6 @@ env {
     R_PROFILE_USER   = "/.Rprofile"
     R_ENVIRON_USER   = "/.Renviron"
     JULIA_DEPOT_PATH = "/usr/local/share/julia"
-    NXF_SINGULARITY_NEW_PID_NAMESPACE = false
 }
 
 // Set bash options

nextflow_schema.json

@@ -536,6 +536,11 @@
                     "description": "Specifies whether to generate a concatenated VCF file containing both nuclear & mitochondrial snv calls",
                     "fa_icon": "fas fa-toggle-on"
                 },
+                "exclude_alt": {
+                    "type": "boolean",
+                    "description": "After aligning the reads to a reference, remove alignments to alt contigs using samtools view, retaining only primary chromosomes (GRCh37: 1-22,X,Y,MT / GRCh38: chr1-chr22,chrX,chrY,chrM).",
+                    "fa_icon": "fas fa-toggle-on"
+                },
                 "extract_alignments": {
                     "type": "boolean",
                     "description": "After aligning the reads to a reference, extract alignments from specific regions/contigs and restrict the analysis to those regions/contigs.",
@@ -576,9 +581,14 @@
                     "description": "Specifies whether to run rtgtools' vcfeval",
                     "fa_icon": "fas fa-toggle-on"
                 },
-                "save_mapped_as_cram": {
+                "save_all_mapped_as_cram": {
+                    "type": "boolean",
+                    "description": "Specifies whether to generate and publish all (unfiltered) alignment files as cram instead of bam",
+                    "fa_icon": "fas fa-toggle-on"
+                },
+                "save_noalt_mapped_as_cram": {
                     "type": "boolean",
-                    "description": "Specifies whether to generate and publish alignment files as cram instead of bam",
+                    "description": "Specifies whether to generate and publish alt-filtered alignment files as cram instead of bam. Requires exclude_alt to be set to true.",
                     "fa_icon": "fas fa-toggle-on"
                 },
                 "scatter_count": {
@@ -756,6 +766,11 @@
                 "hisat2": {
                     "type": "string"
                 },
+                "hisat2_build_memory": {
+                    "type": "string",
+                    "description": "Minimum memory required to build HISAT2 index with splice sites and exons. If available memory is below this threshold, a simpler index is built without splice sites.",
+                    "fa_icon": "fas fa-memory"
+                },
                 "mitosalt_breakspan": {
                     "type": "integer",
                     "default": 15,