Migrate file publishing from publishDir to centralized output {} block - I

.github/workflows/nf-test.yml

@@ -50,7 +50,7 @@ jobs:
         env:
           NFT_VER: ${{ env.NFT_VER }}
         with:
-          max_shards: 14
+          max_shards: 20
 
       - name: debug
         run: |
@@ -77,7 +77,7 @@ jobs:
           - isMain: false
             profile: "singularity"
         NXF_VER:
-          - "25.04.0"
+          - "25.10.4"
           - "latest-everything"
     env:
       NXF_ANSI_LOG: false

CHANGELOG.md

@@ -36,6 +36,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Removed haplocheck [#778](https://github.com/nf-core/raredisease/pull/778)
 - Removed HmtNote [#779](https://github.com/nf-core/raredisease/pull/779)
 - Updated svbd module [#781](https://github.com/nf-core/raredisease/pull/781)
+- Migrate file publishing from publishDir to a centralized output {} block for some workflows [#784](https://github.com/nf-core/raredisease/pull/784)
 - Replace local gens module with nf-core module [#785](https://github.com/nf-core/raredisease/pull/785)
 
 ### `Fixed`

README.md

@@ -11,7 +11,7 @@
 [![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com)
 [![GitHub Actions Linting Status](https://github.com/nf-core/raredisease/actions/workflows/linting.yml/badge.svg)](https://github.com/nf-core/raredisease/actions/workflows/linting.yml)[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/raredisease/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.7995798-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.7995798)
 
-[![Nextflow](https://img.shields.io/badge/version-%E2%89%A525.04.0-green?style=flat&logo=nextflow&logoColor=white&color=%230DC09D&link=https%3A%2F%2Fnextflow.io)](https://www.nextflow.io/)
+[![Nextflow](https://img.shields.io/badge/version-%E2%89%A525.10.4-green?style=flat&logo=nextflow&logoColor=white&color=%230DC09D&link=https%3A%2F%2Fnextflow.io)](https://www.nextflow.io/)
 [![nf-core template version](https://img.shields.io/badge/nf--core_template-3.5.1-green?style=flat&logo=nfcore&logoColor=white&color=%2324B064&link=https%3A%2F%2Fnf-co.re)](https://github.com/nf-core/tools/releases/tag/3.5.1)
 [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/)
 [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/)

conf/modules/align.config

@@ -19,20 +19,10 @@ process{
             '--correction',
             '--overrepresentation_analysis'
         ].join(' ').trim() }
-        publishDir = [
-            path: { "${params.outdir}/trimming" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
     }
 
     withName: '.*ALIGN:SAMTOOLS_VIEW' {
         ext.args   = { '--output-fmt cram' }
         ext.prefix = { "${meta.id}_sort_md" }
-        publishDir = [
-            path: { "${params.outdir}/alignment" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
     }
 }

conf/modules/align_bwa_bwamem2_bwameme.config

@@ -57,20 +57,5 @@ process {
     withName: '.*ALIGN:ALIGN_BWA_BWAMEM2_BWAMEME:MARKDUPLICATES' {
         ext.args = "--TMP_DIR ."
         ext.prefix = { "${meta.id}_sorted_md" }
-        publishDir = [
-            enabled: !params.save_mapped_as_cram,
-            path: { "${params.outdir}/alignment" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
-    }
-
-    withName: '.*ALIGN:ALIGN_BWA_BWAMEM2_BWAMEME:SAMTOOLS_INDEX_MARKDUP' {
-        publishDir = [
-            enabled: !params.save_mapped_as_cram,
-            path: { "${params.outdir}/alignment" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
     }
 }

conf/modules/align_sentieon.config

@@ -38,11 +38,5 @@ process {
     withName: '.*ALIGN:ALIGN_SENTIEON:SENTIEON_DEDUP' {
         ext.args4 = { params.rmdup ? "--rmdup" : '' }
         ext.prefix = { "${meta.id}_dedup.bam" }
-        publishDir = [
-            enabled: !params.save_mapped_as_cram,
-            path: { "${params.outdir}/alignment" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
     }
 }

conf/modules/call_mobile_elements.config

@@ -50,18 +50,5 @@ process {
     withName: '.*CALL_MOBILE_ELEMENTS:SVDB_MERGE_ME' {
         ext.args = { '--bnd_distance 150 --overlap 0.5' }
         ext.prefix = { "${meta.id}_mobile_elements" }
-        publishDir = [
-            path: { "${params.outdir}/call_mobile_elements" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
-    }
-
-    withName: '.*CALL_MOBILE_ELEMENTS:TABIX_ME' {
-        publishDir = [
-            path: { "${params.outdir}/call_mobile_elements" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
     }
 }

conf/modules/call_repeat_expansions.config

@@ -23,22 +23,12 @@ process {
 
     withName: '.*CALL_REPEAT_EXPANSIONS:SAMTOOLS_SORT' {
         ext.prefix = { "${meta.id}_exphunter_sorted" }
-        publishDir = [
-            path: { "${params.outdir}/repeat_expansions" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename },
-        ]
     }
 
     withName: '.*CALL_REPEAT_EXPANSIONS:BCFTOOLS_REHEADER_EXP' {
         ext.args   = "--temp-prefix ."
         ext.args2  = "-O v"
         ext.prefix = { "${meta.id}_repeat_expansion" }
-        publishDir = [
-            path: { "${params.outdir}/repeat_expansions" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename },
-        ]
     }
 
     withName: '.*CALL_REPEAT_EXPANSIONS:SPLIT_MULTIALLELICS_EXP' {

conf/modules/call_snv.config

@@ -16,21 +16,10 @@ process {
     withName: '.*CALL_SNV:GATK4_SELECTVARIANTS' {
         ext.args   = { "--exclude-intervals ${params.mito_name}" }
         ext.prefix = { "${meta.id}_snv" }
-        publishDir = [
-            path: { "${params.outdir}/call_snv/genome" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
     }
 
     withName: '.*CALL_SNV:BCFTOOLS_CONCAT' {
         ext.args   = { "--write-index=tbi" }
         ext.prefix = { "${meta.id}_mt_and_nuclear_snvs" }
-        publishDir = [
-            enabled: params.concatenate_snv_calls,
-            path: { "${params.outdir}/call_snv/concatenated_calls" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
     }
 }

conf/modules/call_structural_variants.config

@@ -20,18 +20,5 @@ process {
     withName: '.*CALL_STRUCTURAL_VARIANTS:SVDB_MERGE' {
         ext.prefix = {"${meta.id}_sv"}
         ext.args = '--pass_only --same_order'
-        publishDir = [
-            path: { "${params.outdir}/call_sv/genome" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
-    }
-
-    withName: '.*CALL_STRUCTURAL_VARIANTS:TABIX_TABIX' {
-        publishDir = [
-            path: { "${params.outdir}/call_sv/genome" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
     }
 }

conf/modules/call_sv_MT.config

@@ -20,19 +20,6 @@ process {
     withName: '.*CALL_SV_MT:MT_DELETION' {
         ext.args = '-s --insert-size 16000'
         ext.prefix = { "${meta.id}_mitochondria_deletions" }
-        publishDir = [
-            path: { "${params.outdir}/call_sv/mitochondria" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
-    }
-
-    withName: '.*CALL_SV_MT:MITOSALT' {
-        publishDir = [
-            path: { "${params.outdir}/call_sv/mitochondria" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
     }
 
 }

conf/modules/postprocess_MT_calls.config

@@ -47,19 +47,6 @@ process {
     withName: '.*POSTPROCESS_MT_CALLS:BCFTOOLS_ANNOTATE' {
         ext.args = "-c CHROM,FROM,TO,FOUND_IN --output-type z --include FILTER='\"PASS\"'"
         ext.prefix = { "${meta.id}_mitochondria" }
-        publishDir = [
-            path: { "${params.outdir}/call_snv/mitochondria" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
-    }
-
-    withName: '.*POSTPROCESS_MT_CALLS:TABIX_ANNOTATE' {
-        publishDir = [
-            path: { "${params.outdir}/call_snv/mitochondria" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
     }
 
 }

conf/modules/qc_bam.config

@@ -16,14 +16,6 @@
 //
 
 process {
-    withName: '.*QC_BAM:.*' {
-        publishDir = [
-            path: { "${params.outdir}/qc_bam" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename },
-        ]
-    }
-
     withName: '.*QC_BAM:PICARD_COLLECTMULTIPLEMETRICS' {
         ext.args = "--TMP_DIR ."
         ext.prefix = { "${meta.id}_multiplemetrics" }
@@ -61,11 +53,6 @@ process {
         //  NGSBITS_SAMPLEGENDER needs a chrX and chrY in order to run so we skip it for the two test profiles
         ext.when = { !(workflow.profile.tokenize(',').intersect(['test', 'test_singleton', 'test_sentieon', 'test_full']).size() >= 1) || workflow.stubRun }
         ext.prefix = { "${meta.id}_ngsbits_sex" }
-        publishDir = [
-            path: { "${params.outdir}/ngsbits_samplegender" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
     }
 
     withName: '.*QC_BAM:PICARD_COLLECTWGSMETRICS_WG' {

conf/modules/raredisease.config

@@ -65,11 +65,6 @@ process {
 process {
     withName: '.*STRANGER' {
         ext.prefix = { "${meta.id}_repeat_expansion_stranger" }
-        publishDir = [
-            path: { "${params.outdir}/repeat_expansions" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename },
-        ]
     }
 }
 

conf/modules/subsample_mt_frac.config

@@ -24,11 +24,6 @@ process {
     withName: '.*SUBSAMPLE_MT_FRAC:SAMTOOLS_VIEW' {
         ext.args = { "--output-fmt BAM -h -F 4 -s ${meta.seedfrac}"  }
         ext.prefix = { "${meta.id}_mt_subsample" }
-        publishDir = [
-            path: { "${params.outdir}/alignment" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
     }
 
 }

conf/modules/subsample_mt_reads.config

@@ -40,11 +40,6 @@ process {
 
     withName: '.*SUBSAMPLE_MT_READS:SAMTOOLS_SORT' {
         ext.prefix = { "${meta.id}_mt_subsample" }
-        publishDir = [
-            path: { "${params.outdir}/alignment" },
-            mode: params.publish_dir_mode,
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
     }
 
 }

main.nf

@@ -466,6 +466,7 @@ workflow NFCORE_RAREDISEASE {
     )
     emit:
     multiqc_report = RAREDISEASE.out.multiqc_report // channel: /path/to/multiqc_report.html
+    publish        = RAREDISEASE.out.publish          // channel: [ val(destination), val(value) ]
 }
 /*
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -586,6 +587,15 @@ workflow {
         params.hook_url,
         NFCORE_RAREDISEASE.out.multiqc_report
     )
+
+    publish:
+    subworkflow_results = NFCORE_RAREDISEASE.out.publish
+}
+
+output {
+    subworkflow_results {
+        path { destination, value -> destination }
+    }
 }
 
 /*

nextflow.config

@@ -175,6 +175,9 @@ params {
     validate_params            = true
 }
 
+outputDir            = params.outdir
+workflow.output.mode = 'copy'
+
 // Load base.config by default for all pipelines
 includeConfig 'conf/base.config'
 
@@ -462,7 +465,7 @@ manifest {
     description     = """call and score variants from WGS/WES of rare disease patients"""
     mainScript      = 'main.nf'
     defaultBranch   = 'master'
-    nextflowVersion = '!>=25.04.0'
+    nextflowVersion = '!>=25.10.4'
     version         = '2.7.0dev'
     doi             = '10.5281/zenodo.7995798'
 }