From cf61456a8c00f362d869cec50350a82c4d55a20f Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 18 Feb 2026 15:00:29 +0100 Subject: [PATCH] make pipeline configs strict syntax compatible --- conf/pipeline/deepmodeloptim/crg.config | 2 +- conf/pipeline/eager/eva.config | 141 +++++++++--------- conf/pipeline/eager/maestro.config | 90 +++++------ .../pipeline/methylseq/ku_sund_danhead.config | 2 +- .../pipeline/multiplesequencealign/crg.config | 2 +- conf/pipeline/proteinfold/nci_gadi.config | 8 +- conf/pipeline/sarek/cfc.config | 26 ++-- conf/pipeline/sarek/icr_davros.config | 4 - conf/pipeline/scflow/imperial.config | 8 +- 9 files changed, 143 insertions(+), 140 deletions(-) diff --git a/conf/pipeline/deepmodeloptim/crg.config b/conf/pipeline/deepmodeloptim/crg.config index cfa193dea..4efda6bff 100644 --- a/conf/pipeline/deepmodeloptim/crg.config +++ b/conf/pipeline/deepmodeloptim/crg.config @@ -15,7 +15,7 @@ profiles { singularity { enabled = true envWhitelist = "CUDA_VISIBLE_DEVICES,NVIDIA_VISIBLE_DEVICES" - if (params.use_gpu) { singularity.runOptions = '--nv' } + runOptions = params.get('use_gpu', false) ? '--nv' : '' pullTimeout = "60m" } } diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index d96b1306e..b9ca79489 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -1,5 +1,10 @@ // Profile config names for nf-core/configs +validation.ignoreParams = [ + 'eager_eva_check_max_func' +] + + params { // Specific nf-core/configs params config_profile_contact = 'James Fellows Yates (@jfy133)' @@ -21,44 +26,44 @@ process { clusterOptions = { "-S /bin/bash -V -j y -o output.log -l h_vmem=${task.memory.toGiga()}G" } withLabel: sc_tiny { - cpus = { check_max(1, 'cpus') } - memory = { check_max(1.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(1, 'cpus') } + memory = { params.eager_eva_check_max_func(1.GB * task.attempt, 'memory') } time = '365.d' } withLabel: sc_small { - cpus = { check_max(1, 'cpus') } - memory = { check_max(4.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(1, 'cpus') } + memory = { params.eager_eva_check_max_func(4.GB * task.attempt, 'memory') } time = '365.d' } withLabel: sc_medium { - cpus = { check_max(1, 'cpus') } - memory = { check_max(8.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(1, 'cpus') } + memory = { params.eager_eva_check_max_func(8.GB * task.attempt, 'memory') } time = '365.d' } withLabel: mc_small { - cpus = { check_max(2, 'cpus') } - memory = { check_max(4.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(2, 'cpus') } + memory = { params.eager_eva_check_max_func(4.GB * task.attempt, 'memory') } time = '365.d' } withLabel: mc_medium { - cpus = { check_max(4, 'cpus') } - memory = { check_max(8.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(4, 'cpus') } + memory = { params.eager_eva_check_max_func(8.GB * task.attempt, 'memory') } time = '365.d' } withLabel: mc_large { - cpus = { check_max(8, 'cpus') } - memory = { check_max(16.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(8, 'cpus') } + memory = { params.eager_eva_check_max_func(16.GB * task.attempt, 'memory') } time = '365.d' } withLabel: mc_huge { - cpus = { check_max(32, 'cpus') } - memory = { check_max(256.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(32, 'cpus') } + memory = { params.eager_eva_check_max_func(256.GB * task.attempt, 'memory') } time = '365.d' } @@ -115,7 +120,7 @@ process { withName: markduplicates { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" } - memory = { check_max(20.GB * task.attempt, 'memory') } + memory = { params.eager_eva_check_max_func(20.GB * task.attempt, 'memory') } errorStrategy = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' } } @@ -129,7 +134,7 @@ process { withName: additional_library_merge { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" } - memory = { check_max(4.GB * task.attempt, 'memory') } + memory = { params.eager_eva_check_max_func(4.GB * task.attempt, 'memory') } } withName: metagenomic_complexity_filter { @@ -138,8 +143,8 @@ process { withName: malt { clusterOptions = { "-S /bin/bash -V -l h_vmem=1000G" } - cpus = { check_max(32, 'cpus') } - memory = { check_max(955.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(32, 'cpus') } + memory = { params.eager_eva_check_max_func(955.GB * task.attempt, 'memory') } } withName: maltextract { @@ -169,7 +174,7 @@ process { withName: damageprofiler { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" } - memory = { check_max(8.GB * task.attempt, 'memory') } + memory = { params.eager_eva_check_max_func(8.GB * task.attempt, 'memory') } errorStrategy = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' } } @@ -237,38 +242,38 @@ profiles { clusterOptions = { "-S /bin/bash -V -j y -o output.log -l h_vmem=${task.memory.toGiga()}G" } withLabel: sc_tiny { - cpus = { check_max(1, 'cpus') } - memory = { check_max(1.5.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(1, 'cpus') } + memory = { params.eager_eva_check_max_func(1.5.GB * task.attempt, 'memory') } } withLabel: sc_small { - cpus = { check_max(1, 'cpus') } - memory = { check_max(6.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(1, 'cpus') } + memory = { params.eager_eva_check_max_func(6.GB * task.attempt, 'memory') } } withLabel: sc_medium { - cpus = { check_max(1, 'cpus') } - memory = { check_max(12.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(1, 'cpus') } + memory = { params.eager_eva_check_max_func(12.GB * task.attempt, 'memory') } } withLabel: mc_small { - cpus = { check_max(2, 'cpus') } - memory = { check_max(6.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(2, 'cpus') } + memory = { params.eager_eva_check_max_func(6.GB * task.attempt, 'memory') } } withLabel: mc_medium { - cpus = { check_max(4, 'cpus') } - memory = { check_max(12.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(4, 'cpus') } + memory = { params.eager_eva_check_max_func(12.GB * task.attempt, 'memory') } } withLabel: mc_large { - cpus = { check_max(8, 'cpus') } - memory = { check_max(24.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(8, 'cpus') } + memory = { params.eager_eva_check_max_func(24.GB * task.attempt, 'memory') } } withLabel: mc_huge { - cpus = { check_max(32, 'cpus') } - memory = { check_max(256.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(32, 'cpus') } + memory = { params.eager_eva_check_max_func(256.GB * task.attempt, 'memory') } } // Fixes for SGE and Java incompatibility due to (and also some samtools?!) using more memory than you tell it to use @@ -308,7 +313,7 @@ profiles { withName: markduplicates { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" } - memory = { check_max(32.GB * task.attempt, 'memory') } + memory = { params.eager_eva_check_max_func(32.GB * task.attempt, 'memory') } errorStrategy = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' } } @@ -322,7 +327,7 @@ profiles { withName: additional_library_merge { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" } - memory = { check_max(4.GB * task.attempt, 'memory') } + memory = { params.eager_eva_check_max_func(4.GB * task.attempt, 'memory') } } withName: metagenomic_complexity_filter { @@ -331,13 +336,13 @@ profiles { withName: malt { clusterOptions = { "-S /bin/bash -V -l h_vmem=1000G" } - cpus = { check_max(32, 'cpus') } - memory = { check_max(955.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(32, 'cpus') } + memory = { params.eager_eva_check_max_func(955.GB * task.attempt, 'memory') } errorStrategy = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' } } withName: hostremoval_input_fastq { - memory = { check_max(32.GB * task.attempt, 'memory') } + memory = { params.eager_eva_check_max_func(32.GB * task.attempt, 'memory') } } withName: maltextract { @@ -367,7 +372,7 @@ profiles { withName: damageprofiler { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" } - memory = { check_max(16.GB * task.attempt, 'memory') } + memory = { params.eager_eva_check_max_func(16.GB * task.attempt, 'memory') } errorStrategy = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' } } @@ -418,38 +423,38 @@ profiles { clusterOptions = { "-S /bin/bash -V -j y -o output.log -l h_vmem=${task.memory.toGiga()}G" } withLabel: sc_tiny { - cpus = { check_max(1, 'cpus') } - memory = { check_max(2.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(1, 'cpus') } + memory = { params.eager_eva_check_max_func(2.GB * task.attempt, 'memory') } } withLabel: sc_small { - cpus = { check_max(1, 'cpus') } - memory = { check_max(8.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(1, 'cpus') } + memory = { params.eager_eva_check_max_func(8.GB * task.attempt, 'memory') } } withLabel: sc_medium { - cpus = { check_max(1, 'cpus') } - memory = { check_max(16.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(1, 'cpus') } + memory = { params.eager_eva_check_max_func(16.GB * task.attempt, 'memory') } } withLabel: mc_small { - cpus = { check_max(2, 'cpus') } - memory = { check_max(8.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(2, 'cpus') } + memory = { params.eager_eva_check_max_func(8.GB * task.attempt, 'memory') } } withLabel: mc_medium { - cpus = { check_max(4, 'cpus') } - memory = { check_max(16.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(4, 'cpus') } + memory = { params.eager_eva_check_max_func(16.GB * task.attempt, 'memory') } } withLabel: mc_large { - cpus = { check_max(8, 'cpus') } - memory = { check_max(32.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(8, 'cpus') } + memory = { params.eager_eva_check_max_func(32.GB * task.attempt, 'memory') } } withLabel: mc_huge { - cpus = { check_max(32, 'cpus') } - memory = { check_max(512.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(32, 'cpus') } + memory = { params.eager_eva_check_max_func(512.GB * task.attempt, 'memory') } } // Fixes for SGE and Java incompatibility due to Java using more memory than you tell it to use @@ -489,13 +494,13 @@ profiles { withName: markduplicates { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" } - memory = { check_max(48.GB * task.attempt, 'memory') } + memory = { params.eager_eva_check_max_func(48.GB * task.attempt, 'memory') } errorStrategy = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' } } withName: library_merge { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" } - memory = { check_max(6.GB * task.attempt, 'memory') } + memory = { params.eager_eva_check_max_func(6.GB * task.attempt, 'memory') } } withName: seqtype_merge { @@ -504,11 +509,11 @@ profiles { withName: additional_library_merge { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" } - memory = { check_max(6.GB * task.attempt, 'memory') } + memory = { params.eager_eva_check_max_func(6.GB * task.attempt, 'memory') } } withName: hostremoval_input_fastq { - memory = { check_max(32.GB * task.attempt, 'memory') } + memory = { params.eager_eva_check_max_func(32.GB * task.attempt, 'memory') } } withName: metagenomic_complexity_filter { @@ -517,8 +522,8 @@ profiles { withName: malt { clusterOptions = { "-S /bin/bash -V -l h_vmem=1000G" } - cpus = { check_max(32, 'cpus') } - memory = { check_max(955.GB * task.attempt, 'memory') } + cpus = { params.eager_eva_check_max_func(32, 'cpus') } + memory = { params.eager_eva_check_max_func(955.GB * task.attempt, 'memory') } errorStrategy = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' } } @@ -549,7 +554,7 @@ profiles { withName: damageprofiler { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" } - memory = { check_max(32.GB * task.attempt, 'memory') } + memory = { params.eager_eva_check_max_func(32.GB * task.attempt, 'memory') } errorStrategy = { task.exitStatus in [1, 143, 137, 104, 134, 139, 140] ? 'retry' : 'finish' } } @@ -627,31 +632,31 @@ profiles { // Function to ensure that resource requirements don't go beyond // a maximum limit // FOR DSL1 PIPELINE ONLY! -def check_max(obj, type) { +params.eager_eva_check_max_func = { obj, type -> if (type == 'memory') { try { - if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1) - return params.max_memory as nextflow.util.MemoryUnit + if (obj.compareTo(params.max_memory as MemoryUnit) == 1) + return params.max_memory as MemoryUnit else return obj - } catch (all) { + } catch (_) { println " ### ERROR ### Max memory '${params.max_memory}' is not valid! Using default value: $obj" return obj } } else if (type == 'time') { try { - if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1) - return params.max_time as nextflow.util.Duration + if (obj.compareTo(params.max_time as Duration) == 1) + return params.max_time as Duration else return obj - } catch (all) { + } catch (_) { println " ### ERROR ### Max time '${params.max_time}' is not valid! Using default value: $obj" return obj } } else if (type == 'cpus') { try { return Math.min( obj, params.max_cpus as int ) - } catch (all) { + } catch (_) { println " ### ERROR ### Max cpus '${params.max_cpus}' is not valid! Using default value: $obj" return obj } diff --git a/conf/pipeline/eager/maestro.config b/conf/pipeline/eager/maestro.config index 01dc96d7c..fd14b4e3e 100644 --- a/conf/pipeline/eager/maestro.config +++ b/conf/pipeline/eager/maestro.config @@ -5,6 +5,12 @@ * nextflow run nf-core/eager -profile maestro,,maestro, (where is long or normal and is nuclear, mitocondrial or unlimitedtime) */ +validation { + ignoreParams = [ + 'eager_maestro_check_max_func' + ] +} + params { config_profile_name = 'nf-core/eager nuclear/mitocondrial - human profiles' @@ -21,41 +27,41 @@ profiles { maxRetries = 2 withName: makeBWAIndex { - cpus = { check_max(8 * task.attempt, 'cpus') } - memory = { check_max(8.GB * task.attempt, 'memory') } - time = { check_max(12.h * task.attempt, 'time') } + cpus = { params.eager_maestro_check_max_func(8 * task.attempt, 'cpus') } + memory = { params.eager_maestro_check_max_func(8.GB * task.attempt, 'memory') } + time = { params.eager_maestro_check_max_func(12.h * task.attempt, 'time') } } withName: adapter_removal { - cpus = { check_max(8 * task.attempt, 'cpus') } - memory = { check_max(16.GB * task.attempt, 'memory') } - time = { check_max(12.h * task.attempt, 'time') } + cpus = { params.eager_maestro_check_max_func(8 * task.attempt, 'cpus') } + memory = { params.eager_maestro_check_max_func(16.GB * task.attempt, 'memory') } + time = { params.eager_maestro_check_max_func(12.h * task.attempt, 'time') } } withName: bwa { - cpus = { check_max(40 * task.attempt, 'cpus') } - memory = { check_max(40.GB * task.attempt, 'memory') } + cpus = { params.eager_maestro_check_max_func(40 * task.attempt, 'cpus') } + memory = { params.eager_maestro_check_max_func(40.GB * task.attempt, 'memory') } time = 24.h cache = 'deep' } withName: markduplicates { errorStrategy = { task.exitStatus in [143, 137, 104, 134, 139] ? 'retry' : 'finish' } - cpus = { check_max(16 * task.attempt, 'cpus') } - memory = { check_max(16.GB * task.attempt, 'memory') } - time = { check_max(12.h * task.attempt, 'time') } + cpus = { params.eager_maestro_check_max_func(16 * task.attempt, 'cpus') } + memory = { params.eager_maestro_check_max_func(16.GB * task.attempt, 'memory') } + time = { params.eager_maestro_check_max_func(12.h * task.attempt, 'time') } } withName: damageprofiler { cpus = 1 - memory = { check_max(8.GB * task.attempt, 'memory') } - time = { check_max(6.h * task.attempt, 'time') } + memory = { params.eager_maestro_check_max_func(8.GB * task.attempt, 'memory') } + time = { params.eager_maestro_check_max_func(6.h * task.attempt, 'time') } } withName: fastp { cpus = 8 - memory = { check_max(8.GB * task.attempt, 'memory') } - time = { check_max(6.h * task.attempt, 'time') } + memory = { params.eager_maestro_check_max_func(8.GB * task.attempt, 'memory') } + time = { params.eager_maestro_check_max_func(6.h * task.attempt, 'time') } } withName: fastqc { cpus = 2 - memory = { check_max(8.GB * task.attempt, 'memory') } - time = { check_max(6.h * task.attempt, 'time') } + memory = { params.eager_maestro_check_max_func(8.GB * task.attempt, 'memory') } + time = { params.eager_maestro_check_max_func(6.h * task.attempt, 'time') } } } } @@ -66,40 +72,40 @@ profiles { maxRetries = 2 withName: makeBWAIndex { - cpus = { check_max(8 * task.attempt, 'cpus') } - memory = { check_max(8.GB * task.attempt, 'memory') } - time = { check_max(12.h * task.attempt, 'time') } + cpus = { params.eager_maestro_check_max_func(8 * task.attempt, 'cpus') } + memory = { params.eager_maestro_check_max_func(8.GB * task.attempt, 'memory') } + time = { params.eager_maestro_check_max_func(12.h * task.attempt, 'time') } } withName: adapter_removal { - cpus = { check_max(8 * task.attempt, 'cpus') } - memory = { check_max(16.GB * task.attempt, 'memory') } - time = { check_max(12.h * task.attempt, 'time') } + cpus = { params.eager_maestro_check_max_func(8 * task.attempt, 'cpus') } + memory = { params.eager_maestro_check_max_func(16.GB * task.attempt, 'memory') } + time = { params.eager_maestro_check_max_func(12.h * task.attempt, 'time') } } withName: bwa { - cpus = { check_max(5 * task.attempt, 'cpus') } - memory = { check_max(5.GB * task.attempt, 'memory') } + cpus = { params.eager_maestro_check_max_func(5 * task.attempt, 'cpus') } + memory = { params.eager_maestro_check_max_func(5.GB * task.attempt, 'memory') } time = 24.h } withName: markduplicates { errorStrategy = { task.exitStatus in [143, 137, 104, 134, 139] ? 'retry' : 'finish' } - cpus = { check_max(5 * task.attempt, 'cpus') } - memory = { check_max(5.GB * task.attempt, 'memory') } - time = { check_max(6.h * task.attempt, 'time') } + cpus = { params.eager_maestro_check_max_func(5 * task.attempt, 'cpus') } + memory = { params.eager_maestro_check_max_func(5.GB * task.attempt, 'memory') } + time = { params.eager_maestro_check_max_func(6.h * task.attempt, 'time') } } withName: damageprofiler { cpus = 1 - memory = { check_max(5.GB * task.attempt, 'memory') } - time = { check_max(3.h * task.attempt, 'time') } + memory = { params.eager_maestro_check_max_func(5.GB * task.attempt, 'memory') } + time = { params.eager_maestro_check_max_func(3.h * task.attempt, 'time') } } withName: fastp { cpus = 8 - memory = { check_max(5.GB * task.attempt, 'memory') } - time = { check_max(3.h * task.attempt, 'time') } + memory = { params.eager_maestro_check_max_func(5.GB * task.attempt, 'memory') } + time = { params.eager_maestro_check_max_func(3.h * task.attempt, 'time') } } withName: fastqc { cpus = 2 - memory = { check_max(8.GB * task.attempt, 'memory') } - time = { check_max(6.h * task.attempt, 'time') } + memory = { params.eager_maestro_check_max_func(8.GB * task.attempt, 'memory') } + time = { params.eager_maestro_check_max_func(6.h * task.attempt, 'time') } } } } @@ -117,31 +123,31 @@ profiles { // Function to ensure that resource requirements don't go beyond // a maximum limit // FOR DSL1 PIPELINE ONLY! -def check_max(obj, type) { +params.eager_maestro_check_max_func = { obj, type -> if (type == 'memory') { try { - if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1) - return params.max_memory as nextflow.util.MemoryUnit + if (obj.compareTo(params.max_memory as MemoryUnit) == 1) + return params.max_memory as MemoryUnit else return obj - } catch (all) { + } catch (_) { println " ### ERROR ### Max memory '${params.max_memory}' is not valid! Using default value: $obj" return obj } } else if (type == 'time') { try { - if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1) - return params.max_time as nextflow.util.Duration + if (obj.compareTo(params.max_time as Duration) == 1) + return params.max_time as Duration else return obj - } catch (all) { + } catch (_) { println " ### ERROR ### Max time '${params.max_time}' is not valid! Using default value: $obj" return obj } } else if (type == 'cpus') { try { return Math.min( obj, params.max_cpus as int ) - } catch (all) { + } catch (_) { println " ### ERROR ### Max cpus '${params.max_cpus}' is not valid! Using default value: $obj" return obj } diff --git a/conf/pipeline/methylseq/ku_sund_danhead.config b/conf/pipeline/methylseq/ku_sund_danhead.config index 872cab827..1e1ebd95e 100644 --- a/conf/pipeline/methylseq/ku_sund_danhead.config +++ b/conf/pipeline/methylseq/ku_sund_danhead.config @@ -11,6 +11,6 @@ process { stageInMode = 'copy' } withName: 'NFCORE_METHYLSEQ:METHYLSEQ:BISMARK:BISMARK_ALIGN' { - multicore = 1 + ext.args = '--multicore 1' } } diff --git a/conf/pipeline/multiplesequencealign/crg.config b/conf/pipeline/multiplesequencealign/crg.config index 791e56b2f..255e876bf 100644 --- a/conf/pipeline/multiplesequencealign/crg.config +++ b/conf/pipeline/multiplesequencealign/crg.config @@ -14,7 +14,7 @@ profiles { singularity { enabled = true envWhitelist = "CUDA_VISIBLE_DEVICES,NVIDIA_VISIBLE_DEVICES" - if (params.use_gpu) { singularity.runOptions = '--nv' } + runOptions = params.get('use_gpu', false) ? '--nv' : '' pullTimeout = "60m" } } diff --git a/conf/pipeline/proteinfold/nci_gadi.config b/conf/pipeline/proteinfold/nci_gadi.config index f07999c01..b82bb4620 100644 --- a/conf/pipeline/proteinfold/nci_gadi.config +++ b/conf/pipeline/proteinfold/nci_gadi.config @@ -12,8 +12,6 @@ profiles { // Define process resource limits process { executor = 'pbspro' - project = System.getenv("PROJECT") - storage = params.storage_account?.trim() ? params.storage_account : "scratch/${params.project}+gdata/${params.project}" module = 'singularity' cache = 'lenient' stageInMode = 'symlink' @@ -22,7 +20,6 @@ profiles { withName: 'RUN_ALPHAFOLD2|RUN_ALPHAFOLD2_PRED|RUN_ALPHAFOLD2_MSA' { queue = params.use_gpu ? 'gpuvolta' : 'normal' cpus = 48 - gpus = 4 time = '4h' memory = 380.GB } @@ -31,7 +28,6 @@ profiles { container = "nf-core/proteinfold_colabfold:1.1.1" queue = params.use_gpu ? 'gpuvolta' : 'normal' cpus = 48 - gpus = 4 time = '4h' memory = 380.GB } @@ -40,18 +36,16 @@ profiles { container = "nf-core/proteinfold_esmfold:1.1.1" queue = params.use_gpu ? 'gpuvolta' : 'normal' cpus = 48 - gpus = 4 time = '4h' memory = 380.GB } } // Write custom trace file with outputs required for SU calculation - def trace_timestamp = new java.util.Date().format('yyyy-MM-dd_HH-mm-ss') trace { enabled = true overwrite = false - file = "./gadi-nf-core-trace-${trace_timestamp}.txt" + file = "./gadi-nf-core-trace-${new java.util.Date().format('yyyy-MM-dd_HH-mm-ss')}.txt" fields = 'name,status,exit,duration,realtime,cpus,%cpu,memory,%mem,rss' } } diff --git a/conf/pipeline/sarek/cfc.config b/conf/pipeline/sarek/cfc.config index e43b81c2a..da6d0d997 100644 --- a/conf/pipeline/sarek/cfc.config +++ b/conf/pipeline/sarek/cfc.config @@ -1,5 +1,7 @@ // Profile config names for nf-core/configs +validation.ignoreParams = ['sarek_cfc_check_resource_func'] + params { // Specific nf-core/configs params config_profile_contact = 'Friederike Hanssen (@FriederikeHanssen)' @@ -9,34 +11,34 @@ params { // Specific nf-core/sarek process configuration process { withName:'StrelkaSingle|Strelka|StrelkaBP|MantaSingle|Manta' { - cpus = { check_resource( 20 * task.attempt) } - memory = { check_resource( 59.GB * task.attempt) } + cpus = { params.sarek_cfc_check_resource_func( 20 * task.attempt) } + memory = { params.sarek_cfc_check_resource_func( 59.GB * task.attempt) } } withName:'MSIsensor_scan|MSIsensor_msi' { - memory = { check_resource( 55.GB * task.attempt ) } + memory = { params.sarek_cfc_check_resource_func( 55.GB * task.attempt ) } } withName:BamQC { - memory = { check_resource( 372.GB * task.attempt) } + memory = { params.sarek_cfc_check_resource_func( 372.GB * task.attempt) } } withName:MapReads{ - cpus = { check_resource( 20 * task.attempt ) } - memory = { check_resource( 59.GB * task.attempt) } + cpus = { params.sarek_cfc_check_resource_func( 20 * task.attempt ) } + memory = { params.sarek_cfc_check_resource_func( 59.GB * task.attempt) } } } -def check_resource(obj) { +params.sarek_cfc_check_resource_func = { obj -> try { - if (obj.getClass() == nextflow.util.MemoryUnit && obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1) - return params.max_memory as nextflow.util.MemoryUnit - else if (obj.getClass() == nextflow.util.Duration && obj.compareTo(params.max_time as nextflow.util.Duration) == 1) - return params.max_time as nextflow.util.Duration + if (obj.getClass() == nextflow.util.MemoryUnit && obj.compareTo(params.max_memory as MemoryUnit) == 1) + return params.max_memory as MemoryUnit + else if (obj.getClass() == nextflow.util.Duration && obj.compareTo(params.max_time as Duration) == 1) + return params.max_time as Duration else if (obj.getClass() == java.lang.Integer) return Math.min(obj, params.max_cpus as int) else return obj - } catch (all) { + } catch (_) { println " ### ERROR ### Max params max_memory:'${params.max_memory}', max_time:'${params.max_time}' or max_cpus:'${params.max_cpus}' is not valid! Using default value: $obj" } } diff --git a/conf/pipeline/sarek/icr_davros.config b/conf/pipeline/sarek/icr_davros.config index c33486326..ec7e2772d 100644 --- a/conf/pipeline/sarek/icr_davros.config +++ b/conf/pipeline/sarek/icr_davros.config @@ -6,8 +6,4 @@ process { errorStrategy = { task.exitStatus in [104, 134, 137, 139, 141, 143, 255] ? 'retry' : 'finish' } maxRetries = 5 - withName: MapReads { - memory = { check_resource(12.GB) } - time = { check_resource(48.h * task.attempt) } - } } diff --git a/conf/pipeline/scflow/imperial.config b/conf/pipeline/scflow/imperial.config index e49f17608..39b514def 100644 --- a/conf/pipeline/scflow/imperial.config +++ b/conf/pipeline/scflow/imperial.config @@ -6,13 +6,13 @@ params { config_profile_contact = 'NA' // Analysis Resource Params - ctd_folder = "/rds/general/user/${USER}/projects/ukdrmultiomicsproject/live/Analyses/scFlowResources/refs/ctd" - ensembl_mappings = "/rds/general/user/${USER}/projects/ukdrmultiomicsproject/live/Analyses/scFlowResources/src/ensembl-ids/ensembl_mappings.tsv" + ctd_folder = "/rds/general/user/${System.getenv('USER')}/projects/ukdrmultiomicsproject/live/Analyses/scFlowResources/refs/ctd" + ensembl_mappings = "/rds/general/user/${System.getenv('USER')}/projects/ukdrmultiomicsproject/live/Analyses/scFlowResources/src/ensembl-ids/ensembl_mappings.tsv" } singularity { enabled = true autoMounts = true - cacheDir = "/rds/general/user/${USER}/projects/ukdrmultiomicsproject/live/.singularity-cache" - runOptions = "-B /rds/,/rdsgpfs/,/rds/general/user/${USER}/ephemeral/tmp/:/tmp,/rds/general/user/${USER}/ephemeral/tmp/:/var/tmp" + cacheDir = "/rds/general/user/${System.getenv('USER')}/projects/ukdrmultiomicsproject/live/.singularity-cache" + runOptions = "-B /rds/,/rdsgpfs/,/rds/general/user/${System.getenv('USER')}/ephemeral/tmp/:/tmp,/rds/general/user/${System.getenv('USER')}/ephemeral/tmp/:/var/tmp" }