Merge branch 'nf-core:master' into master

Author: lianov

.github/workflows/main.yml

@@ -46,7 +46,6 @@ jobs:
           - "bi"
           - "bigpurple"
           - "bih"
-          - "binac"
           - "binac2"
           - "biohpc_gen"
           - "biowulf"
@@ -138,7 +137,6 @@ jobs:
           - "pawsey_nimbus"
           - "pawsey_setonix"
           - "pdc_kth"
-          - "pe2"
           - "phoenix"
           - "psmn"
           - "qmul_apocrita"

conf/bi.config

@@ -1,14 +1,17 @@
-params{
+// Set parameters to ignore for validation
+validation {
+    ignoreParams = ['bi_globalConfig']
+}
+
+params {
     config_profile_description = 'Boehringer Ingelheim internal profile provided by nf-core/configs.'
     config_profile_contact = 'Alexander Peltzer (@apeltzer)'
     config_profile_url = 'https://www.boehringer-ingelheim.com/'
 }
 
-params.globalConfig = System.getenv('NXF_GLOBAL_CONFIG')
-if(params.globalConfig == null)
-{
-    def errorMessage = "WARNING: For bi.config requires NXF_GLOBAL_CONFIG env var to be set. Point it to global.config file if you want to use this profile."
-    System.err.println(errorMessage)
-}else{
-    includeConfig params.globalConfig
+params.bi_globalConfig = System.getenv('NXF_GLOBAL_CONFIG')
+if (params.bi_globalConfig == null) {
+    System.err.println("WARNING: For bi.config requires NXF_GLOBAL_CONFIG env var to be set. Point it to global.config file if you want to use this profile.")
+} else {
+    includeConfig params.bi_globalConfig
 }

conf/bigpurple.config

@@ -1,11 +1,9 @@
-singularityDir = "/gpfs/scratch/${USER}/singularity_images_nextflow"
-
 params {
     config_profile_description = """
     NYU School of Medicine BigPurple cluster profile provided by nf-core/configs.
     module load both singularity/3.1 and squashfs-tools/4.3 before running the pipeline with this profile!!
     Run from your scratch or lab directory - Nextflow makes a lot of files!!
-    Also consider running the pipeline on a compute node (srun --pty /bin/bash -t=01:00:00) the first time, as it will be pulling the docker image, which will be converted into a singularity image, which is heavy on the login node and will take some time. Subsequent runs can be done on the login node, as the docker image will only be pulled and converted once. By default the images will be stored in ${singularityDir}
+    Also consider running the pipeline on a compute node (srun --pty /bin/bash -t=01:00:00) the first time, as it will be pulling the docker image, which will be converted into a singularity image, which is heavy on the login node and will take some time. Subsequent runs can be done on the login node, as the docker image will only be pulled and converted once. By default the images will be stored in /gpfs/scratch/${System.getenv("USER")}/singularity_images_nextflow
     """.stripIndent()
     config_profile_contact     = 'Tobias Schraink (@tobsecret)'
     config_profile_url         = 'https://github.com/nf-core/configs/blob/master/docs/bigpurple.md'
@@ -14,7 +12,7 @@ params {
 singularity {
     enabled    = true
     autoMounts = true
-    cacheDir   = singularityDir
+    cacheDir   = "/gpfs/scratch/${System.getenv("USER")}/singularity_images_nextflow"
 }
 
 process {

conf/binac.config

@@ -22,7 +22,7 @@ profiles {
             enabled      = true
             autoMounts   = true
             pullTimeout  = '120m'
-            cacheDir     = "/pfs/10/project/apptainer_cache/${USER}"
+            cacheDir     = "/pfs/10/project/apptainer_cache/${System.getenv('USER')}"
             envWhitelist = 'CUDA_VISIBLE_DEVICES'
         }
 

conf/binac2.config

@@ -7,35 +7,63 @@ params{
     max_time = 14.d
 }
 
+// disable this if you want to use conda environments instead
+// docker is not supported on the Cannon cluster
 singularity {
     enabled = true
     autoMounts = true
 }
 
-process {
-    executor = 'slurm'
+executor {
+    name = 'slurm'
     queueSize = 2000
     submitRateLimit = '10/sec'
+}
+
+process {
+    executor = 'slurm'
     resourceLimits = [
         memory: 2000.GB,
         cpus: 112,
         time: 14.d
     ]
+    // default time, cpu, and memory for processes that do not specify these resources
+    // These will be overridden by any process that specifies its own
+    // see Running Jobs https://docs.rc.fas.harvard.edu/kb/running-jobs/
+    time = 10.m // time is required or job won't run
+    cpus = 1
+    memory = 100.MB
+
     scratch = true
     queue = {
-        switch (true) {
-            case { task.memory >= 1000.GB && task.time >= 3.d }:
-                return 'bigmem_intermediate'
-            case { task.memory >= 1000.GB }:
-                return 'bigmem'
-            case { task.memory >= 184.GB && task.time >= 3.d }:
-                return 'intermediate'
-            case { task.memory >= 184.GB }:
-                return 'sapphire'
-            case { task.time >= 3.d }:
-                return 'intermediate'
-            default:
-                return 'shared'
+        // GPU handling
+        if (task.ext.gpu == true) {
+            if (task.ext.gpu_type == 'h200') {
+                return 'gpu_h200'
+            } else {
+                return 'gpu'
+            }
+        }
+
+        def mem = task.memory
+        def t   = task.time
+
+        if (t > 3.d) {
+            // Long jobs
+            if (mem > 990.GB) {
+                return 'bigmem_intermediate'  // limit 14d, 2000 GB
+            } else {
+                return 'intermediate'         // limit 14d, 990 GB
+            }
+        } else {
+            // ≤ 3d jobs
+            if (mem > 990.GB) {
+                return 'bigmem'               // limit 3d, 1988 GB
+            } else if (mem > 184.GB) {
+                return 'sapphire'             // limit 3d, 990 GB
+            } else {
+                return 'shared'               // largest pool for small jobs, limit 3d, 184 GB
+            }
         }
     }
 }

conf/cannon.config

@@ -1,11 +1,11 @@
 params {
     config_profile_description = 'USDA ARS SCINet Ceres Cluster profile'
-    config_profile_contact = 'Thomas A. Christensen II (@MillironX)'
-    config_profile_url = 'https://scinet.usda.gov/guide/ceres/'
+    config_profile_contact = 'SCINet VRSC (scinet_vrsc@usda.gov)'
+    config_profile_url = 'https://scinet.usda.gov/guides/resources/ceres'
 
     max_memory = 640.GB
     max_cpus = 36
-    max_time = 60.d
+    max_time = 21.d
 }
 
 singularity {
@@ -17,30 +17,10 @@ process {
     resourceLimits = [
         memory: 640.GB,
         cpus: 36,
-        time: 60.d
+        time: 21.d
     ]
     executor = 'slurm'
     scratch = true
-    queue = {
-        switch (task.memory) {
-            case { it >= 216.GB }:
-                switch (task.time) {
-                    case { it >= 7.d }:
-                        return 'longmem'
-                    default:
-                        return 'mem'
-                }
-            default:
-                switch (task.time) {
-                    case { it >= 21.d }:
-                        return 'long60'
-                    case { it >= 7.d }:
-                        return 'long'
-                    case { it >= 48.h }:
-                        return 'medium'
-                    default:
-                        return 'short'
-                }
-        }
-    }
+    queue = 'ceres'
+    clusterOptions = " -A ${System.getenv('SLURM_JOB_ACCOUNT')} "
 }

conf/ceres.config

@@ -9,20 +9,24 @@ params {
 }
 
 // define workDir in /nfs/scratch/nf-core_work/ named after the launch dir
-def work_dir = "/nfs/scratch/nf-core_work/"
-if(new File(work_dir).exists() && System.getenv("PWD")) {
-    work_dir = work_dir+System.getenv("PWD").tokenize('/').join('.')
-    workDir = work_dir
+workDir = {
+    def work_dir = "/nfs/scratch/nf-core_work/"
+    if(new File(work_dir).exists() && System.getenv("PWD")) {
+        work_dir = work_dir+System.getenv("PWD").tokenize('/').join('.')
 
-    // if directory does not exist, create it and set the group to the group launch dir
-    if(!new File(work_dir).exists()) {
-        "mkdir -p ${work_dir}".execute()
-        def pwd = System.getenv("PWD")
-        def group = "stat -c %g ${pwd}".execute().text.trim()
-        "chgrp -R ${group} ${work_dir}".execute()
-        "chmod -R g+s ${work_dir}".execute()
+        // if directory does not exist, create it and set the group to the group launch dir
+        if(!new File(work_dir).exists()) {
+            "mkdir -p ${work_dir}".execute()
+            def pwd = System.getenv("PWD")
+            def group = "stat -c %g ${pwd}".execute().text.trim()
+            "chgrp -R ${group} ${work_dir}".execute()
+            "chmod -R g+s ${work_dir}".execute()
+        }
+        return work_dir
+    } else {
+        return "work"
     }
-}
+}.call()
 
 process {
     resourceLimits = [
@@ -75,6 +79,15 @@ profiles {
         apptainer.cacheDir = '/nfs/scratch/apptainer_cache'
     }
 
+    // profile for gpu queue
+    gpu {
+        docker.runOptions       = '-u $(id -u):$(id -g) --gpus all'
+        apptainer.runOptions    = '--nv'
+        singularity.runOptions  = '--nv'
+        process.queue = 'shared-gpu'
+        process.clusterOptions = '--qos=limitgpus --gpus=a40:1 --exclude compms-gpu-1.exbio.wzw.tum.de'
+        executor.queueSize = 5
+    }
 }
 
 

conf/daisybio.config

@@ -7,6 +7,7 @@
 params {
     config_profile_description = "Novo Nordisk's Dirac cluster profile"
     config_profile_contact     = 'Ashot Margaryan (ashotmarg2004@gmail.com), Vincent Aaskov (vincentrose88@gmail.com)'
+    hpc_user                   = System.getenv('USER')
 }
 
 process {
@@ -18,7 +19,7 @@ process {
     executor     = 'slurm'
     queue        = 'compute'
     maxRetries   = 2
-    scratch      = "/scratch/${USER}"
+    scratch      = "/scratch/${params.hpc_user}"
 }
 
 executor {
@@ -30,7 +31,7 @@ apptainer {
     enabled     = true
     autoMounts  = true
     ociAutoPull = true
-    cacheDir    = "/scratch/${USER}"
+    cacheDir    = "/scratch/${params.hpc_user}"
     pullTimeout = '600 min'
     runOptions  = "--no-mount hostfs"
 }

conf/dirac.config

@@ -7,6 +7,7 @@
 params {
     config_profile_description = "Novo Nordisk's Einstein cluster profile"
     config_profile_contact     = 'Ashot Margaryan (ashotmarg2004@gmail.com)'
+    hpc_user                   = System.getenv('USER')
     max_memory                 = "750.GB"
     max_cpus                   = 64
     max_time                   = "240.h"
@@ -21,7 +22,7 @@ process {
     executor     = 'slurm'
     scratch      = true
     maxRetries   = 3
-    scratch      = "/scratch/users/${USER}"
+    scratch      = "/scratch/users/${params.hpc_user}"
 }
 
 executor {
@@ -32,6 +33,6 @@ executor {
 apptainer {
     enabled    = true
     autoMounts = true
-    cacheDir   = "/scratch/users/${USER}"
+    cacheDir   = "/scratch/users/${params.hpc_user}"
     pullTimeout = '600 min'
 }