setting option for nextflow automatically handle parallel jobs

Author: fmalmeida

configuration_template/bacannot.config

@@ -76,9 +76,10 @@ params {
 
 // Number of jobs to run in parallel. Be aware that each job (in parallel) can consume
 // N threads (set above). Be sure to carefully check your resources before augmenting
-// this parameter. For example: parallel_jobs = 2 + threads = 5 can consume up until 10
+// this parameter. For example: parallel_jobs = 2 + threads = 5 can consume until 10
 // threads at once.
-  parallel_jobs = 1
+// If not given, let's nextflow automatically handle it.
+      parallel_jobs = 
 
 // Number of minimum overlapping base pairs required for merging
 // Negative values, such as -20, means the number of required overlapping bases for merging.
@@ -212,11 +213,23 @@ report {
 profiles {
   standard {
     // Executor
-    process.executor = 'local'
+    process.executor = "local"
     // QueueSize limit
-    qs = (params.parallel_jobs) ? params.parallel_jobs : 1
+    if (params.parallel_jobs || params.parallel_jobs != '') {
+      qs = params.parallel_jobs
+    }
     executor {
-          name = 'local'
-          queueSize = qs
+      name = "local"
+      if (params.parallel_jobs || params.parallel_jobs != '') {
+        queueSize = qs
+      }
     }
   }
+
+  awsbatch {
+    process.executor = 'awsbatch'
+    process.queue = 'my-batch-queue'
+    // cpu allocation
+    process.cpus = params.threads
+  }
+}

docs/config.rst

@@ -89,9 +89,10 @@ Default configuration:
 
   // Number of jobs to run in parallel. Be aware that each job (in parallel) can consume
   // N threads (set above). Be sure to carefully check your resources before augmenting
-  // this parameter. For example: parallel_jobs = 2 + threads = 5 can consume up until 10
+  // this parameter. For example: parallel_jobs = 2 + threads = 5 can consume until 10
   // threads at once.
-    parallel_jobs = 1
+  // If not given, let's nextflow automatically handle it.
+        parallel_jobs = 
 
   // Number of minimum overlapping base pairs required for merging
   // Negative values, such as -20, means the number of required overlapping bases for merging.
@@ -225,14 +226,16 @@ Default configuration:
   profiles {
     standard {
       // Executor
-      process.executor = 'local'
+      process.executor = "local"
       // QueueSize limit
-      qs = (params.parallel_jobs) ? params.parallel_jobs : 1
+      if (params.parallel_jobs || params.parallel_jobs != '') {
+        qs = params.parallel_jobs
+      }
       executor {
-            name = 'local'
-            queueSize = qs
+        name = "local"
+        if (params.parallel_jobs || params.parallel_jobs != '') {
+          queueSize = qs
+        }
       }
-      // cpu allocation
-      process.cpus = params.threads
     }
   }

nextflow.config

@@ -76,9 +76,10 @@ params {
 
 // Number of jobs to run in parallel. Be aware that each job (in parallel) can consume
 // N threads (set above). Be sure to carefully check your resources before augmenting
-// this parameter. For example: parallel_jobs = 2 + threads = 5 can consume up until 10
+// this parameter. For example: parallel_jobs = 2 + threads = 5 can consume until 10
 // threads at once.
-  parallel_jobs = 1
+// If not given, let's nextflow automatically handle it.
+      parallel_jobs = 
 
 // Number of minimum overlapping base pairs required for merging
 // Negative values, such as -20, means the number of required overlapping bases for merging.
@@ -212,15 +213,17 @@ report {
 profiles {
   standard {
     // Executor
-    process.executor = 'local'
+    process.executor = "local"
     // QueueSize limit
-    qs = (params.parallel_jobs) ? params.parallel_jobs : 1
+    if (params.parallel_jobs || params.parallel_jobs != '') {
+      qs = params.parallel_jobs
+    }
     executor {
-          name = 'local'
-          queueSize = qs
+      name = "local"
+      if (params.parallel_jobs || params.parallel_jobs != '') {
+        queueSize = qs
+      }
     }
-    // cpu allocation
-    process.cpus = params.threads
   }
 
   awsbatch {

nextflow_schema.json

@@ -116,9 +116,8 @@
                 },
                 "parallel_jobs": {
                     "type": "integer",
-                    "default": 1,
                     "description": "Number of jobs to run in parallel",
-                    "help_text": "Number of jobs to run in parallel. Be aware that each job (in parallel) can consume\nN threads (set above). Be sure to carefully check your resources before augmenting\nthis parameter. For example: parallel_jobs = 2 + threads = 5 can consume until 10\nthreads at once."
+                    "help_text": "Number of jobs to run in parallel. Be aware that each job (in parallel) can consume\nN threads (set above). Be sure to carefully check your resources before augmenting\nthis parameter. For example: parallel_jobs = 2 + threads = 5 can consume until 10\nthreads at once.\nIf not given, let's nextflow automatically handle it"
                 }
             }
         },

nf_functions/help.nf

@@ -64,7 +64,8 @@ def helpMessage() {
     --threads <int>                                Number of threads to use
 
     --parallel_jobs <int>                          Number of jobs to run in parallel. Each job can consume up
-                                                   to N threads (--threads). Default: 1.
+                                                   to N threads (--threads). If not given, let's nextflow automatically
+                                                   handle it. Default: NA.
 
     --bedtools_merge_distance <int>                By default, this process is not executed. For execution
                                                    one needs to provide a minimum number of overlapping