refine how API limits are handled

Author: Zachary Foster

conf/modules.config

@@ -102,8 +102,8 @@ process {
             enabled: params.data_dir == "false"
         ]
         ext.args = { "--type ${params.bakta_db_type}" }
-        //storeDir = { params.data_dir == "false" ? null : "${params.data_dir}/bakta_db_${workflow.containerEngine}" }
         storeDir = { params.data_dir == "false" ? null : "${params.data_dir}/bakta_db" }
+        beforeScript = { task.attempt == 1 ? null : "sleep ${Math.pow(5, task.attempt)}" }
     }
 
     withName: BBMAP_SENDSKETCH {
@@ -117,8 +117,11 @@ process {
         memory = { 1.GB * task.attempt }
         time   = { 1.h  * task.attempt }
         maxRetries = 10
+
+        // Settings to avoid API rate limits and not put too much stress on servers
         maxForks = 3
-        errorStrategy = { sleep(Math.pow(4, task.attempt) * 6000 as long); return task.attempt > 2 ? 'finish' : 'retry' }
+        errorStrategy = { task.attempt >= 3 ? 'finish' : 'retry' }
+        beforeScript = { task.attempt == 1 ? 'sleep "0.$(($RANDOM % 100))"' : "sleep ${Math.pow(5, task.attempt)}" }
     }
 
     withName: BUSCO_BUSCO {
@@ -141,6 +144,7 @@ process {
             enabled: params.data_dir == "false"
         ]
         storeDir = { params.data_dir == "false" ? null : "${params.data_dir}/busco_db" }
+        beforeScript = { task.attempt == 1 ? null : "sleep ${Math.pow(5, task.attempt)}" }
     }
 
     withName: BWA_INDEX {
@@ -194,13 +198,15 @@ process {
                 pattern: "*.fasta.gz"
             ]
         ]
-        cpus   = { 1                   }
+        cpus   = { 1                     }
         memory = { 500.MB * task.attempt }
-        time   = { 24.h  * task.attempt }
+        time   = { 24.h  * task.attempt  }
         storeDir = { params.data_dir == "false" ? null : "${params.data_dir}/assemblies" }
-        maxForks = 3
+
+        // Settings to avoid API rate limits and not put too much stress on servers
+        maxForks = 1 // NCBI seems to be not allowing concurrent downloads, althogh I cannot find any documentation about this
         errorStrategy = { task.attempt >= 3 ? 'ignore' : 'retry' }
-        beforeScript = { task.attempt == 1 ? 'sleep "0.$(($RANDOM % 100))"' : "sleep ${Math.pow(10, task.attempt - 1)}" }
+        beforeScript = { task.attempt == 1 ? 'sleep "0.$(($RANDOM % 100))"' : "sleep ${Math.pow(5, task.attempt)}" }
     }
 
     withName: EXTRACT_FEATURE_SEQUENCES {
@@ -245,18 +251,13 @@ process {
         cpus   = { 1    * task.attempt }
         memory = { 1.GB * task.attempt }
         time   = { 12.h * task.attempt }
-        errorStrategy = {
-            if (task.attempt > 1) {
-                return 'ignore'
-            } else {
-                sleep(Math.pow(5, task.attempt - 1) * 6000 as long)
-                return 'retry'
-            }
-        }
-        maxForks = 3
         storeDir = { params.data_dir == "false" ? null : "${params.data_dir}/assembly_metadata" }
         ext.args = { "--as-json-lines" }
-        beforeScript = 'sleep "0.$(($RANDOM % 100))"'
+
+        // Settings to avoid API rate limits and not put too much stress on servers
+        maxForks = 3
+        errorStrategy = { task.attempt >= 3 ? 'ignore' : 'retry' }
+        beforeScript = { task.attempt == 1 ? 'sleep "0.$(($RANDOM % 100))"' : "sleep ${Math.pow(5, task.attempt)}" }
     }
 
     withName: 'FLYE.*' {
@@ -267,7 +268,7 @@ process {
         ]
         cpus   = { meta.domain == 'Bacteria' ? 4 * task.attempt                      : 8 * task.attempt                      }
         memory = { meta.domain == 'Bacteria' ? 15.GB * Math.pow(2, task.attempt - 1) : 30.GB * Math.pow(4, task.attempt - 1) }
-        time   = { 24.h  * task.attempt                                                                                       }
+        time   = { 24.h  * task.attempt                                                                                      }
         errorStrategy = { return task.attempt > 3 ? 'ignore' : 'retry' }
         maxRetries = 3
     }
@@ -313,8 +314,11 @@ process {
         cpus   = { 1                   }
         memory = { 1.GB * task.attempt }
         time   = { 1.h  * task.attempt }
+
+        // Settings to avoid API rate limits and not put too much stress on servers
         maxForks = 3
-        beforeScript = 'sleep "0.$(($RANDOM % 100))"'
+        errorStrategy = { task.attempt >= 3 ? 'ignore' : 'retry' }
+        beforeScript = { task.attempt == 1 ? 'sleep "0.$(($RANDOM % 100))"' : "sleep ${Math.pow(5, task.attempt)}" }
     }
 
     withName: IQTREE_BUSCO {
@@ -500,6 +504,10 @@ process {
         cpus   = { 1                   }
         memory = { 1.GB * task.attempt }
         time   = { 1.h  * task.attempt }
+
+        // Settings to avoid API rate limits and not put too much stress on servers
+        maxForks = 3
+        beforeScript = { task.attempt == 1 ? 'sleep "0.$(($RANDOM % 100))"' : "sleep ${Math.pow(5, task.attempt)}" }
     }
 
     withName: SAMTOOLS_FAIDX {
@@ -585,10 +593,12 @@ process {
         time   = { 48.h * task.attempt }
         ext.args = { params.temp_dir ? "--temp ${params.temp_dir}" : "" }
         storeDir = { params.data_dir == "false" ? null : "${params.data_dir}/reads" }
-        errorStrategy = { return task.attempt > 3 ? 'ignore' : 'retry' }
-        maxRetries = 3
+
+        // Settings to avoid API rate limits and not put too much stress on servers
         maxForks = 3
-        beforeScript = 'sleep "0.$(($RANDOM % 100))"'
+        maxRetries = 3
+        errorStrategy = { task.attempt >= 4 ? 'finish' : 'retry' }
+        beforeScript = { task.attempt == 1 ? 'sleep "0.$(($RANDOM % 100))"' : "sleep ${Math.pow(5, task.attempt)}" }
     }
 
     withName: SUBSET_BUSCO_GENES {

modules/local/custom/download_assemblies/main.nf

@@ -1,6 +1,7 @@
 process DOWNLOAD_ASSEMBLIES {
     tag "${ref_meta.id}"
     label 'process_single'
+    secret 'NCBI_API_KEY'
 
     conda "conda-forge::ncbi-datasets-cli=15.11.0 bioconda::samtools=1.18 conda-forge::unzip=6.0"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
@@ -26,6 +27,8 @@ process DOWNLOAD_ASSEMBLIES {
     prefix = task.ext.prefix ?: "${ref_meta.id}"
     def args = task.ext.args ?: ''
     """
+    echo \$NCBI_API_KEY
+
     # Download assemblies as zip archives
     datasets download genome accession $id --include gff3,genome --filename ${prefix}.zip
 

modules/nf-core/sratools/fasterqdump/sratools-fasterqdump.diff

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+# Watch the .command.log files in active docker containers
+
+watch 'docker stats --no-stream | tail -n +2 | cut -d " " -f 1 | xargs -I {} bash -c "echo -e "---------------{}---------------";docker exec {} tail -n 7 .command.log"'