Merge pull request #12 from icgc-argo/sanger-wxs-variant-calling@3.1.6-3.3.0

[release]

Author: junjun-zhang

sanger-wxs-variant-calling/main.nf

@@ -2,7 +2,7 @@
 nextflow.enable.dsl = 2
 name = 'sanger-wxs-variant-calling'
 short_name = 'sanger-wxs'
-version = '3.1.6-3.2.1'
+version = '3.1.6-3.3.0'
 
 
 /*
@@ -127,6 +127,8 @@ params.cleanup = true
 
 params.cpus = 1
 params.mem = 1
+params.max_retries = 5  // set to 0 will disable retry
+params.first_retry_wait_time = 1  // in seconds
 
 params.download = [:]
 params.generateBas = [:]
@@ -140,6 +142,8 @@ params.uploadVariant = [:]
 params.extractSangerCall = [:]
 
 download_params = [
+    'max_retries': params.max_retries,
+    'first_retry_wait_time': params.first_retry_wait_time,
     'song_url': params.song_url,
     'score_url': params.score_url,
     'api_token': params.api_token,
@@ -203,6 +207,9 @@ payloadGenVariantCall_params = [
 upload_params = [
     'cpus': params.cpus,
     'mem': params.mem,
+    'max_retries': params.max_retries,
+    'publish_dir': params.publish_dir,
+    'first_retry_wait_time': params.first_retry_wait_time,
     'song_url': params.song_url,
     'score_url': params.score_url,
     'api_token': params.api_token,
@@ -211,7 +218,7 @@ upload_params = [
 
 
 // Include all modules and pass params
-include { songScoreDownload as dnldT; songScoreDownload as dnldN } from './song-score-utils/song-score-download' params(download_params)
+include { SongScoreDownload as dnldT; SongScoreDownload as dnldN } from './wfpr_modules/github.com/icgc-argo/nextflow-data-processing-utility-tools/song-score-download@2.6.1/main.nf' params(download_params)
 include { generateBas as basT; generateBas as basN; } from './modules/raw.githubusercontent.com/icgc-argo/variant-calling-tools/generate-bas.0.2.1.0/tools/generate-bas/generate-bas' params(generateBas_params)
 include { sangerWxsVariantCaller as sangerWxs } from './modules/raw.githubusercontent.com/icgc-argo/variant-calling-tools/sanger-wxs-variant-caller.3.1.6-3/tools/sanger-wxs-variant-caller/sanger-wxs-variant-caller.nf' params(sangerWxsVariantCaller_params)
 include { repackSangerResults as repack } from './modules/raw.githubusercontent.com/icgc-argo/variant-calling-tools/repack-sanger-results.0.2.0.0/tools/repack-sanger-results/repack-sanger-results' params(repackSangerResults_params)
@@ -220,7 +227,7 @@ include { prepSangerSupplement as prepSupp } from './modules/raw.githubuserconte
 include { prepSangerQc as prepQc } from './modules/raw.githubusercontent.com/icgc-argo/variant-calling-tools/prep-sanger-qc.0.1.2.0/tools/prep-sanger-qc/prep-sanger-qc' params(prepSangerQc_params)
 include { extractFilesFromTarball as extractVarSnv; extractFilesFromTarball as extractVarIndel; extractFilesFromTarball as extractQC } from './modules/raw.githubusercontent.com/icgc-argo/data-processing-utility-tools/extract-files-from-tarball.0.2.0.0/tools/extract-files-from-tarball/extract-files-from-tarball' params(extractSangerCall_params)
 include { payloadGenVariantCalling as pGenVarSnv; payloadGenVariantCalling as pGenVarIndel; payloadGenVariantCalling as pGenVarSupp; payloadGenVariantCalling as pGenQc } from "./modules/raw.githubusercontent.com/icgc-argo/data-processing-utility-tools/payload-gen-variant-calling.0.3.6.0/tools/payload-gen-variant-calling/payload-gen-variant-calling" params(payloadGenVariantCall_params)
-include { songScoreUpload as upSnv; songScoreUpload as upIndel; songScoreUpload as upQc; songScoreUpload as upSupp} from './song-score-utils/song-score-upload' params(upload_params)
+include { SongScoreUpload as upSnv; SongScoreUpload as upIndel; SongScoreUpload as upQc; SongScoreUpload as upSupp} from './wfpr_modules/github.com/icgc-argo/nextflow-data-processing-utility-tools/song-score-upload@2.6.1/main.nf' params(upload_params)
 include { cleanupWorkdir as cleanup } from './wfpr_modules/github.com/icgc-argo/data-processing-utility-tools/cleanup-workdir@1.0.0/main'
 include { getSecondaryFiles } from './wfpr_modules/github.com/icgc-argo/data-processing-utility-tools/helper-functions@1.0.0/main'
 include { payloadAddUniformIds as pAddIdT; payloadAddUniformIds as pAddIdN } from './wfpr_modules/github.com/icgc-argo/data-processing-utility-tools/payload-add-uniform-ids@0.1.1/main'
@@ -253,13 +260,13 @@ workflow SangerWxs {
             dnldT(study_id, tumour_aln_analysis_id)
             tumour_aln_seq = dnldT.out.files.flatten().first()
             tumour_aln_seq_idx = dnldT.out.files.flatten().last()
-            tumour_aln_meta = dnldT.out.song_analysis
+            tumour_aln_meta = dnldT.out.analysis_json
 
             // download normal aligned seq and metadata from song/score (analysis type: sequencing_alignment)
             dnldN(study_id, normal_aln_analysis_id)
             normal_aln_seq = dnldN.out.files.flatten().first()
             normal_aln_seq_idx = dnldN.out.files.flatten().last()
-            normal_aln_meta = dnldN.out.song_analysis
+            normal_aln_meta = dnldN.out.analysis_json
         } else if (
             !tumour_aln_metadata.startsWith('NO_FILE') && \
             !tumour_aln_cram.startsWith('NO_FILE') && \

sanger-wxs-variant-calling/pkg.json

@@ -1,6 +1,6 @@
 {
     "name": "sanger-wxs-variant-calling",
-    "version": "3.1.6-3.2.1",
+    "version": "3.1.6-3.3.0",
     "description": "ICGC ARGO Sanger WXS Variant Calling Workflow",
     "main": "main.nf",
     "deprecated": false,
@@ -16,7 +16,9 @@
     "dependencies": [
         "github.com/icgc-argo/data-processing-utility-tools/payload-add-uniform-ids@0.1.1",
         "github.com/icgc-argo/data-processing-utility-tools/helper-functions@1.0.0",
-        "github.com/icgc-argo/data-processing-utility-tools/cleanup-workdir@1.0.0"
+        "github.com/icgc-argo/data-processing-utility-tools/cleanup-workdir@1.0.0",
+        "github.com/icgc-argo/nextflow-data-processing-utility-tools/song-score-download@2.6.1",
+        "github.com/icgc-argo/nextflow-data-processing-utility-tools/song-score-upload@2.6.1"
     ],
     "devDependencies": [],
     "contributors": [

sanger-wxs-variant-calling/song-score-utils

@@ -2,6 +2,8 @@
   "study_id": "TEST-PR",
   "tumour_aln_analysis_id": "03905962-f1e1-41e8-9059-62f1e1c1e8fc",
   "normal_aln_analysis_id": "9940db0f-c100-496a-80db-0fc100d96ac1",
+  "max_retries": 3,
+  "first_retry_wait_time": 5,
   "cleanup": false,
   "song_url": "https://song.rdpc-qa.cancercollaboratory.org",
   "score_url": "https://score.rdpc-qa.cancercollaboratory.org",

sanger-wxs-variant-calling/tests/local-test-1.nf.json

@@ -5,6 +5,11 @@ nextflow.enable.dsl=2
 params.cpus = 8
 params.mem = 20
 
+params.publish_dir = ""
+
+params.max_retries = 5  // set to 0 will disable retry
+params.first_retry_wait_time = 1  // in seconds
+
 // required params w/ default
 params.container_version = "5.0.0"
 params.transport_mem = 2 // Transport memory is in number of GBs
@@ -18,12 +23,19 @@ params.api_token = "" // song/score API token for download process
 
 // TODO: Replace with score container once it can download files via analysis_id
 process scoreDownload {
+    maxRetries params.max_retries
+    errorStrategy {
+        sleep(Math.pow(2, task.attempt) * params.first_retry_wait_time * 1000 as long);  // backoff time increases exponentially before each retry
+        return params.max_retries ? 'retry' : 'finish'
+    }
+
     pod = [secret: workflow.runName + "-secret", mountPath: "/tmp/rdpc_secret"]
 
     cpus params.cpus
     memory "${params.mem} GB"
 
     container "overture/score:${params.container_version}"
+    publishDir "${params.publish_dir}/${task.process.replaceAll(':', '_')}", mode: "copy", enabled: params.publish_dir
 
     label "scoreDownload"
     tag "${analysis_id}"

song-score-utils/song-score-download.nf

@@ -5,6 +5,11 @@ nextflow.enable.dsl=2
 params.cpus = 1
 params.mem = 1
 
+params.publish_dir = ""
+
+params.max_retries = 5  // set to 0 will disable retry
+params.first_retry_wait_time = 1  // in seconds
+
 // required params w/ default
 params.container_version = "4.2.1"
 
@@ -16,12 +21,19 @@ params.api_token = "" // song/score API token for download process
 // --score_url        score url for download process
 
 process songGetAnalysis {
+    maxRetries params.max_retries
+    errorStrategy {
+        sleep(Math.pow(2, task.attempt) * params.first_retry_wait_time * 1000 as long);  // backoff time increases exponentially before each retry
+        return params.max_retries ? 'retry' : 'finish'
+    }
+
     pod = [secret: workflow.runName + "-secret", mountPath: "/tmp/rdpc_secret"]
 
     cpus params.cpus
     memory "${params.mem} GB"
 
     container "overture/song-client:${params.container_version}"
+    publishDir "${params.publish_dir}/${task.process.replaceAll(':', '_')}", mode: "copy", enabled: params.publish_dir
 
     tag "${analysis_id}"