Skip to content

Commit f843d1d

Browse files
lindaxianglindaxiang
authored
Merge pull request #5 from icgc-argo-workflows/open-access-variant-filtering-wf@0.5.0
[release]
2 parents 90670cb + 19186a1 commit f843d1d

17 files changed

Lines changed: 163 additions & 70 deletions

File tree

‎.github/workflows/build-test-release.yml‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,13 @@ jobs:
1919
steps:
2020
- uses: actions/checkout@v2
2121

22-
- name: Set up Python 3.6
23-
uses: actions/setup-python@v2
22+
- name: Set up Python 3.10
23+
uses: actions/setup-python@v3
2424
with:
25-
python-version: 3.6
25+
python-version: "3.10"
26+
architecture: "x64"
27+
env:
28+
AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache
2629

2730
- name: Extract package name and version from branch name
2831
id: get_pkg_info
@@ -80,10 +83,13 @@ jobs:
8083
steps:
8184
- uses: actions/checkout@v2
8285

83-
- name: Set up Python 3.6
84-
uses: actions/setup-python@v2
86+
- name: Set up Python 3.10
87+
uses: actions/setup-python@v3
8588
with:
86-
python-version: 3.6
89+
python-version: "3.10"
90+
architecture: "x64"
91+
env:
92+
AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache
8793

8894
- name: Install dependencies
8995
run: |
@@ -177,6 +183,8 @@ jobs:
177183
body: |
178184
* Release `${{ steps.get_pkg_info.outputs.pkg_name }}.v${{ steps.get_pkg_info.outputs.pkg_ver }}` (${{ github.sha }})
179185
* Package `${{ steps.prep_assets.outputs.pkg_tar }}` (sha256: `${{ steps.prep_assets.outputs.pkg_tar_sha }}`)
186+
* Package URI `github.com/${{ needs.build.outputs.repo_lowercase }}/${{ steps.get_pkg_info.outputs.pkg_name }}@${{ steps.get_pkg_info.outputs.pkg_ver }}`
187+
* Run the package: `nextflow run ${{ needs.build.outputs.repo_lowercase }}/${{ steps.get_pkg_info.outputs.pkg_name }}/main.nf -r ${{ steps.get_pkg_info.outputs.pkg_name }}.v${{ steps.get_pkg_info.outputs.pkg_ver }} -params-file <params-json-file>`
180188
draft: false
181189
prerelease: false
182190

‎open-access-variant-filtering-wf/main.nf‎

Lines changed: 39 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,15 @@
2323
nextflow.enable.dsl = 2
2424
name = 'open-access-variant-filtering'
2525
short_name = 'open-filter'
26-
version = '0.4.0'
26+
version = '0.5.0'
2727

2828
// universal params go here, change default value as needed
29-
params.container = ""
30-
params.container_registry = ""
31-
params.container_version = ""
3229
params.cpus = 1
3330
params.mem = 1 // GB
3431
params.publish_dir = "" // set to empty string will disable publishDir
32+
params.cleanup = true
33+
params.max_retries = 5 // set to 0 will disable retry
34+
params.first_retry_wait_time = 1 // in seconds
3535

3636
// tool specific parmas go here, add / change as needed
3737
params.study_id = ""
@@ -59,33 +59,57 @@ params.open = true
5959
params.analysis_metadata = "NO_FILE_metadata"
6060
params.vcf_file = "NO_FILE_vcf"
6161

62-
params.api_token = ""
62+
// song/score setting
6363
params.song_url = ""
64+
params.song_container = "ghcr.io/overture-stack/song-client"
65+
params.song_container_version = "5.0.2"
6466
params.score_url = ""
65-
params.cleanup = true
67+
params.score_container = "ghcr.io/overture-stack/score"
68+
params.score_container_version = "5.9.0"
69+
params.score_mem = 20
70+
params.score_cpus = 8
71+
params.score_force = false
72+
params.api_token = ""
6673

6774
params.download = [:]
6875
params.upload = [:]
6976
params.filter = [:]
7077
params.payloadGenVcf = [:]
7178

7279
download_params = [
80+
'max_retries': params.max_retries,
81+
'first_retry_wait_time': params.first_retry_wait_time,
82+
'song_url': params.song_url,
83+
'song_container': params.song_container,
84+
'song_container_version': params.song_container_version,
7385
'song_cpus': params.cpus,
7486
'song_mem': params.mem,
75-
'score_cpus': params.cpus,
76-
'score_mem': params.mem,
77-
'song_url': params.song_url,
7887
'score_url': params.score_url,
88+
'score_container': params.score_container,
89+
'score_container_version': params.score_container_version,
90+
'score_cpus' : params.score_cpus,
91+
'score_mem' : params.score_mem,
92+
'score_transport_mem' : params.score_mem,
7993
'api_token': params.api_token,
8094
'publish_dir': params.publish_dir,
8195
*:(params.download ?: [:])
8296
]
8397

8498
upload_params = [
85-
'cpus': params.cpus,
86-
'mem': params.mem,
99+
'max_retries': params.max_retries,
100+
'first_retry_wait_time': params.first_retry_wait_time,
87101
'song_url': params.song_url,
102+
'song_container': params.song_container,
103+
'song_container_version': params.song_container_version,
104+
'song_cpus': params.cpus,
105+
'song_mem': params.mem,
88106
'score_url': params.score_url,
107+
'score_container': params.score_container,
108+
'score_container_version': params.score_container_version,
109+
'score_force' : params.score_force,
110+
'score_cpus' : params.score_cpus,
111+
'score_mem' : params.score_mem,
112+
'score_transport_mem' : params.score_mem,
89113
'api_token': params.api_token,
90114
'publish_dir': params.publish_dir,
91115
*:(params.upload ?: [:])
@@ -94,7 +118,7 @@ upload_params = [
94118
filter_params = [
95119
'cpus': params.cpus,
96120
'mem': params.mem,
97-
'publish_dir': '',
121+
'publish_dir': params.publish_dir,
98122
'regions_file': params.regions_file,
99123
'apply_filters': params.apply_filters,
100124
'output_type': params.output_type,
@@ -110,11 +134,11 @@ payloadGenVcf_params = [
110134
*:(params.payloadGenVcf ?: [:])
111135
]
112136

113-
include { SongScoreDownload as dnVcf } from './wfpr_modules/github.com/icgc-argo/nextflow-data-processing-utility-tools/song-score-download@2.6.1/main.nf' params(download_params)
137+
include { SongScoreDownload as dnVcf } from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/main.nf' params(download_params)
114138
include { metadataParser as mParser } from "./modules/raw.githubusercontent.com/icgc-argo/data-processing-utility-tools/metadata-parser.0.2.0.0/tools/metadata-parser/metadata-parser.nf"
115139
include { variantFilter as vFilter } from './wfpr_modules/github.com/icgc-argo/variant-calling-tools/variant-filter@0.1.0/main.nf' params(filter_params)
116140
include { payloadGenVariantProcessing as pGenVar } from "./wfpr_modules/github.com/icgc-argo-workflows/data-processing-utility-tools/payload-gen-variant-processing@0.3.0/main.nf" params(payloadGenVcf_params)
117-
include { SongScoreUpload as upVcf } from './wfpr_modules/github.com/icgc-argo/nextflow-data-processing-utility-tools/song-score-upload@2.6.1/main.nf' params(upload_params)
141+
include { SongScoreUpload as upVcf } from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/main.nf' params(upload_params)
118142
include { getSecondaryFiles as getSec } from './wfpr_modules/github.com/icgc-argo/data-processing-utility-tools/helper-functions@1.0.0/main.nf'
119143
include { cleanupWorkdir as cleanup } from './wfpr_modules/github.com/icgc-argo/data-processing-utility-tools/cleanup-workdir@1.0.0/main.nf'
120144

@@ -175,7 +199,7 @@ workflow OpenFilterWf {
175199
// skip upload if in local_mode
176200
if (!local_mode) {
177201
// uploadVariant
178-
upVcf(study_id, pGenVar.out.payload, pGenVar.out.files_to_upload)
202+
upVcf(study_id, pGenVar.out.payload, pGenVar.out.files_to_upload, '')
179203
}
180204

181205
// cleanup, skip cleanup when running in local mode

‎open-access-variant-filtering-wf/pkg.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "open-access-variant-filtering-wf",
3-
"version": "0.4.0",
3+
"version": "0.5.0",
44
"description": "ICGC-ARGO Open-access Somatic Variant Filtering Workflow",
55
"main": "main.nf",
66
"deprecated": false,
@@ -18,8 +18,8 @@
1818
"github.com/icgc-argo/data-processing-utility-tools/helper-functions@1.0.0",
1919
"github.com/icgc-argo-workflows/data-processing-utility-tools/payload-gen-variant-processing@0.3.0",
2020
"github.com/icgc-argo/variant-calling-tools/variant-filter@0.1.0",
21-
"github.com/icgc-argo/nextflow-data-processing-utility-tools/song-score-download@2.6.1",
22-
"github.com/icgc-argo/nextflow-data-processing-utility-tools/song-score-upload@2.6.1"
21+
"github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0",
22+
"github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3"
2323
],
2424
"devDependencies": [],
2525
"contributors": [

wfpr_modules/github.com/icgc-argo/nextflow-data-processing-utility-tools/song-score-download@2.6.1/local_modules/score-download.nf renamed to wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/local_modules/score-download.nf

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ params.max_retries = 5 // set to 0 will disable retry
1111
params.first_retry_wait_time = 1 // in seconds
1212

1313
// required params w/ default
14-
params.container_version = "5.0.0"
14+
params.container = "ghcr.io/overture-stack/score"
15+
params.container_version = "5.8.1"
1516
params.transport_mem = 2 // Transport memory is in number of GBs
1617

1718
// optional if secret mounted from pod else required
@@ -34,12 +35,18 @@ process scoreDownload {
3435
cpus params.cpus
3536
memory "${params.mem} GB"
3637

37-
container "overture/score:${params.container_version}"
38-
publishDir "${params.publish_dir}/${task.process.replaceAll(':', '_')}", mode: "copy", enabled: params.publish_dir
38+
container "${ params.score_container ?: params.container}:${params.score_container_version ?: params.container_version}"
39+
publishDir "${params.publish_dir}/${task.process.replaceAll(':', '_')}", mode: "copy", enabled: params.publish_dir ? true : false
3940

4041
label "scoreDownload"
4142
tag "${analysis_id}"
4243

44+
if (workflow.containerEngine == "singularity") {
45+
containerOptions "--bind \$(pwd):/score-client/logs"
46+
} else if (workflow.containerEngine == "docker") {
47+
containerOptions "-v \$(pwd):/score-client/logs"
48+
}
49+
4350
input:
4451
path analysis
4552
val study_id

wfpr_modules/github.com/icgc-argo/nextflow-data-processing-utility-tools/song-score-download@2.6.1/local_modules/song-get-analysis.nf renamed to wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/local_modules/song-get-analysis.nf

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ params.max_retries = 5 // set to 0 will disable retry
1111
params.first_retry_wait_time = 1 // in seconds
1212

1313
// required params w/ default
14-
params.container_version = "4.2.1"
14+
params.container = "ghcr.io/overture-stack/song-client"
15+
params.container_version = "5.0.2"
1516

1617
// optional if secret mounted from pod else required
1718
params.api_token = "" // song/score API token for download process
@@ -31,12 +32,18 @@ process songGetAnalysis {
3132

3233
cpus params.cpus
3334
memory "${params.mem} GB"
34-
35-
container "overture/song-client:${params.container_version}"
36-
publishDir "${params.publish_dir}/${task.process.replaceAll(':', '_')}", mode: "copy", enabled: params.publish_dir
35+
36+
container "${ params.song_container ?: params.container}:${params.song_container_version ?: params.container_version}"
37+
publishDir "${params.publish_dir}/${task.process.replaceAll(':', '_')}", mode: "copy", enabled: params.publish_dir ? true : false
3738

3839
tag "${analysis_id}"
3940

41+
if (workflow.containerEngine == "singularity") {
42+
containerOptions "--bind \$(pwd):/song-client/logs"
43+
} else if (workflow.containerEngine == "docker") {
44+
containerOptions "-v \$(pwd):/song-client/logs"
45+
}
46+
4047
input:
4148
val study_id
4249
val analysis_id

wfpr_modules/github.com/icgc-argo/nextflow-data-processing-utility-tools/song-score-download@2.6.1/main.nf renamed to wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/main.nf

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
*/
2222

2323
nextflow.enable.dsl = 2
24-
version = '2.6.1'
24+
version = '2.9.0'
2525

2626
// universal params go here, change default value as needed
2727
params.publish_dir = "" // set to empty string will disable publishDir
@@ -30,33 +30,35 @@ params.max_retries = 5 // set to 0 will disable retry
3030
params.first_retry_wait_time = 1 // in seconds
3131

3232
// tool specific parmas go here, add / change as needed
33-
params.study_id = "TEST-PR"
34-
params.analysis_id = "9940db0f-c100-496a-80db-0fc100d96ac1"
33+
params.study_id = ""
34+
params.analysis_id = ""
3535

3636
params.api_token = ""
3737

3838
params.song_cpus = 1
3939
params.song_mem = 1 // GB
4040
params.song_url = "https://song.rdpc-qa.cancercollaboratory.org"
4141
params.song_api_token = ""
42-
params.song_container_version = "4.2.1"
42+
params.song_container = "ghcr.io/overture-stack/song-client"
43+
params.song_container_version = "5.0.2"
4344

4445
params.score_cpus = 1
4546
params.score_mem = 1 // GB
4647
params.score_transport_mem = 1 // GB
4748
params.score_url = "https://score.rdpc-qa.cancercollaboratory.org"
4849
params.score_api_token = ""
49-
params.score_container_version = "5.0.0"
50+
params.score_container = "ghcr.io/overture-stack/score"
51+
params.score_container_version = "5.8.1"
5052

5153

5254
song_params = [
5355
*:params,
5456
'cpus': params.song_cpus,
5557
'mem': params.song_mem,
5658
'song_url': params.song_url,
59+
'song_container': params.song_container,
5760
'song_container_version': params.song_container_version,
58-
'api_token': params.song_api_token ?: params.api_token,
59-
'publish_dir': ''
61+
'api_token': params.song_api_token ?: params.api_token
6062
]
6163

6264
score_params = [
@@ -66,6 +68,7 @@ score_params = [
6668
'transport_mem': params.score_transport_mem,
6769
'song_url': params.song_url,
6870
'score_url': params.score_url,
71+
'score_container': params.score_container,
6972
'score_container_version': params.score_container_version,
7073
'api_token': params.score_api_token ?: params.api_token
7174
]

wfpr_modules/github.com/icgc-argo/nextflow-data-processing-utility-tools/song-score-download@2.6.1/nextflow.config renamed to wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/nextflow.config

File renamed without changes.

wfpr_modules/github.com/icgc-argo/nextflow-data-processing-utility-tools/song-score-download@2.6.1/pkg.json renamed to wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/pkg.json

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "song-score-download",
3-
"version": "2.6.1",
3+
"version": "2.9.0",
44
"description": "SONG/SCORE download",
55
"main": "main.nf",
66
"deprecated": false,
@@ -12,7 +12,7 @@
1212
],
1313
"repository": {
1414
"type": "git",
15-
"url": "https://github.com/icgc-argo/nextflow-data-processing-utility-tools.git"
15+
"url": "https://github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools.git"
1616
},
1717
"dependencies": [],
1818
"devDependencies": [],
@@ -22,9 +22,12 @@
2222
},
2323
{
2424
"name": "Junjun Zhang"
25+
},
26+
{
27+
"name": "Linda Xiang"
2528
}
2629
],
2730
"license": "Apache License 2.0",
28-
"bugReport": "https://github.com/icgc-argo/nextflow-data-processing-utility-tools/issues",
29-
"homepage": "https://github.com/icgc-argo/nextflow-data-processing-utility-tools#readme"
31+
"bugReport": "https://github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/issues",
32+
"homepage": "https://github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools#readme"
3033
}

wfpr_modules/github.com/icgc-argo/nextflow-data-processing-utility-tools/song-score-download@2.6.1/wfpr_modules renamed to wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/wfpr_modules

File renamed without changes.

wfpr_modules/github.com/icgc-argo/nextflow-data-processing-utility-tools/song-score-upload@2.6.1/local_modules/score-upload.nf renamed to wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/local_modules/score-upload.nf

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,13 @@ params.max_retries = 5 // set to 0 will disable retry
99
params.first_retry_wait_time = 1 // in seconds
1010

1111
// required params w/ default
12-
params.container_version = "5.0.0"
12+
params.container = "ghcr.io/overture-stack/score"
13+
params.container_version = "latest"
1314
params.transport_mem = 2 // Transport memory is in number of GBs
1415

1516
// optional if secret mounted from pod else required
1617
params.api_token = "" // song/score API token for download process
17-
18+
params.score_force = false
1819
// required params, no default
1920
// --song_url song url for download process
2021
// --score_url score url for download process
@@ -30,8 +31,14 @@ process scoreUpload {
3031

3132
cpus params.cpus
3233
memory "${params.mem} GB"
33-
34-
container "overture/score:${params.container_version}"
34+
35+
container "${ params.score_container ?: params.container}:${params.score_container_version ?: params.container_version}"
36+
37+
if (workflow.containerEngine == "singularity") {
38+
containerOptions "--bind \$(pwd):/score-client/logs"
39+
} else if (workflow.containerEngine == "docker") {
40+
containerOptions "-v \$(pwd):/score-client/logs"
41+
}
3542

3643
tag "${analysis_id}"
3744

@@ -45,13 +52,14 @@ process scoreUpload {
4552

4653
script:
4754
accessToken = params.api_token ? params.api_token : "`cat /tmp/rdpc_secret/secret`"
55+
forceFlag = params.score_force ? "--force" : ""
4856
"""
4957
export METADATA_URL=${params.song_url}
5058
export STORAGE_URL=${params.score_url}
5159
export TRANSPORT_PARALLEL=${params.cpus}
5260
export TRANSPORT_MEMORY=${params.transport_mem}
5361
export ACCESSTOKEN=${accessToken}
5462
55-
score-client upload --manifest ${manifest}
63+
score-client upload --manifest ${manifest} ${forceFlag}
5664
"""
5765
}

0 commit comments

Comments
 (0)