Skip to content

Commit 2e85325

Browse files
dpark01dpark01
committed
Merge branch 'dp-metrics' of https://github.com/broadinstitute/viral-pipelines into dp-metrics
2 parents de2d943 + 7166858 commit 2e85325

17 files changed

Lines changed: 221 additions & 37 deletions

‎.dockstore.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,11 @@ workflows:
225225
primaryDescriptorPath: /pipes/WDL/workflows/multiqc_only.wdl
226226
testParameterFiles:
227227
- empty.json
228+
- name: sarscov2_biosample_load
229+
subclass: WDL
230+
primaryDescriptorPath: /pipes/WDL/workflows/sarscov2_biosample_load.wdl
231+
testParameterFiles:
232+
- empty.json
228233
- name: sarscov2_genbank
229234
subclass: WDL
230235
primaryDescriptorPath: /pipes/WDL/workflows/sarscov2_genbank.wdl

‎.travis.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ jobs:
8686
- language: java
8787
jdk: openjdk11
8888
stage: build
89+
if: branch = master
8990
env:
9091
- TRAVIS_JOB=deploy_dnanexus
9192
# DX_API_TOKEN (for DNAnexus builds) -- token #3, expires 12/31/2024

‎pipes/WDL/tasks/tasks_assembly.wdl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -291,7 +291,7 @@ task align_reads {
291291
Boolean? skip_mark_dupes = false
292292

293293
Int? machine_mem_gb
294-
String docker = "quay.io/broadinstitute/viral-core:2.1.26"
294+
String docker = "quay.io/broadinstitute/viral-core:2.1.28"
295295

296296
String sample_name = basename(basename(basename(reads_unmapped_bam, ".bam"), ".taxfilt"), ".clean")
297297
}
@@ -708,7 +708,7 @@ task run_discordance {
708708
String out_basename = "run"
709709
Int min_coverage = 4
710710

711-
String docker = "quay.io/broadinstitute/viral-core:2.1.26"
711+
String docker = "quay.io/broadinstitute/viral-core:2.1.28"
712712
}
713713

714714
command {

‎pipes/WDL/tasks/tasks_demux.wdl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ task merge_tarballs {
66
String out_filename
77

88
Int? machine_mem_gb
9-
String docker = "quay.io/broadinstitute/viral-core:2.1.26"
9+
String docker = "quay.io/broadinstitute/viral-core:2.1.28"
1010
}
1111

1212
command {
@@ -142,7 +142,7 @@ task illumina_demux {
142142
Int? maxRecordsInRam
143143

144144
Int? machine_mem_gb
145-
String docker = "quay.io/broadinstitute/viral-core:2.1.26"
145+
String docker = "quay.io/broadinstitute/viral-core:2.1.28"
146146
}
147147
parameter_meta {
148148
flowcell_tgz: {

‎pipes/WDL/tasks/tasks_interhost.wdl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ task index_ref {
142142
File? novocraft_license
143143

144144
Int? machine_mem_gb
145-
String docker = "quay.io/broadinstitute/viral-core:2.1.26"
145+
String docker = "quay.io/broadinstitute/viral-core:2.1.28"
146146
}
147147

148148
command {

‎pipes/WDL/tasks/tasks_ncbi.wdl‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,7 @@ task structured_comments {
186186

187187
File? filter_to_ids
188188

189-
String docker = "quay.io/broadinstitute/viral-core:2.1.26"
189+
String docker = "quay.io/broadinstitute/viral-core:2.1.28"
190190
}
191191
String out_base = basename(assembly_stats_tsv, '.txt')
192192
command <<<
@@ -264,7 +264,7 @@ task rename_fasta_header {
264264

265265
String out_basename = basename(genome_fasta, ".fasta")
266266

267-
String docker = "quay.io/broadinstitute/viral-core:2.1.26"
267+
String docker = "quay.io/broadinstitute/viral-core:2.1.28"
268268
}
269269
command {
270270
set -e
@@ -417,9 +417,7 @@ task register_biosamples {
417417
node src/main.js -i=input.tsv \
418418
~{'-c="' + comment + '"'} \
419419
~{'-d=' + embargo_date} \
420-
-u="~{true='Test' false='Production' test}/~{meta_basename}"
421-
422-
# huh.. need to strip off empty entry at the end of xml....
420+
-u="/~{true='Test' false='Production' test}/~{meta_basename}"
423421
424422
}
425423
output {
@@ -447,7 +445,7 @@ task sra_meta_prep {
447445
Boolean paired
448446

449447
String out_name = "sra_metadata.tsv"
450-
String docker="quay.io/broadinstitute/viral-core:2.1.26"
448+
String docker="quay.io/broadinstitute/viral-core:2.1.28"
451449
}
452450
parameter_meta {
453451
cleaned_bam_filepaths: {

‎pipes/WDL/tasks/tasks_ncbi_tools.wdl‎

Lines changed: 136 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ task Fetch_SRA_to_BAM {
66
String SRA_ID
77

88
Int? machine_mem_gb
9-
String docker = "quay.io/broadinstitute/ncbi-tools:2.10.7.1"
9+
String docker = "quay.io/broadinstitute/ncbi-tools:2.10.7.2"
1010
}
1111

1212
command <<<
@@ -139,6 +139,141 @@ task Fetch_SRA_to_BAM {
139139
}
140140
}
141141

142+
task fetch_biosamples {
143+
144+
input {
145+
Array[String] biosample_ids
146+
147+
String out_basename = "biosample_attributes"
148+
String docker = "quay.io/broadinstitute/ncbi-tools:2.10.7.2"
149+
}
150+
151+
command <<<
152+
set -e
153+
/opt/docker/scripts/biosample-fetch_attributes.py \
154+
~{sep=' ' biosample_ids} "~{out_basename}"
155+
>>>
156+
157+
output {
158+
File biosample_attributes_tsv = "~{out_basename}.tsv"
159+
File biosample_attributes_json = "~{out_basename}.json"
160+
}
161+
162+
runtime {
163+
cpu: 2
164+
memory: "3 GB"
165+
disks: "local-disk 50 HDD"
166+
dx_instance_type: "mem2_ssd1_v2_x2"
167+
docker: docker
168+
}
169+
}
170+
171+
task ncbi_ftp_upload {
172+
input {
173+
Array[File] submit_files
174+
File config_js
175+
String target_path
176+
177+
String wait_for="1" # all, disabled, some number
178+
179+
String docker = "quay.io/broadinstitute/ncbi-tools:2.10.7.2"
180+
}
181+
182+
command <<<
183+
set -e
184+
cd /opt/converter
185+
cp "~{config_js}" src/
186+
rm -f files/sample.tsv reports/sample-report.xml
187+
cp ~{sep=' ' submit_files} files/
188+
MANIFEST=$(ls -1 files | paste -sd,)
189+
echo "uploading: $MANIFEST to destination ftp folder ~{target_path}"
190+
node src/main.js --debug \
191+
--uploadFiles="$MANIFEST" \
192+
--poll="~{wait_for}" \
193+
--uploadFolder="~{target_path}"
194+
cd -
195+
cp /opt/converter/reports/*report*.xml .
196+
ls -alF files reports
197+
>>>
198+
199+
output {
200+
Array[File] reports_xmls = glob("*report*.xml")
201+
}
202+
203+
runtime {
204+
cpu: 2
205+
memory: "2 GB"
206+
disks: "local-disk 100 HDD"
207+
dx_instance_type: "mem2_ssd1_v2_x2"
208+
docker: docker
209+
}
210+
}
211+
212+
task biosample_submit_tsv_to_xml {
213+
input {
214+
File meta_submit_tsv
215+
File config_js
216+
217+
String docker = "quay.io/broadinstitute/ncbi-tools:2.10.7.2"
218+
}
219+
command <<<
220+
set -e
221+
cd /opt/converter
222+
cp "~{config_js}" src/
223+
rm files/sample.tsv
224+
cp "~{meta_submit_tsv}" files/
225+
node src/main.js --debug \
226+
-i=$(basename "~{meta_submit_tsv}") \
227+
--runTestMode=true
228+
cd -
229+
cp "/opt/converter/files/~{basename(meta_submit_tsv, '.tsv')}-submission.xml" .
230+
>>>
231+
output {
232+
File submission_xml = "~{basename(meta_submit_tsv, '.tsv')}-submission.xml"
233+
}
234+
runtime {
235+
cpu: 2
236+
memory: "2 GB"
237+
disks: "local-disk 100 HDD"
238+
dx_instance_type: "mem2_ssd1_v2_x2"
239+
docker: docker
240+
}
241+
}
242+
243+
task biosample_submit_tsv_ftp_upload {
244+
input {
245+
File meta_submit_tsv
246+
File config_js
247+
String target_path
248+
249+
String docker = "quay.io/broadinstitute/ncbi-tools:2.10.7.2"
250+
}
251+
String base=basename(meta_submit_tsv, '.tsv')
252+
command <<<
253+
set -e
254+
cd /opt/converter
255+
cp "~{config_js}" src/
256+
rm -f files/sample.tsv reports/sample-report.xml
257+
cp "~{meta_submit_tsv}" files/
258+
node src/main.js --debug \
259+
-i=$(basename "~{meta_submit_tsv}") \
260+
--uploadFolder="~{target_path}"
261+
cd -
262+
cp /opt/converter/reports/~{base}-attributes.tsv /opt/converter/files/~{base}-submission.xml /opt/converter/reports/~{base}-report.*.xml .
263+
>>>
264+
output {
265+
File attributes_tsv = "~{base}-attributes.tsv"
266+
File submission_xml = "~{base}-submission.xml"
267+
Array[File] reports_xmls = glob("~{base}-report.*.xml")
268+
}
269+
runtime {
270+
cpu: 2
271+
memory: "2 GB"
272+
disks: "local-disk 100 HDD"
273+
dx_instance_type: "mem2_ssd1_v2_x2"
274+
docker: docker
275+
}
276+
}
142277

143278
task group_sra_bams_by_biosample {
144279
input {

‎pipes/WDL/tasks/tasks_nextstrain.wdl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ task derived_cols {
8181
String? lab_highlight_loc
8282
Array[File] table_map = []
8383

84-
String docker = "quay.io/broadinstitute/viral-core:2.1.26"
84+
String docker = "quay.io/broadinstitute/viral-core:2.1.28"
8585
}
8686
parameter_meta {
8787
lab_highlight_loc: {
@@ -480,7 +480,7 @@ task filter_sequences_by_length {
480480
File sequences_fasta
481481
Int min_non_N = 1
482482

483-
String docker = "quay.io/broadinstitute/viral-core:2.1.26"
483+
String docker = "quay.io/broadinstitute/viral-core:2.1.28"
484484
}
485485
parameter_meta {
486486
sequences_fasta: {

‎pipes/WDL/tasks/tasks_read_utils.wdl‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ task get_sample_meta {
7979
input {
8080
Array[File] samplesheets_extended
8181

82-
String docker = "quay.io/broadinstitute/viral-core:2.1.26"
82+
String docker = "quay.io/broadinstitute/viral-core:2.1.28"
8383
}
8484
command <<<
8585
python3 << CODE
@@ -137,7 +137,7 @@ task merge_and_reheader_bams {
137137
File? reheader_table
138138
String out_basename
139139

140-
String docker = "quay.io/broadinstitute/viral-core:2.1.26"
140+
String docker = "quay.io/broadinstitute/viral-core:2.1.28"
141141
}
142142

143143
command {
@@ -197,7 +197,7 @@ task rmdup_ubam {
197197
String method = "mvicuna"
198198

199199
Int? machine_mem_gb
200-
String? docker = "quay.io/broadinstitute/viral-core:2.1.26"
200+
String? docker = "quay.io/broadinstitute/viral-core:2.1.28"
201201
}
202202

203203
parameter_meta {
@@ -251,7 +251,7 @@ task downsample_bams {
251251
Boolean? deduplicateAfter = false
252252

253253
Int? machine_mem_gb
254-
String docker = "quay.io/broadinstitute/viral-core:2.1.26"
254+
String docker = "quay.io/broadinstitute/viral-core:2.1.28"
255255
}
256256

257257
command {
@@ -310,7 +310,7 @@ task FastqToUBAM {
310310
String? platform_name
311311
String? sequencing_center
312312

313-
String docker = "quay.io/broadinstitute/viral-core:2.1.26"
313+
String docker = "quay.io/broadinstitute/viral-core:2.1.28"
314314
}
315315
parameter_meta {
316316
fastq_1: { description: "Unaligned read1 file in fastq format", patterns: ["*.fastq", "*.fastq.gz", "*.fq", "*.fq.gz"] }

‎pipes/WDL/tasks/tasks_reports.wdl‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ task alignment_metrics {
77
File? primers_bed
88

99
Int? machine_mem_gb
10-
String docker = "quay.io/broadinstitute/viral-core:2.1.26"
10+
String docker = "quay.io/broadinstitute/viral-core:2.1.28"
1111
}
1212

1313
String out_basename = basename(aligned_bam, ".bam")
@@ -93,7 +93,7 @@ task plot_coverage {
9393
Boolean bin_large_plots = false
9494
String? binning_summary_statistic = "max" # max or min
9595
96-
String docker = "quay.io/broadinstitute/viral-core:2.1.26"
96+
String docker = "quay.io/broadinstitute/viral-core:2.1.28"
9797
}
9898

9999
command {
@@ -168,7 +168,7 @@ task coverage_report {
168168
Array[File] mapped_bam_idx # optional.. speeds it up if you provide it, otherwise we auto-index
169169
String out_report_name = "coverage_report.txt"
170170

171-
String docker = "quay.io/broadinstitute/viral-core:2.1.26"
171+
String docker = "quay.io/broadinstitute/viral-core:2.1.28"
172172
}
173173

174174
command {
@@ -227,7 +227,7 @@ task fastqc {
227227
input {
228228
File reads_bam
229229

230-
String docker = "quay.io/broadinstitute/viral-core:2.1.26"
230+
String docker = "quay.io/broadinstitute/viral-core:2.1.28"
231231
}
232232

233233
String reads_basename=basename(reads_bam, ".bam")
@@ -260,7 +260,7 @@ task align_and_count {
260260
Int topNHits = 3
261261

262262
Int? machine_mem_gb
263-
String docker = "quay.io/broadinstitute/viral-core:2.1.26"
263+
String docker = "quay.io/broadinstitute/viral-core:2.1.28"
264264
}
265265

266266
String reads_basename=basename(reads_bam, ".bam")
@@ -305,7 +305,7 @@ task align_and_count_summary {
305305

306306
String output_prefix = "count_summary"
307307

308-
String docker = "quay.io/broadinstitute/viral-core:2.1.26"
308+
String docker = "quay.io/broadinstitute/viral-core:2.1.28"
309309
}
310310

311311
command {

0 commit comments

Comments
 (0)