diff --git a/all_of_us/PCA/pca_only_no_labels.wdl b/all_of_us/PCA/pca_only_no_labels.wdl index c3808a14c0..1cbe748a35 100644 --- a/all_of_us/PCA/pca_only_no_labels.wdl +++ b/all_of_us/PCA/pca_only_no_labels.wdl @@ -75,7 +75,7 @@ task ConcatenateChromosomalVcfs { runtime { docker: bcftools_docker memory: "${memory_gb} GB" - cpu: "${cpu}" + cpu: cpu disk: "local-disk ${disk_gb} HDD" preemptible: num_preemptible_attempts } @@ -155,7 +155,7 @@ task create_hw_pca_training { runtime { docker: "hailgenetics/hail:0.2.67" memory: "${mem_gb} GB" - cpu: "${cpu}" + cpu: cpu disks: "local-disk ${disk_gb} ${disk_type}" # large SSD is recommended for increased processing speed } } @@ -244,7 +244,7 @@ task plot_pca { runtime { docker: "faizanbashir/python-datascience:3.6" memory: "${mem_gb} GB" - cpu: "${cpu}" + cpu: cpu disks: "local-disk ${disk_gb} HDD" } } \ No newline at end of file diff --git a/all_of_us/ancestry/determine_hq_sites_intersection.wdl b/all_of_us/ancestry/determine_hq_sites_intersection.wdl index 4fea51e42a..c87fed6b18 100644 --- a/all_of_us/ancestry/determine_hq_sites_intersection.wdl +++ b/all_of_us/ancestry/determine_hq_sites_intersection.wdl @@ -51,7 +51,7 @@ workflow determine_hq_sites_intersection { vcf_intervals = training_vcf_so_bgz, vcf_intervals_idx = training_vcf_so_bgz_idx, intersecting_intervals=intersecting_intervals, - id = i + id = "~{i}" } call intersect_vcfs_as_sites_only { @@ -92,7 +92,7 @@ workflow determine_hq_sites_intersection { sites_only_vcf_idx = merge_sites_only_intersection.merged_vcf_idx, output_name="full_data_sites_filtered", service_account_json=service_account_json, - id = j + id = "~{j}" } } @@ -135,7 +135,7 @@ task sitesOnlyAndHQFilterVcf { String output_filename = basename(vcf) + "." + id + ".sites_only.vcf.gz" String output_filename_idx = output_filename + ".tbi" String has_service_account_file = if (defined(service_account_json)) then 'true' else 'false' - String service_account_basename_pre = if (defined(service_account_json)) then service_account_json else '' + String service_account_basename_pre = select_first([service_account_json, '']) String service_account_basename = basename(service_account_basename_pre) String input_vcf_basename = basename(vcf) String updated_input_vcf = if (defined(service_account_json)) then input_vcf_basename else vcf @@ -178,7 +178,7 @@ task sitesOnlyAndHQFilterVcf { runtime { docker:"us.gcr.io/broad-gatk/gatk:4.2.0.0" memory: "12 GB" - cpu: "4" + cpu: 4 disks: "local-disk 100 HDD" } } @@ -244,7 +244,7 @@ task merge_vcf_bgzs { runtime { docker: "mgibio/bcftools-cwl:1.12" memory: "120 GB" - cpu: "16" + cpu: 16 disks: "local-disk 1500 HDD" bootDiskSizeGb: 1500 } @@ -270,7 +270,7 @@ task filter_by_sites_only { String output_filename = basename(output_name) + "." + id + ".vcf.bgz" String output_filename_idx = output_filename + ".tbi" String has_service_account_file = if (defined(service_account_json)) then 'true' else 'false' - String service_account_basename_pre = if (defined(service_account_json)) then service_account_json else '' + String service_account_basename_pre = select_first([service_account_json, '']) String service_account_basename = basename(service_account_basename_pre) String input_vcf_basename = basename(vcf) String updated_input_vcf = if (defined(service_account_json)) then input_vcf_basename else vcf @@ -303,7 +303,7 @@ task filter_by_sites_only { runtime { docker:"us.gcr.io/broad-gatk/gatk:4.2.0.0" memory: "7 GB" - cpu: "1" + cpu: 1 disks: "local-disk 100 HDD" } } @@ -340,7 +340,7 @@ task intersect_vcfs_as_sites_only { runtime { docker: "us.gcr.io/broad-gatk/gatk:4.2.0.0" memory: "7 GB" - cpu: "1" + cpu: 1 disks: "local-disk 500 HDD" } } \ No newline at end of file diff --git a/all_of_us/ancestry/run_ancestry.wdl b/all_of_us/ancestry/run_ancestry.wdl index f6379b2efe..b8d948ef8b 100644 --- a/all_of_us/ancestry/run_ancestry.wdl +++ b/all_of_us/ancestry/run_ancestry.wdl @@ -161,7 +161,7 @@ task create_hw_pca_training { runtime { docker: "hailgenetics/hail:0.2.67" memory: "123 GB" - cpu: "4" + cpu: 4 disks: "local-disk 500 HDD" } } @@ -280,7 +280,7 @@ task call_ancestry { runtime { docker: "hailgenetics/hail:0.2.67" memory: "240 GB" - cpu: "4" + cpu: 4 disks: "local-disk 700 HDD" } } @@ -377,7 +377,7 @@ task plot_ancestry { runtime { docker: "hailgenetics/hail:0.2.67" memory: "7 GB" - cpu: "4" + cpu: 4 disks: "local-disk 100 HDD" } } \ No newline at end of file diff --git a/all_of_us/ancestry/run_sample_outlier_qc.wdl b/all_of_us/ancestry/run_sample_outlier_qc.wdl index fa162b0e83..229c3ffcb9 100644 --- a/all_of_us/ancestry/run_sample_outlier_qc.wdl +++ b/all_of_us/ancestry/run_sample_outlier_qc.wdl @@ -100,7 +100,7 @@ task join_ancestry_to_stats { runtime { docker: "hailgenetics/hail:0.2.67" memory: "15 GB" - cpu: "2" + cpu: 2 disks: "local-disk 500 HDD" } } @@ -187,7 +187,7 @@ task determine_outlier_qc { runtime { docker: "hailgenetics/hail:0.2.67" memory: "15 GB" - cpu: "4" + cpu: 4 disks: "local-disk 500 HDD" } } diff --git a/all_of_us/ancestry/run_sample_outlier_qc_plotting.wdl b/all_of_us/ancestry/run_sample_outlier_qc_plotting.wdl index 9b84c3d00c..e80b8ccc13 100644 --- a/all_of_us/ancestry/run_sample_outlier_qc_plotting.wdl +++ b/all_of_us/ancestry/run_sample_outlier_qc_plotting.wdl @@ -100,7 +100,7 @@ task join_ancestry_to_demographics { runtime { docker: "hailgenetics/hail:0.2.67" memory: "7 GB" - cpu: "4" + cpu: 4 disks: "local-disk 100 HDD" } } @@ -144,7 +144,7 @@ task plot_first_pcs { runtime { docker: "hailgenetics/hail:0.2.67" memory: "7 GB" - cpu: "4" + cpu: 4 disks: "local-disk 100 HDD" } } @@ -246,7 +246,7 @@ task plot_metrics_and_fitting { runtime { docker: "hailgenetics/hail:0.2.67" memory: "26 GB" - cpu: "4" + cpu: 4 disks: "local-disk 100 HDD" } } \ No newline at end of file diff --git a/all_of_us/pgx/CyriusStarAlleleCalling.wdl b/all_of_us/pgx/CyriusStarAlleleCalling.wdl index 53c9a6a63f..45c401725f 100644 --- a/all_of_us/pgx/CyriusStarAlleleCalling.wdl +++ b/all_of_us/pgx/CyriusStarAlleleCalling.wdl @@ -77,7 +77,7 @@ task PrintReads { docker: "us.gcr.io/broad-gotc-prod/gatk:1.3.0-4.2.6.1-1649964384" preemptible: preemptible_tries memory: "10000 MiB" - cpu: "1" + cpu: 1 disks: "local-disk " + disk_size + " HDD" } output { diff --git a/all_of_us/pgx/StargazerFromJointVCF.wdl b/all_of_us/pgx/StargazerFromJointVCF.wdl index a098874684..a964d82666 100644 --- a/all_of_us/pgx/StargazerFromJointVCF.wdl +++ b/all_of_us/pgx/StargazerFromJointVCF.wdl @@ -121,7 +121,7 @@ task SelectVariants { >>> runtime { memory: "7 GB" - cpu: "1" + cpu: 1 disks: "local-disk " + disk_size + " HDD" docker: "us.gcr.io/broad-gatk/gatk:4.4.0.0" } diff --git a/all_of_us/rna_seq/AggregateSusieWorkflow.wdl b/all_of_us/rna_seq/AggregateSusieWorkflow.wdl index cd60b5be18..00caac5c00 100644 --- a/all_of_us/rna_seq/AggregateSusieWorkflow.wdl +++ b/all_of_us/rna_seq/AggregateSusieWorkflow.wdl @@ -30,7 +30,7 @@ task AggregateSusie{ docker: "ghcr.io/aou-multiomics-analysis/aggregate_susie:main@sha256:ede17b5112eadb765f22cdfbd2a987da96087a1e2f0ad224994c16f1af645443" disks: "local-disk 500 SSD" memory: "~{Memory}GB" - cpu: "~{NumThreads}" + cpu: NumThreads } @@ -77,7 +77,7 @@ task AnnotateSusie { docker: "ghcr.io/aou-multiomics-analysis/aggregate_susie:main@sha256:ede17b5112eadb765f22cdfbd2a987da96087a1e2f0ad224994c16f1af645443" disks: "local-disk 500 SSD" memory: "~{Memory}GB" - cpu: "1" + cpu: 1 } diff --git a/all_of_us/rna_seq/CalculatePhenotypeGroups.wdl b/all_of_us/rna_seq/CalculatePhenotypeGroups.wdl index 4b9c3c11e2..5ed7b00404 100644 --- a/all_of_us/rna_seq/CalculatePhenotypeGroups.wdl +++ b/all_of_us/rna_seq/CalculatePhenotypeGroups.wdl @@ -26,7 +26,7 @@ task PrepareSpliceData { docker: "ghcr.io/aou-multiomics-analysis/prepare_qtl@sha256:b9986a803ad82aee02945e9131f180fc028c73eadb889184e23aff8dabf573fe" memory: "${memory}GB" disks: "local-disk ${disk_space} HDD" - cpu: "${num_threads}" + cpu: num_threads } output { diff --git a/all_of_us/rna_seq/GTEx/aggregate_rsem_results.wdl b/all_of_us/rna_seq/GTEx/aggregate_rsem_results.wdl index 09fd1d6c74..7a1897f5b3 100644 --- a/all_of_us/rna_seq/GTEx/aggregate_rsem_results.wdl +++ b/all_of_us/rna_seq/GTEx/aggregate_rsem_results.wdl @@ -52,7 +52,7 @@ task rsem_aggregate_results { docker: "gcr.io/broad-cga-francois-gtex/gtex_rnaseq:V10" memory: "${memory}GB" disks: "local-disk ${disk_space} HDD" - cpu: "${num_threads}" + cpu: num_threads preemptible: "${num_preempt}" } diff --git a/all_of_us/rna_seq/GTEx/markduplicates.wdl b/all_of_us/rna_seq/GTEx/markduplicates.wdl index 870d92d11b..425b062f13 100644 --- a/all_of_us/rna_seq/GTEx/markduplicates.wdl +++ b/all_of_us/rna_seq/GTEx/markduplicates.wdl @@ -1,16 +1,19 @@ -task markduplicates { +version 1.0 - File input_bam - String prefix - Int? max_records_in_ram - Float? sorting_collection_size_ratio +task markduplicates { + input { + File input_bam + String prefix + Int? max_records_in_ram + Float? sorting_collection_size_ratio + + Float memory + Int disk_space + Int num_threads + Int num_preempt + } - Float memory Int java_memory = floor(memory - 0.5) - Int disk_space - Int num_threads - Int num_preempt - String output_bam = sub(basename(input_bam), "\\.bam$", ".md.bam") command { @@ -32,7 +35,7 @@ task markduplicates { docker: "gcr.io/broad-cga-francois-gtex/gtex_rnaseq:V10" memory: "${memory}GB" disks: "local-disk ${disk_space} HDD" - cpu: "${num_threads}" + cpu: num_threads preemptible: "${num_preempt}" } @@ -44,4 +47,4 @@ task markduplicates { workflow markduplicates_workflow { call markduplicates -} \ No newline at end of file +} diff --git a/all_of_us/rna_seq/GTEx/remove_IDS_reads.wdl b/all_of_us/rna_seq/GTEx/remove_IDS_reads.wdl index 28a3f54b93..4291b174e5 100644 --- a/all_of_us/rna_seq/GTEx/remove_IDS_reads.wdl +++ b/all_of_us/rna_seq/GTEx/remove_IDS_reads.wdl @@ -1,12 +1,15 @@ -task remove_IDS_reads { - - File transcriptome_bam - String prefix +version 1.0 - Int memory - Int disk_space - Int num_threads - Int num_preempt +task remove_IDS_reads { + input { + File transcriptome_bam + String prefix + + Int memory + Int disk_space + Int num_threads + Int num_preempt + } command { set -euo pipefail @@ -22,7 +25,7 @@ task remove_IDS_reads { docker: "gcr.io/broad-cga-francois-gtex/gtex_rnaseq:V10" memory: "${memory}GB" disks: "local-disk ${disk_space} HDD" - cpu: "${num_threads}" + cpu: num_threads preemptible: "${num_preempt}" } @@ -34,4 +37,4 @@ task remove_IDS_reads { workflow rsem_preprocessing_workflow { call remove_IDS_reads -} \ No newline at end of file +} diff --git a/all_of_us/rna_seq/GTEx/rnaseq_aou.wdl b/all_of_us/rna_seq/GTEx/rnaseq_aou.wdl index b4a434a5f8..8d3fd52ec2 100644 --- a/all_of_us/rna_seq/GTEx/rnaseq_aou.wdl +++ b/all_of_us/rna_seq/GTEx/rnaseq_aou.wdl @@ -1,3 +1,5 @@ +version 1.0 + # import "https://api.firecloud.org/ga4gh/v1/tools/broad_drc_aou_aux:samToFastq/versions/1/plain-WDL/descriptor" as samtofastq_wdl # import "https://api.firecloud.org/ga4gh/v1/tools/gtex_v10_pg:star_align_v10pg/versions/12/plain-WDL/descriptor" as star_align_wdl # import "https://api.firecloud.org/ga4gh/v1/tools/gtex_v10_pg:pre_RSEM_processing_v10pg/versions/6/plain-WDL/descriptor" as prersem_wdl @@ -14,8 +16,10 @@ import "./rnaseqc2.wdl" as rnaseqc_wdl workflow rnaseq_pipeline_bam_workflow { + input { + String prefix + } - String prefix String pipeline_version = "aou_9.0.1" call samtofastq_wdl.samtofastq { @@ -39,4 +43,4 @@ workflow rnaseq_pipeline_bam_workflow { call rnaseqc_wdl.rnaseqc2 { input: bam_file=markduplicates.bam_file, sample_id=prefix } -} \ No newline at end of file +} diff --git a/all_of_us/rna_seq/GTEx/rnaseqc2.wdl b/all_of_us/rna_seq/GTEx/rnaseqc2.wdl index 68538940d0..ed257b676e 100644 --- a/all_of_us/rna_seq/GTEx/rnaseqc2.wdl +++ b/all_of_us/rna_seq/GTEx/rnaseqc2.wdl @@ -1,18 +1,21 @@ -task rnaseqc2 { - - File bam_file - File genes_gtf - String sample_id - String? strandedness - File? intervals_bed - File? reference_fasta - File? reference_fasta_index - String? flags +version 1.0 - Int memory - Int disk_space - Int num_threads - Int num_preempt +task rnaseqc2 { + input { + File bam_file + File genes_gtf + String sample_id + String? strandedness + File? intervals_bed + File? reference_fasta + File? reference_fasta_index + String? flags + + Int memory + Int disk_space + Int num_threads + Int num_preempt + } command { set -euo pipefail @@ -38,7 +41,7 @@ task rnaseqc2 { docker: "gcr.io/broad-cga-francois-gtex/gtex_rnaseq:V10" memory: "${memory}GB" disks: "local-disk ${disk_space} HDD" - cpu: "${num_threads}" + cpu: num_threads preemptible: "${num_preempt}" } @@ -50,4 +53,4 @@ task rnaseqc2 { workflow rnaseqc2_workflow { call rnaseqc2 -} \ No newline at end of file +} diff --git a/all_of_us/rna_seq/GTEx/rsem.wdl b/all_of_us/rna_seq/GTEx/rsem.wdl index bf09627c6e..b5c3f44a89 100644 --- a/all_of_us/rna_seq/GTEx/rsem.wdl +++ b/all_of_us/rna_seq/GTEx/rsem.wdl @@ -1,18 +1,21 @@ -task rsem { - - File transcriptome_bam - File rsem_reference - String prefix - - Int memory - Int disk_space - Int num_threads - Int num_preempt +version 1.0 - Int? max_frag_len - String? estimate_rspd - String? is_stranded - String? paired_end +task rsem { + input { + File transcriptome_bam + File rsem_reference + String prefix + + Int memory + Int disk_space + Int num_threads + Int num_preempt + + Int? max_frag_len + String? estimate_rspd + String? is_stranded + String? paired_end + } command { set -euo pipefail @@ -38,7 +41,7 @@ task rsem { docker: "gcr.io/broad-cga-francois-gtex/gtex_rnaseq:V10" memory: "${memory}GB" disks: "local-disk ${disk_space} HDD" - cpu: "${num_threads}" + cpu: num_threads preemptible: "${num_preempt}" } @@ -50,4 +53,4 @@ task rsem { workflow rsem_workflow { call rsem -} \ No newline at end of file +} diff --git a/all_of_us/rna_seq/GTEx/samtofastq.wdl b/all_of_us/rna_seq/GTEx/samtofastq.wdl index fa4d74f757..f7f755226b 100644 --- a/all_of_us/rna_seq/GTEx/samtofastq.wdl +++ b/all_of_us/rna_seq/GTEx/samtofastq.wdl @@ -1,15 +1,19 @@ -task samtofastq { +version 1.0 - File input_bam_cram - String prefix - File? reference_fasta - File? reference_fasta_index +task samtofastq { + input { + File input_bam_cram + String prefix + File? reference_fasta + File? reference_fasta_index + + Float memory + Int disk_space + Int num_threads + Int num_preempt + } - Float memory Int java_memory = floor(memory - 0.5) - Int disk_space - Int num_threads - Int num_preempt command { set -euo pipefail @@ -34,7 +38,7 @@ task samtofastq { docker: "gcr.io/broad-cga-francois-gtex/gtex_rnaseq:V10" memory: "${memory}GB" disks: "local-disk ${disk_space} HDD" - cpu: "${num_threads}" + cpu: num_threads preemptible: "${num_preempt}" } @@ -46,4 +50,4 @@ task samtofastq { workflow samtofastq_workflow { call samtofastq -} \ No newline at end of file +} diff --git a/all_of_us/rna_seq/GTEx/star.wdl b/all_of_us/rna_seq/GTEx/star.wdl index 8368c48197..113380bf78 100644 --- a/all_of_us/rna_seq/GTEx/star.wdl +++ b/all_of_us/rna_seq/GTEx/star.wdl @@ -1,46 +1,49 @@ +version 1.0 + # This wdl comes from https://github.com/broadinstitute/gtex-pipeline/blob/v10_pg_update/rnaseq/star.wdl task star { + input { + File fastq1 + File? fastq2 + String prefix + File star_index - File fastq1 - File? fastq2 - String prefix - File star_index - - # STAR options - Int? outFilterMultimapNmax - Int? alignSJoverhangMin - Int? alignSJDBoverhangMin - Int? outFilterMismatchNmax - Float? outFilterMismatchNoverLmax - Int? alignIntronMin - Int? alignIntronMax - Int? alignMatesGapMax - String? outFilterType - Float? outFilterScoreMinOverLread - Float? outFilterMatchNminOverLread - Int? limitSjdbInsertNsj - String? outSAMstrandField - String? outFilterIntronMotifs - String? alignSoftClipAtReferenceEnds - String? quantMode - String? outSAMattrRGline - String? outSAMattributes - File? varVCFfile - String? waspOutputMode - Int? chimSegmentMin - Int? chimJunctionOverhangMin - String? chimOutType - Int? chimMainSegmentMultNmax - Int? chimOutJunctionFormat - File? sjdbFileChrStartEnd - String? quantTranscriptomeSAMoutput - Int? winAnchorMultimapNmax - String? genomeTransformOutput + # STAR options + Int? outFilterMultimapNmax + Int? alignSJoverhangMin + Int? alignSJDBoverhangMin + Int? outFilterMismatchNmax + Float? outFilterMismatchNoverLmax + Int? alignIntronMin + Int? alignIntronMax + Int? alignMatesGapMax + String? outFilterType + Float? outFilterScoreMinOverLread + Float? outFilterMatchNminOverLread + Int? limitSjdbInsertNsj + String? outSAMstrandField + String? outFilterIntronMotifs + String? alignSoftClipAtReferenceEnds + String? quantMode + String? outSAMattrRGline + String? outSAMattributes + File? varVCFfile + String? waspOutputMode + Int? chimSegmentMin + Int? chimJunctionOverhangMin + String? chimOutType + Int? chimMainSegmentMultNmax + Int? chimOutJunctionFormat + File? sjdbFileChrStartEnd + String? quantTranscriptomeSAMoutput + Int? winAnchorMultimapNmax + String? genomeTransformOutput - Int memory - Int disk_space - Int num_threads - Int num_preempt + Int memory + Int disk_space + Int num_threads + Int num_preempt + } command { set -euo pipefail @@ -132,7 +135,7 @@ task star { docker: "gcr.io/broad-cga-francois-gtex/gtex_rnaseq@sha256:80c2db3cec3c08630237665e2d2f044f065022e0bbf7a62d0765f51f811818e2" memory: "${memory}GB" disks: "local-disk ${disk_space} HDD" - cpu: "${num_threads}" + cpu: num_threads preemptible: "${num_preempt}" } @@ -144,4 +147,4 @@ task star { workflow star_workflow { call star -} \ No newline at end of file +} diff --git a/all_of_us/rna_seq/leafcutter_bam_to_junc.wdl b/all_of_us/rna_seq/leafcutter_bam_to_junc.wdl index 8c3e7be133..058d07c970 100644 --- a/all_of_us/rna_seq/leafcutter_bam_to_junc.wdl +++ b/all_of_us/rna_seq/leafcutter_bam_to_junc.wdl @@ -27,7 +27,7 @@ task leafcutter_bam_to_junc { docker: "gcr.io/broad-cga-francois-gtex/leafcutter:latest" memory: "~{memory}GB" disks: "local-disk ~{disk_space} HDD" - cpu: "~{num_threads}" + cpu: num_threads preemptible: "~{num_preempt}" } diff --git a/all_of_us/rna_seq/leafcutter_cluster.wdl b/all_of_us/rna_seq/leafcutter_cluster.wdl index 1ca2fbe0f6..f4d38ffb49 100644 --- a/all_of_us/rna_seq/leafcutter_cluster.wdl +++ b/all_of_us/rna_seq/leafcutter_cluster.wdl @@ -62,7 +62,7 @@ task leafcutter_cluster { docker: "us.gcr.io/broad-gotc-prod/leafcutter:1.0.0" memory: "~{memory}GB" disks: "local-disk ~{disk_space} HDD" - cpu: "~{num_threads}" + cpu: num_threads preemptible: "~{num_preempt}" } diff --git a/all_of_us/rna_seq/prepare_QTL/calculateAF.wdl b/all_of_us/rna_seq/prepare_QTL/calculateAF.wdl index 786372007d..e7138f15b2 100644 --- a/all_of_us/rna_seq/prepare_QTL/calculateAF.wdl +++ b/all_of_us/rna_seq/prepare_QTL/calculateAF.wdl @@ -59,7 +59,7 @@ workflow CaclulateAF { docker: "quay.io/biocontainers/plink2:2.0.0a.6.9--h9948957_0" memory: "~{memory}GB" disks: "local-disk ~{disk_space} HDD" - cpu: "~{num_threads}" + cpu: num_threads } diff --git a/all_of_us/rna_seq/prepare_QTL/calculateGenotypeDosage.wdl b/all_of_us/rna_seq/prepare_QTL/calculateGenotypeDosage.wdl index 58b9abd405..48d19f533b 100644 --- a/all_of_us/rna_seq/prepare_QTL/calculateGenotypeDosage.wdl +++ b/all_of_us/rna_seq/prepare_QTL/calculateGenotypeDosage.wdl @@ -50,7 +50,7 @@ task BcftoolsDosage { runtime { docker: "quay.io/eqtlcatalogue/susie-finemapping:v20.08.1" memory: "32G" - cpu: "${threads}" + cpu: threads disks: "local-disk 500 SSD" } diff --git a/all_of_us/rna_seq/prepare_QTL/calculate_phenotypePCs.wdl b/all_of_us/rna_seq/prepare_QTL/calculate_phenotypePCs.wdl index 5c36c19bb7..aa26297c41 100644 --- a/all_of_us/rna_seq/prepare_QTL/calculate_phenotypePCs.wdl +++ b/all_of_us/rna_seq/prepare_QTL/calculate_phenotypePCs.wdl @@ -18,7 +18,7 @@ task ComputePCs{ docker: "us.gcr.io/broad-gotc-prod/aou_rna_prepareqtl:0.0.1" memory: "~{memory}GB" disks: "local-disk ~{disk_space} HDD" - cpu: "~{num_threads}" + cpu: num_threads } output { diff --git a/all_of_us/rna_seq/prepare_QTL/prepare_eQTL.wdl b/all_of_us/rna_seq/prepare_QTL/prepare_eQTL.wdl index 9ad3f388de..00acbbab82 100644 --- a/all_of_us/rna_seq/prepare_QTL/prepare_eQTL.wdl +++ b/all_of_us/rna_seq/prepare_QTL/prepare_eQTL.wdl @@ -30,7 +30,7 @@ task eqtl_prepare_expression { docker: "us.gcr.io/broad-gotc-prod/aou_rna_prepareqtl:0.0.1" memory: "${memory}GB" disks: "local-disk ${disk_space} HDD" - cpu: "${num_threads}" + cpu: num_threads } output { diff --git a/all_of_us/rna_seq/prepare_QTL/prepare_sQTL.wdl b/all_of_us/rna_seq/prepare_QTL/prepare_sQTL.wdl index f733e92103..9eab4bb82b 100644 --- a/all_of_us/rna_seq/prepare_QTL/prepare_sQTL.wdl +++ b/all_of_us/rna_seq/prepare_QTL/prepare_sQTL.wdl @@ -22,7 +22,7 @@ task PrepareSpliceData { docker: "us.gcr.io/broad-gotc-prod/aou_rna_prepareqtl:0.0.1" memory: "${memory}GB" disks: "local-disk ${disk_space} HDD" - cpu: "${num_threads}" + cpu: num_threads } output { diff --git a/all_of_us/rna_seq/susieR_workflow.wdl b/all_of_us/rna_seq/susieR_workflow.wdl index 207ecd1de4..9b91bf7166 100644 --- a/all_of_us/rna_seq/susieR_workflow.wdl +++ b/all_of_us/rna_seq/susieR_workflow.wdl @@ -73,7 +73,7 @@ task PrepInputs { disks: "local-disk 500 SSD" preemptible: "${NumPrempt}" memory: "2GB" - cpu: "1" + cpu: 1 } output { @@ -121,7 +121,7 @@ task susieR { disks: "local-disk 500 SSD" bootDiskSizeGb: 25 preemptible: "${NumPrempt}" - cpu: "1" + cpu: 1 } output { diff --git a/all_of_us/rna_seq/tensorQTL_cis_permutations/tensorqtl_cis_permutations.wdl b/all_of_us/rna_seq/tensorQTL_cis_permutations/tensorqtl_cis_permutations.wdl index 07a05cd19a..447c993c71 100644 --- a/all_of_us/rna_seq/tensorQTL_cis_permutations/tensorqtl_cis_permutations.wdl +++ b/all_of_us/rna_seq/tensorQTL_cis_permutations/tensorqtl_cis_permutations.wdl @@ -50,7 +50,7 @@ task tensorqtl_cis_permutations { memory: "~{memory}GB" disks: "local-disk ~{disk_space} HDD" bootDiskSizeGb: 25 - cpu: "~{num_threads}" + cpu: num_threads preemptible: "~{num_preempt}" gpuType: "nvidia-tesla-p100" gpuCount: "~{num_gpus}" diff --git a/pipelines/wdl/build_indices/BuildIndices.wdl b/pipelines/wdl/build_indices/BuildIndices.wdl index 31ec85d183..149e73d1c8 100644 --- a/pipelines/wdl/build_indices/BuildIndices.wdl +++ b/pipelines/wdl/build_indices/BuildIndices.wdl @@ -362,7 +362,7 @@ task CalculateChromosomeSizes { docker: "us.gcr.io/broad-gotc-prod/samtools:1.0.0-1.11-1624651616" preemptible: 3 memory: "3 GiB" - cpu: "1" + cpu: 1 disks: "local-disk 50 HDD" } output { @@ -536,7 +536,7 @@ task BuildStarSingleNucleus { memory: "64 GiB" disks: "local-disk ${disk} HDD" disk: disk + " GB" # TES - cpu:"16" + cpu: 16 } } @@ -652,7 +652,7 @@ String reference_name = "bwa-mem2-2.2.1-~{organism}-~{genome_source}-build-~{gen memory: "96GB" disks: "local-disk 100 HDD" disk: "100 GB" # TES - cpu: "4" + cpu: 4 } output { @@ -805,7 +805,7 @@ task RecordMetadata { docker: "ubuntu:20.04" memory: "5 GiB" disks: "local-disk 100 HDD" - cpu: "1" + cpu: 1 } } diff --git a/pipelines/wdl/glimpse/low_pass_imputation/Glimpse2LowPassImputation.wdl b/pipelines/wdl/glimpse/low_pass_imputation/Glimpse2LowPassImputation.wdl index b11f66d70b..548ae502cf 100644 --- a/pipelines/wdl/glimpse/low_pass_imputation/Glimpse2LowPassImputation.wdl +++ b/pipelines/wdl/glimpse/low_pass_imputation/Glimpse2LowPassImputation.wdl @@ -288,7 +288,7 @@ task ComputeShardsAndMemoryPerShard { output { Array[String] reference_chunk_file_paths = read_lines("reference_shard_file_paths.tsv") - Array[Int] mem_gb_per_chunk = read_lines("memory_per_chunk.tsv") + Array[String] mem_gb_per_chunk = read_lines("memory_per_chunk.tsv") } } diff --git a/tasks/wdl/Alignment.wdl b/tasks/wdl/Alignment.wdl index f3c150bbfc..0e35c01e11 100644 --- a/tasks/wdl/Alignment.wdl +++ b/tasks/wdl/Alignment.wdl @@ -116,7 +116,7 @@ task SamToFastqAndBwaMemAndMba { docker: "us.gcr.io/broad-gotc-prod/samtools-picard-bwa:1.0.2-0.7.15-2.26.10-1643840748" preemptible: preemptible_tries memory: "14 GiB" - cpu: "16" + cpu: 16 disks: "local-disk " + disk_size + " HDD" } output { diff --git a/tasks/wdl/BamProcessing.wdl b/tasks/wdl/BamProcessing.wdl index cf4ff4d4e2..2f99d60969 100644 --- a/tasks/wdl/BamProcessing.wdl +++ b/tasks/wdl/BamProcessing.wdl @@ -50,7 +50,7 @@ task SortSam { runtime { docker: docker disks: "local-disk " + disk_size + " HDD" - cpu: "1" + cpu: 1 memory: "${machine_mem_mb} MiB" preemptible: preemptible_tries } diff --git a/tasks/wdl/CopyFilesFromCloudToCloud.wdl b/tasks/wdl/CopyFilesFromCloudToCloud.wdl index aa52c83716..b98806e4db 100644 --- a/tasks/wdl/CopyFilesFromCloudToCloud.wdl +++ b/tasks/wdl/CopyFilesFromCloudToCloud.wdl @@ -68,7 +68,7 @@ task CopyFilesFromCloudToCloud { # We don't have to use an external IP. runtime { memory: "2 GiB" - cpu: "1" + cpu: 1 disks: "local-disk 20 HDD" docker: "us.gcr.io/broad-gotc-prod/dsde-toolbox:stable_04-18-2022" preemptible: 3 diff --git a/tasks/wdl/Funcotator.wdl b/tasks/wdl/Funcotator.wdl index 73b8621d13..12e0f9bcb6 100644 --- a/tasks/wdl/Funcotator.wdl +++ b/tasks/wdl/Funcotator.wdl @@ -160,9 +160,9 @@ task Funcotate { -O ~{output_vcf} \ ~{true="-L" false="" defined(interval_list)} ~{default="" interval_list} \ ~{true="--transcript-selection-mode" false="" defined(transcript_selection_mode)} ~{default="" transcript_selection_mode} \ - ~{true="--transcript-list" false="" defined(transcript_selection_list)} ~{default="" sep=" --transcript-list " transcript_selection_list} \ - ~{true="--annotation-default" false="" defined(annotation_defaults)} ~{default="" sep=" --annotation-default " annotation_defaults} \ - ~{true="--annotation-override" false="" defined(annotation_overrides)} ~{default="" sep=" --annotation-override " annotation_overrides} \ + ~{sep=" " prefix("--transcript-list ", select_first([transcript_selection_list, []]))} \ + ~{sep=" " prefix("--annotation-default ", select_first([annotation_defaults, []]))} \ + ~{sep=" " prefix("--annotation-override ", select_first([annotation_overrides, []]))} \ ~{true="--remove-filtered-variants" false="" filter_funcotations} \ ~{default="" extra_args} >>> diff --git a/tasks/wdl/GermlineVariantDiscovery.wdl b/tasks/wdl/GermlineVariantDiscovery.wdl index 4ab5964cdb..c70dcf980f 100644 --- a/tasks/wdl/GermlineVariantDiscovery.wdl +++ b/tasks/wdl/GermlineVariantDiscovery.wdl @@ -64,7 +64,7 @@ task HaplotypeCaller_GATK35_GVCF { --max_alternate_alleles 3 \ -variant_index_parameter 128000 \ -variant_index_type LINEAR \ - -contamination ~{default=0 contamination} \ + -contamination ~{default="0" contamination} \ --read_filter OverclippedRead } @@ -72,7 +72,7 @@ task HaplotypeCaller_GATK35_GVCF { docker: docker preemptible: preemptible_tries memory: "10000 MiB" - cpu: "1" + cpu: 1 disks: "local-disk " + disk_size + " HDD" } output { @@ -147,7 +147,7 @@ task HaplotypeCaller_GATK4_VCF { -I ~{input_bam} \ -L ~{interval_list} \ -O ~{output_file_name} \ - -contamination ~{default=0 contamination} \ + -contamination ~{default="0" contamination} \ -G StandardAnnotation -G StandardHCAnnotation ~{true="-G AS_StandardAnnotation" false="" make_gvcf} \ ~{true="--dragen-mode" false="" run_dragen_mode_variant_calling} \ ~{false="--disable-spanning-event-genotyping" true="" use_spanning_event_genotyping} \ @@ -164,7 +164,7 @@ task HaplotypeCaller_GATK4_VCF { docker: gatk_docker preemptible: preemptible_tries memory: "~{memory_size_mb} MiB" - cpu: "2" + cpu: 2 bootDiskSizeGb: 15 disks: "local-disk " + disk_size + " HDD" } @@ -380,7 +380,7 @@ task CNNScoreVariants { docker: gatk_docker preemptible: preemptible_tries memory: "15000 MiB" - cpu: "2" + cpu: 2 bootDiskSizeGb: 15 disks: "local-disk " + disk_size + " HDD" } @@ -436,7 +436,7 @@ task FilterVariantTranches { runtime { memory: "7000 MiB" - cpu: "2" + cpu: 2 bootDiskSizeGb: 15 disks: "local-disk " + disk_size + " HDD" preemptible: preemptible_tries diff --git a/tasks/wdl/IlluminaGenotypingArrayTasks.wdl b/tasks/wdl/IlluminaGenotypingArrayTasks.wdl index ab362104a9..eb1f2e9261 100644 --- a/tasks/wdl/IlluminaGenotypingArrayTasks.wdl +++ b/tasks/wdl/IlluminaGenotypingArrayTasks.wdl @@ -538,7 +538,7 @@ task MergePedIntoVcf { runtime { docker: "us.gcr.io/broad-gotc-prod/picard-cloud:2.26.11" memory: "3500 MiB" - cpu: "1" + cpu: 1 disks: "local-disk " + disk_size + " HDD" preemptible: preemptible_tries } diff --git a/tasks/wdl/JointGenotypingTasks.wdl b/tasks/wdl/JointGenotypingTasks.wdl index 9e6378a9b5..c73f403b20 100644 --- a/tasks/wdl/JointGenotypingTasks.wdl +++ b/tasks/wdl/JointGenotypingTasks.wdl @@ -301,7 +301,7 @@ task HardFilterAndMakeSitesOnlyVcf { runtime { memory: "~{machine_mem_mb} MiB" - cpu: "1" + cpu: 1 bootDiskSizeGb: 15 disks: "local-disk " + disk_size_gb + " HDD" preemptible: 1 @@ -363,7 +363,7 @@ task IndelsVariantRecalibrator { runtime { memory: "~{machine_mem_mb} MiB" - cpu: "2" + cpu: 2 bootDiskSizeGb: 15 disks: "local-disk " + disk_size_gb + " HDD" preemptible: 1 @@ -431,7 +431,7 @@ task SNPsVariantRecalibratorCreateModel { runtime { memory: "~{machine_mem_mb} MiB" - cpu: "2" + cpu: 2 bootDiskSizeGb: 15 disks: "local-disk " + disk_size_gb + " HDD" preemptible: 1 @@ -578,7 +578,7 @@ task GatherTranches { runtime { memory: "~{machine_mem_mb} MiB" - cpu: "2" + cpu: 2 bootDiskSizeGb: 15 disks: "local-disk " + disk_size_gb + " HDD" preemptible: 1 @@ -638,7 +638,7 @@ task ApplyRecalibration { runtime { memory: "~{machine_mem_mb} MiB" - cpu: "1" + cpu: 1 bootDiskSizeGb: 15 disks: "local-disk " + disk_size_gb + " HDD" preemptible: 1 @@ -685,7 +685,7 @@ task GatherVcfs { runtime { memory: "~{machine_mem_mb} MiB" - cpu: "1" + cpu: 1 bootDiskSizeGb: 15 disks: "local-disk " + disk_size_gb + " HDD" preemptible: 1 @@ -856,7 +856,7 @@ task GatherVariantCallingMetrics { runtime { memory: "~{machine_mem_mb} MiB" - cpu: "1" + cpu: 1 bootDiskSizeGb: 15 disks: "local-disk " + disk_size_gb + " HDD" preemptible: 1 @@ -897,7 +897,7 @@ task CrossCheckFingerprint { Int cpu = if num_gvcfs < 32 then num_gvcfs else 32 # Compute memory to use based on the CPU count, following the pattern of # 3.75GiB / cpu used by GCP's pricing: https://cloud.google.com/compute/pricing - Int memory = if defined(machine_mem_mb) then machine_mem_mb else round(cpu * 3.75 * 1024) + Int memory = select_first([machine_mem_mb, round(cpu * 3.75 * 1024)]) Int java_mem = memory - 512 String output_name = output_base_name + ".fingerprintcheck" @@ -1057,10 +1057,18 @@ task GetFingerprintingIntervalIndices { else touch indices.out fi + + python3 -c " +with open('indices.out') as f: + indices = [int(line.strip()) for line in f if line.strip()] +import json +with open('indices.json', 'w') as out: + json.dump(indices, out) +" >>> output { - Array[String] indices_to_fingerprint = read_lines("indices.out") + Array[Int] indices_to_fingerprint = read_json("indices.json") File all_sorted_interval_list = "all.sorted.interval_list" File all_interval_list = "all.interval_list" File hdb_interval_list = "hdb.interval_list" diff --git a/tasks/wdl/Qc.wdl b/tasks/wdl/Qc.wdl index 36e4708dfb..3d2492e34d 100644 --- a/tasks/wdl/Qc.wdl +++ b/tasks/wdl/Qc.wdl @@ -431,9 +431,9 @@ task ValidateSamFile { OUTPUT=~{report_filename} \ REFERENCE_SEQUENCE=~{ref_fasta} \ ~{"MAX_OUTPUT=" + max_output} \ - IGNORE=~{default="null" sep=" IGNORE=" ignore} \ + ~{sep=" " prefix("IGNORE=", select_first([ignore, ["null"]]))} \ MODE=VERBOSE \ - ~{default='SKIP_MATE_VALIDATION=false' true='SKIP_MATE_VALIDATION=true' false='SKIP_MATE_VALIDATION=false' is_outlier_data} \ + ~{true='SKIP_MATE_VALIDATION=true' false='SKIP_MATE_VALIDATION=false' select_first([is_outlier_data, false])} \ IS_BISULFITE_SEQUENCED=false } runtime { @@ -503,7 +503,7 @@ task CollectRawWgsMetrics { Int disk_size = ceil(size(input_bam, "GiB") + ref_size) + additional_disk Int memory_size = ceil((if (disk_size < 110) then 5 else 7) * memory_multiplier) - String java_memory_size = (memory_size - 1) * 1000 + Int java_memory_size = (memory_size - 1) * 1000 command { java -Xms~{java_memory_size}m -jar /usr/picard/picard.jar \ diff --git a/tasks/wdl/StarAlign.wdl b/tasks/wdl/StarAlign.wdl index 3a1454d509..77a1e721da 100644 --- a/tasks/wdl/StarAlign.wdl +++ b/tasks/wdl/StarAlign.wdl @@ -876,6 +876,6 @@ task STARGenomeRefVersion { memory: "2 GiB" disks: "local-disk ${disk} HDD" disk: disk + " GB" # TES - cpu:"1" + cpu: 1 } } \ No newline at end of file diff --git a/tasks/wdl/TerraCopyFilesFromCloudToCloud.wdl b/tasks/wdl/TerraCopyFilesFromCloudToCloud.wdl index 1860151bff..9c86095249 100644 --- a/tasks/wdl/TerraCopyFilesFromCloudToCloud.wdl +++ b/tasks/wdl/TerraCopyFilesFromCloudToCloud.wdl @@ -41,7 +41,7 @@ task TerraCopyFilesFromCloudToCloud { runtime { memory: "16 GiB" - cpu: "1" + cpu: 1 disks: "local-disk 32 HDD" docker: "gcr.io/google.com/cloudsdktool/google-cloud-cli:499.0.0-slim" preemptible: 3 diff --git a/tasks/wdl/UltimaGenomicsWholeGenomeGermlineTasks.wdl b/tasks/wdl/UltimaGenomicsWholeGenomeGermlineTasks.wdl index d0e5ca85fc..3a3072e596 100644 --- a/tasks/wdl/UltimaGenomicsWholeGenomeGermlineTasks.wdl +++ b/tasks/wdl/UltimaGenomicsWholeGenomeGermlineTasks.wdl @@ -811,7 +811,7 @@ task CollectWgsMetrics { USE_FAST_ALGORITHM=false \ COUNT_UNPAIRED=true \ COVERAGE_CAP=12500 \ - READ_LENGTH=~{default=250 read_length} + READ_LENGTH=~{default="250" read_length} } runtime { @@ -863,7 +863,7 @@ task CollectRawWgsMetrics { OUTPUT=~{metrics_filename} \ COUNT_UNPAIRED=true \ USE_FAST_ALGORITHM=false \ - READ_LENGTH=~{default=250 read_length} + READ_LENGTH=~{default="250" read_length} } runtime { diff --git a/tasks/wdl/Utilities.wdl b/tasks/wdl/Utilities.wdl index e6a1aeec17..63bb6e7342 100644 --- a/tasks/wdl/Utilities.wdl +++ b/tasks/wdl/Utilities.wdl @@ -149,7 +149,7 @@ task ConvertToCram { docker: "us.gcr.io/broad-gotc-prod/samtools:1.0.0-1.11-1624651616" preemptible: preemptible_tries memory: "3 GiB" - cpu: "1" + cpu: 1 disks: "local-disk " + disk_size + " HDD" } output { @@ -180,7 +180,7 @@ task ConvertToBam { docker: "us.gcr.io/broad-gotc-prod/samtools:1.0.0-1.11-1624651616" preemptible: 3 memory: "3 GiB" - cpu: "1" + cpu: 1 disks: "local-disk 200 HDD" } output { diff --git a/tasks/wdl/accessory_workflows/build_bwa_reference/bwa-mk-index.wdl b/tasks/wdl/accessory_workflows/build_bwa_reference/bwa-mk-index.wdl index 81df8d68dd..790028eab1 100644 --- a/tasks/wdl/accessory_workflows/build_bwa_reference/bwa-mk-index.wdl +++ b/tasks/wdl/accessory_workflows/build_bwa_reference/bwa-mk-index.wdl @@ -45,7 +45,7 @@ task BuildBWAreference { memory: "96GB" disks: "local-disk 100 HDD" disk: "100 GB" # TES - cpu: "4" + cpu: 4 } output { diff --git a/verification/VerifyMetrics.wdl b/verification/VerifyMetrics.wdl index 3baa400998..b0cc4ee9d9 100644 --- a/verification/VerifyMetrics.wdl +++ b/verification/VerifyMetrics.wdl @@ -84,7 +84,7 @@ task CompareMetricFiles { --INPUT ~{file1} \ --INPUT ~{file2} \ --OUTPUT ~{output_file} \ - ~{true="--METRICS_TO_IGNORE" false="" length(metrics_to_ignore) > 0} ~{default="" sep=" --METRICS_TO_IGNORE " metrics_to_ignore} \ + ~{sep=" " prefix("--METRICS_TO_IGNORE ", metrics_to_ignore)} \ ~{sep=" " extra_args} >>> diff --git a/verification/VerifyUltimaGenomicsWholeGenomeCramOnly.wdl b/verification/VerifyUltimaGenomicsWholeGenomeCramOnly.wdl index c0a3cfaedd..3956de1619 100644 --- a/verification/VerifyUltimaGenomicsWholeGenomeCramOnly.wdl +++ b/verification/VerifyUltimaGenomicsWholeGenomeCramOnly.wdl @@ -99,7 +99,7 @@ task CompareOldMetricFiles { --INPUT ~{file1} \ --INPUT ~{file2} \ --OUTPUT ~{output_file} \ - ~{true="--METRICS_TO_IGNORE" false="" length(metrics_to_ignore) > 0} ~{default="" sep=" --METRICS_TO_IGNORE " metrics_to_ignore} + ~{sep=" " prefix("--METRICS_TO_IGNORE ", metrics_to_ignore)} >>> runtime { diff --git a/verification/VerifyUltimaGenomicsWholeGenomeGermline.wdl b/verification/VerifyUltimaGenomicsWholeGenomeGermline.wdl index ec2f2bcbea..95aa575c54 100755 --- a/verification/VerifyUltimaGenomicsWholeGenomeGermline.wdl +++ b/verification/VerifyUltimaGenomicsWholeGenomeGermline.wdl @@ -184,7 +184,7 @@ task CompareOldMetricFiles { --INPUT ~{file1} \ --INPUT ~{file2} \ --OUTPUT ~{output_file} \ - ~{true="--METRICS_TO_IGNORE" false="" length(metrics_to_ignore) > 0} ~{default="" sep=" --METRICS_TO_IGNORE " metrics_to_ignore} + ~{sep=" " prefix("--METRICS_TO_IGNORE ", metrics_to_ignore)} >>> runtime { diff --git a/verification/test-wdls/TestArrayImputationQC.wdl b/verification/test-wdls/TestArrayImputationQC.wdl index 4bc162b93f..064c6c74da 100644 --- a/verification/test-wdls/TestArrayImputationQC.wdl +++ b/verification/test-wdls/TestArrayImputationQC.wdl @@ -49,7 +49,7 @@ workflow TestArrayImputationQC { call WriteMapToTsv { input: input_map = { - "passes_qc": InputQC.passes_qc, + "passes_qc": "~{InputQC.passes_qc}", "qc_messages": InputQC.qc_messages } }