Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ verification/test-wdls/__pycache__/

# VSCode project settings
.vscode
warp.code-workspace

# python virtual environments
venv/
Expand Down
4 changes: 2 additions & 2 deletions all_of_us/mitochondria/merge/mitochondria_merge.changelog.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@

* Updated coverage HDF5 dtype from inferred uint16 to a fixed uint32 default to prevent silent truncation of high-coverage positions; added an explicit overflow guard that raises an error if any coverage value exceeds the dtype maximum

aou_9.0.0
# aou_9.0.0
2026-03-23 (Date of Last Commit)

* Version of the pipeline use to process AoU v9 data


aou_9_beta
# aou_9_beta
2025-10-31 (Date of Last Commit)

* Added support for optional subsetting of inputs using a Terra data table TSV
Expand Down
14 changes: 7 additions & 7 deletions pipeline_versions.txt
Original file line number Diff line number Diff line change
Expand Up @@ -12,22 +12,22 @@ IlluminaGenotypingArray 1.12.27 2026-01-21
Imputation 1.1.23 2025-10-03
ImputationBeagle 3.0.1 2026-02-23
JointGenotyping 1.7.3 2025-08-11
MultiSampleSmartSeq2SingleNucleus 2.2.7 2026-05-15
Multiome 7.0.1 2026-05-15
Optimus 9.1.0 2026-05-15
PairedTag 3.0.1 2026-05-15
MultiSampleSmartSeq2SingleNucleus 2.2.8 2026-06-03
Multiome 7.0.2 2026-06-01
Optimus 9.1.1 2026-06-01
PairedTag 3.0.2 2026-06-03
PeakCalling 1.0.1 2025-08-11
Pipeline Name Version Date of Last Commit
RNAWithUMIsPipeline 1.0.20 2026-01-21
ReblockGVCF 2.4.4 2026-01-29
SlideSeq 3.6.7 2026-05-15
SlideTags 2.0.1 2026-05-15
SlideSeq 3.6.8 2026-06-03
SlideTags 2.0.2 2026-06-03
UltimaGenomicsJointGenotyping 1.2.3 2025-08-11
UltimaGenomicsWholeGenomeCramOnly 1.1.3 2026-01-21
UltimaGenomicsWholeGenomeGermline 1.2.2 2026-01-29
VariantCalling 2.2.8 2026-01-21
WholeGenomeGermlineSingleSample 3.3.7 2026-01-21
WholeGenomeReprocessing 3.3.7 2026-01-21
atac 2.9.3 2025-09-19
atac 2.9.4 2026-06-01
scANVI 1.0.0 2026-04-17
snm3C 4.1.1 2025-09-19
6 changes: 6 additions & 0 deletions pipelines/wdl/atac/atac.changelog.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
# 2.9.4
2026-06-01 (Date of Last Commit)

* Added optional `input_id_name` input (default `"input_id"`) to allow callers to control the metadata key name under which `input_id` is stored in the ATAC h5ad `obs` columns and `uns` global attributes
* Updated `warp-tools` docker tag to `2.7.1` (required for `--input_id_name` support)

Comment thread
jessicaway marked this conversation as resolved.
# 2.9.3
2025-09-19 (Date of Last Commit)

Expand Down
17 changes: 13 additions & 4 deletions pipelines/wdl/atac/atac.wdl
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ workflow ATAC {

# Output prefix/base name for all intermediate files and pipeline outputs
String input_id
# Key name used to label the input_id value in h5ad obs and uns metadata
String input_id_name = "input_id"
String cloud_provider
# Additional library aliquot ID
String? atac_nhash_id
Expand Down Expand Up @@ -55,15 +57,15 @@ workflow ATAC {
File? aligned_ATAC_bam
}

String pipeline_version = "2.9.3"
String pipeline_version = "2.9.4"

# Determine docker prefix based on cloud provider
String gcr_docker_prefix = "us.gcr.io/broad-gotc-prod/"
String acr_docker_prefix = "dsppipelinedev.azurecr.io/"
String docker_prefix = if cloud_provider == "gcp" then gcr_docker_prefix else acr_docker_prefix

# Docker image names
String warp_tools_docker = "warp-tools:2.6.1"
String warp_tools_docker = "warp-tools:2.7.1"
String cutadapt_docker = "cutadapt:1.0.0-4.4-1686752919"
String samtools_docker = "samtools-dist-bwa:3.0.0"
String upstools_docker = "upstools:1.0.0-2023.03.03-1704300311"
Expand Down Expand Up @@ -153,7 +155,8 @@ workflow ATAC {
docker_path = docker_prefix + snap_atac_docker,
atac_nhash_id = atac_nhash_id,
atac_expected_cells = atac_expected_cells,
input_id = input_id
input_id = input_id,
input_id_name = input_id_name
}
}
if (!preindex) {
Expand All @@ -166,7 +169,8 @@ workflow ATAC {
docker_path = docker_prefix + snap_atac_docker,
atac_nhash_id = atac_nhash_id,
atac_expected_cells = atac_expected_cells,
input_id = input_id
input_id = input_id,
input_id_name = input_id_name
}
if (peak_calling) {
call peakcalling.PeakCalling as PeakCalling{
Expand Down Expand Up @@ -546,6 +550,7 @@ task CreateFragmentFile {
String docker_path
String atac_nhash_id = ""
String input_id
String input_id_name = "input_id"
Int atac_expected_cells = 3000
String gtf_path = annotations_gtf
}
Expand Down Expand Up @@ -578,6 +583,7 @@ task CreateFragmentFile {
# set parameters
bam = "~{bam}"
input_id = "~{input_id}"
input_id_name = "~{input_id_name}"
chrom_sizes = "~{chrom_sizes}"
atac_gtf = "~{annotations_gtf}"
preindex = "~{preindex}"
Expand Down Expand Up @@ -636,6 +642,9 @@ task CreateFragmentFile {
atac_data = ad.read_h5ad("temp_metrics.h5ad")
# Add nhash_id to h5ad file as unstructured metadata
atac_data.uns['NHashID'] = atac_nhash_id
# Add input_id to obs (per-cell) and uns using the configurable key name
atac_data.obs[input_id_name] = input_id
atac_data.uns[input_id_name] = input_id

# Add GTF to uns field
# Original path from args.annotation_file
Expand Down
5 changes: 5 additions & 0 deletions pipelines/wdl/multiome/Multiome.changelog.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
# 7.0.2
2026-06-01 (Date of Last Commit)

* Added optional `input_id_name` input (default `"input_id"`) threaded through to both Optimus (GEX) and ATAC sub-workflows, allowing callers to control the metadata key name under which `input_id` is stored in the h5ad outputs

# 7.0.1
2026-05-15 (Date of Last Commit)

Expand Down
7 changes: 5 additions & 2 deletions pipelines/wdl/multiome/Multiome.wdl
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,13 @@ import "../../../tasks/wdl/Utilities.wdl" as utils

workflow Multiome {

String pipeline_version = "7.0.1"
String pipeline_version = "7.0.2"

input {
String cloud_provider
String input_id
# Key name used to label the input_id value in h5ad obs and uns metadata
String input_id_name = "input_id"
# Additional library aliquot ID
String? gex_nhash_id
String? atac_nhash_id
Expand Down Expand Up @@ -93,6 +95,7 @@ workflow Multiome {
r2_fastq = gex_r2_fastq,
i1_fastq = gex_i1_fastq,
input_id = input_id + "_gex",
input_id_name = input_id_name,
output_bam_basename = input_id + "_gex",
Comment thread
jessicaway marked this conversation as resolved.
gex_nhash_id = gex_nhash_id,
tar_star_reference = tar_star_reference,
Expand All @@ -105,7 +108,6 @@ workflow Multiome {
ignore_r1_read_length = ignore_r1_read_length,
star_strand_mode = star_strand_mode,
soloMultiMappers = soloMultiMappers,
cloud_provider = cloud_provider,
gex_expected_cells = expected_cells,
run_cellbender = run_cellbender
}
Expand All @@ -118,6 +120,7 @@ workflow Multiome {
read2_fastq_gzipped = atac_r2_fastq,
read3_fastq_gzipped = atac_r3_fastq,
input_id = input_id + "_atac",
input_id_name = input_id_name,
tar_bwa_reference = tar_bwa_reference,
chrom_sizes = chrom_sizes,
whitelist = atac_whitelist,
Expand Down
6 changes: 6 additions & 0 deletions pipelines/wdl/optimus/Optimus.changelog.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
# 9.1.1
2026-06-01 (Date of Last Commit)

* Added optional `input_id_name` input (default `"input_id"`) to allow callers to control the metadata key name under which `input_id` is stored in the GEX h5ad `obs` columns and `uns` global attributes
* Updated `warp-tools` docker tag to `2.7.1` (required for `--input_id_name` support)

Comment thread
jessicaway marked this conversation as resolved.
# 9.1.0
2026-05-15 (Date of Last Commit)

Expand Down
8 changes: 6 additions & 2 deletions pipelines/wdl/optimus/Optimus.wdl
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ workflow Optimus {
String input_id
# String for additional library aliquot ID
String? gex_nhash_id
# Key name used to label the input_id value in h5ad obs and uns metadata
String input_id_name = "input_id"
String output_bam_basename = input_id
String? input_name
String? input_id_metadata_field
Expand Down Expand Up @@ -76,7 +78,7 @@ workflow Optimus {
}

# Version of this pipeline
String pipeline_version = "9.1.0"
String pipeline_version = "9.1.1"

# this is used to scatter matched [r1_fastq, r2_fastq, i1_fastq] arrays
Array[Int] indices = range(length(r1_fastq))
Expand All @@ -98,7 +100,7 @@ workflow Optimus {
String pytools_docker = "pytools:1.0.0-1661263730"
String empty_drops_docker = "empty-drops:1.0.1-4.2"
String star_docker = "star:1.0.1-2.7.11a-1692706072"
String warp_tools_docker = "warp-tools:2.6.1"
String warp_tools_docker = "warp-tools:2.7.1"
String star_merge_docker = "star-merge-npz:1.3.0"
String samtools_star = "samtools-star:1.0.0-1.11-2.7.11a-1731516196"
String samtools_star_python = "samtools-star-python:1.0.0"
Expand All @@ -117,6 +119,7 @@ workflow Optimus {
r2_fastq: "reverse read, contains cDNA fragment generated from captured mRNA"
i1_fastq: "index read used for demultiplexing; required when tenx_chemistry_version is 4"
input_id: "name of sample matching this file, inserted into read group header"
input_id_name: "key name used to label the input_id value in h5ad obs and uns metadata (default: 'input_id')"
input_id_metadata_field: "String that describes the metadata field containing the input_id"
input_name: "User provided sample name or cell_names"
input_name_metadata_field: "String that describes the metadata field containing the input_name"
Expand Down Expand Up @@ -215,6 +218,7 @@ workflow Optimus {
call H5adUtils.OptimusH5adGeneration {
input:
input_id = input_id,
input_id_name = input_id_name,
gex_nhash_id = gex_nhash_id,
expected_cells = gex_expected_cells,
input_name = input_name,
Expand Down
5 changes: 5 additions & 0 deletions pipelines/wdl/paired_tag/PairedTag.changelog.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
# 3.0.2
2026-06-03 (Date of Last Commit)

* Updated warp-tools docker to 2.7.1 (via Optimus sub-workflow); no functional impact on PairedTag outputs

# 3.0.1
2026-05-15 (Date of Last Commit)

Expand Down
2 changes: 1 addition & 1 deletion pipelines/wdl/paired_tag/PairedTag.wdl
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import "../../../tasks/wdl/Utilities.wdl" as utils

workflow PairedTag {

String pipeline_version = "3.0.1"
String pipeline_version = "3.0.2"

input {
String input_id
Expand Down
5 changes: 5 additions & 0 deletions pipelines/wdl/slideseq/SlideSeq.changelog.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
# 3.6.8
2026-06-03 (Date of Last Commit)

* Updated warp-tools docker to 2.7.1 (via H5adUtils.wdl); no functional impact on SlideSeq outputs

# 3.6.7
2026-05-15 (Date of Last Commit)

Expand Down
2 changes: 1 addition & 1 deletion pipelines/wdl/slideseq/SlideSeq.wdl
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import "../../../tasks/wdl/Utilities.wdl" as utils

workflow SlideSeq {

String pipeline_version = "3.6.7"
String pipeline_version = "3.6.8"

input {
Array[File] r1_fastq
Expand Down
5 changes: 5 additions & 0 deletions pipelines/wdl/slidetags/SlideTags.changelog.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
# 2.0.2
2026-06-03 (Date of Last Commit)

* Updated warp-tools docker to 2.7.1 (via Optimus sub-workflow); no functional impact on SlideTags outputs

# 2.0.1
2026-05-15 (Date of Last Commit)

Expand Down
2 changes: 1 addition & 1 deletion pipelines/wdl/slidetags/SlideTags.wdl
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import "../optimus/Optimus.wdl" as optimus

workflow SlideTags {

String pipeline_version = "2.0.1"
String pipeline_version = "2.0.2"

input {

Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
# 2.2.8
2026-06-03 (Date of Last Commit)

* Updated warp-tools docker to 2.7.1; no functional impact on MultiSampleSmartSeq2SingleNucleus outputs

# 2.2.7
2026-05-15 (Date of Last Commit)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ workflow MultiSampleSmartSeq2SingleNucleus {
}

# Version of this pipeline
String pipeline_version = "2.2.7"
String pipeline_version = "2.2.8"

if (false) {
String? none = "None"
Expand Down
9 changes: 7 additions & 2 deletions tasks/wdl/H5adUtils.wdl
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ task OptimusH5adGeneration {
String warp_tools_docker_path
# name of the sample
String input_id
String input_id_name = "input_id"
String? gex_nhash_id
# user provided id
String counting_mode = "sc_rna"
Expand Down Expand Up @@ -71,6 +72,7 @@ task OptimusH5adGeneration {
--gene_id ~{gene_id} \
--output_path_for_h5ad "~{input_id}" \
--input_id ~{input_id} \
--input_id_name ~{input_id_name} \
~{"--input_name " + input_name} \
~{"--input_id_metadata_field " + input_id_metadata_field} \
~{"--input_name_metadata_field " + input_name_metadata_field} \
Expand All @@ -87,6 +89,7 @@ task OptimusH5adGeneration {
--gene_id ~{gene_id} \
--output_path_for_h5ad "~{input_id}" \
--input_id ~{input_id} \
--input_id_name ~{input_id_name} \
~{"--input_name " + input_name} \
~{"--input_id_metadata_field " + input_id_metadata_field} \
~{"--input_name_metadata_field " + input_name_metadata_field} \
Expand Down Expand Up @@ -143,6 +146,7 @@ task SingleNucleusOptimusH5adOutput {
String warp_tools_docker_path
# name of the sample
String input_id
String input_id_name = "input_id"
# additional aliquot id
String? gex_nhash_id
# user provided id
Expand Down Expand Up @@ -210,6 +214,7 @@ task SingleNucleusOptimusH5adOutput {
--gene_id_2 ~{gene_id_exon} \
--output_path_for_h5ad "~{input_id}" \
--input_id ~{input_id} \
--input_id_name ~{input_id_name} \
~{"--input_name " + input_name} \
~{"--input_id_metadata_field " + input_id_metadata_field} \
~{"--input_name_metadata_field " + input_name_metadata_field} \
Expand Down Expand Up @@ -601,7 +606,7 @@ task SingleNucleusSlideseqH5adOutput {
task SingleNucleusSmartSeq2H5adOutput {
input {
#runtime values
String docker = "us.gcr.io/broad-gotc-prod/warp-tools:2.6.1"
String docker = "us.gcr.io/broad-gotc-prod/warp-tools:2.7.1"
Comment thread
jessicaway marked this conversation as resolved.

Array[File] alignment_summary_metrics
Array[File] dedup_metrics
Expand Down Expand Up @@ -696,7 +701,7 @@ task AggregateSmartSeq2H5ad {
Array[File] h5ad_input
String batch_id
String pipeline_version
String docker = "us.gcr.io/broad-gotc-prod/warp-tools:2.6.1"
String docker = "us.gcr.io/broad-gotc-prod/warp-tools:2.7.1"
Int disk = 200
Int machine_mem_mb = 4000
Int cpu = 1
Expand Down
Loading