develop -> staging

.dockstore.yml

@@ -71,7 +71,6 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /pipelines/wdl/reprocessing/exome/ExomeReprocessing.wdl
 
-
   - name: FilterMTAndExportToVCF
     subclass: WDL
     primaryDescriptorPath: /all_of_us/rna_seq/MTtoVCF/FilterMTAndExportToVCF.wdl
@@ -176,6 +175,18 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /all_of_us/mitochondria/merge/mito_post_processing.wdl
 
+  - name: MMIDAS_Analyze
+    subclass: WDL
+    primaryDescriptorPath: /pipelines/wdl/mmidas/MMIDAS_Analyze.wdl
+
+  - name: MMIDAS_DataPrep
+    subclass: WDL
+    primaryDescriptorPath: /pipelines/wdl/mmidas/MMIDAS_DataPrep.wdl
+
+  - name: MMIDAS_Train
+    subclass: WDL
+    primaryDescriptorPath: /pipelines/wdl/mmidas/MMIDAS_Train.wdl
+
   - name: Multiome
     subclass: WDL
     primaryDescriptorPath: /pipelines/wdl/multiome/Multiome.wdl

.gitignore

@@ -37,3 +37,6 @@ scanvi_gpu_test/
 
 # Personal devcontainer (references local NAS mount + GPU; not shared)
 .devcontainer/devcontainer.json
+
+# Jupyter notebook checkpoints
+.ipynb_checkpoints/

AGENTS.md

@@ -128,7 +128,7 @@ Test JSON inputs live at two locations per pipeline:
 
 When adding a new test input for a regression case, also wire it into the relevant GitHub Actions workflow under `.github/workflows/` so CI picks it up.
 
-**Stale inputs break silently** — they are not validated by womtool. When renaming a workflow or removing/renaming inputs, audit `example_inputs/*.json` and `test_inputs/**/*.json`.
+**Stale inputs break silently** — they are not validated by womtool. When **adding**, renaming, or removing/renaming inputs, audit `example_inputs/*.json` and `test_inputs/**/*.json` **and** thread the change through the `Test<Name>.wdl` wrapper (see [Registering a CI test](#registering-a-ci-test-for-a-pipeline-plumbing--scientific) step 2 — *forward every testable input*). Mind the framework's `UpdateTestInputs.py` key rewrite: a test-JSON key with more than two dotted parts (e.g. `Pipeline.Sub.input`, reaching into a sub-workflow) is rewritten to `TestPipeline.Pipeline.Sub.input`, which the wrapper rejects as an extra input. Prefer exposing the input at the pipeline's **top level** (`Pipeline.input`) and setting it that way — a two-part key rewrites cleanly to `TestPipeline.input`.
 
 ### Registering a CI test for a pipeline (Plumbing / Scientific)
 

all_of_us/phasing/README.md

@@ -53,5 +53,41 @@ Runtime Parameters:
 - `String aou_vcf_header` – URL for single chromosome output VCF header
 - `String report` – URL for the report file
 
+## Beagle5Phasing
+#### Background
+
+This WDL workflow phases genomic variants using Beagle5. It is designed for use in the All of Us Local Ancestry
+pipeline and operates on a single VCF at a time. A user-supplied shell script drives the Beagle5 execution,
+allowing flexibility in phasing parameters.
+
+Key characteristics:
+- Each VCF is processed independently; outputs are not merged across chromosomes.
+- Phasing is performed with a sliding-window approach controlled by the `window_markers` parameter.
+- Designed for large-scale WGS datasets (e.g., AoU cohort).
+- Input Data: Accepts a GCS-hosted VCF and a genetic map file.
 
+#### Inputs
+Analysis Parameters:
+- `String vcf_gs` – GCS URL of the input VCF to phase
+- `String map_gs` – GCS URL of the genetic map file
+- `String out_prefix` – Prefix for output filenames
+- `String out_dir_gs` – GCS directory for storing outputs
+- `Int window_markers` – Window size in markers for phasing (default: 3500000)
+- `String java_xmx` – Java max heap size for Beagle5 (default: "80g")
+- `File run_beagle_sh` – Shell script executed to invoke Beagle5; defines phasing steps
+
+Runtime Parameters:
+- `Int runtime_cpu` – Number of CPUs (default: 32)
+- `Int mem_gb` – Memory in GB (default: 120)
+- `Int runtime_disk_gb` – Disk size in GB (default: 2000)
+- `String runtime_disk_type` – Disk type, "HDD" or "SSD" (default: "HDD")
+
+#### Step 1. PhaseWithBeagle
+- Script Execution: Runs the user-supplied shell script with the provided phasing parameters.
+- Phasing: Invokes Beagle5 to phase the input VCF using the supplied genetic map and window size.
+- Output Generation: Saves the phased VCF and log file to the specified GCS output directory.
+
+#### Outputs
 
+- `String phased_vcf_gs` – GCS URL for the phased output VCF
+- `String phased_log_gs` – GCS URL for the Beagle5 phasing log file

all_of_us/rna_seq/AggregateSusieWorkflow.changelog.md

@@ -1,6 +1,11 @@
-# aou_9.0.2
+# aou_9.0.3
+2026-05-11 (Date of Last Commit)
+
+* Removed unnecessary file list from aggregate task
 
+# aou_9.0.2
 2026-01-29 (Date of Last Commit)
+
 * Added set euo pipefail to tasks
 
 # aou_9.0.1

all_of_us/rna_seq/AggregateSusieWorkflow.wdl

@@ -19,8 +19,6 @@ task AggregateSusie{
     mkdir -p localized
     gsutil -m cp -I localized/ < file_paths.txt 
 
-    # Write the new local file paths into filelist.txt
-    ls -1 "$(pwd)/localized/*" > filelist.txt
     Rscript /tmp/merge_susie.R --FilePaths file_paths.txt  --OutputPrefix ~{OutputPrefix}
     >>>
 
@@ -108,7 +106,7 @@ workflow AggregateSusieWorkflow {
 
     }
 
-    String pipeline_version = "aou_9.0.2"
+    String pipeline_version = "aou_9.0.3"
 
     call AggregateSusie {
         input:

all_of_us/rna_seq/susieR_workflow.changelog.md

@@ -1,3 +1,10 @@
+# aou_9.1.0
+2026-08-25 (Date of Last Commit)
+
+* Added a new tsv input for all mapping inputs per ancestry, including qtls, phenotype beds, covariates, and genotype dosages
+* Added a new task to the worfklow to dynamically set file inputs from a new input tsv
+* Removed the individual mapping file inputs because they were now set dynamically by the pipeline
+
 # aou_9.0.1
 2026-01-29 (Date of Last Commit)
 

all_of_us/rna_seq/susieR_workflow.wdl

@@ -2,28 +2,53 @@ version 1.0
 
 # Docker fixed to quay.io/biocontainers/htslib@sha256:ff9d466929dc2d587128afc213fc4516d
 
-#task splitPhenotypeBed {
-#    input {
-#        File TensorQTLPermutations
-#    }
+# InputMappingTsv: tab-separated, one row per ancestry, with a header row containing at least:
+#   mapping_inputs_id (ancestry code: AFR, AMR, COMB, EUR, SAS, EAS, MID), GenotypeDosage,
+#   GenotypeDosagei, PhenotypePCsOut, PlinkAF, QtlCovariates, Sample_list, VCF, cis_qtl,
+#   genotype_pcs, pgen, phenotype_bed, psam, pvar. Only the columns consumed below are read;
+# each cell (other than mapping_inputs_id) must be a gs:// path to the corresponding file.
+task ParseInputMapping {
+    input {
+        File InputMappingTsv
+        String Ancestry
+        Int NumPrempt
+    }
+    command <<<
+        set -euo pipefail
+        python3 <<CODE
+        import csv
+
+        cols = ["GenotypeDosage", "GenotypeDosagei", "QtlCovariates", "Sample_list", "phenotype_bed", "cis_qtl"]
+        with open("~{InputMappingTsv}") as f:
+            reader = csv.DictReader(f, delimiter="\t")
+            for row in reader:
+                if row["mapping_inputs_id"] == "~{Ancestry}":
+                    for col in cols:
+                        with open(f"{col}.txt", "w") as out:
+                            out.write(row[col].strip())
+                    break
+            else:
+                raise ValueError("No row found for mapping_inputs_id '~{Ancestry}' in InputMappingTsv")
+        CODE
+    >>>
 
-    #String baseName = basename(PhenotypeBed, ".gz")
+    runtime {
+        docker: "python:3.11-slim"
+        disks: "local-disk 10 SSD"
+        preemptible: "${NumPrempt}"
+        memory: "2GB"
+        cpu: "1"
+    }
 
-#    command <<<
-#        zcat ~{TensorQTLPermutations} | awk '$18 < 0.05' | head -n 100  > significant_qtls.txt 
-#        awk 'NR==1 {header=$0; next} {out=$1".txt"; print header > out; print >> out}' significant_qtls.txt 
-#    >>>
-#
-#    output {
-#        Array[File] splitFiles = glob("*.txt")
-#    }
-#    runtime {
-#        docker: "quay.io/biocontainers/htslib:1.22.1--h566b1c6_0"
-#        disks: "local-disk 500 SSD"
-#        memory: "2GB"
-#        cpu: "1"
-#    }
-#}
+    output {
+        String GenotypeDosagePath = read_string("GenotypeDosage.txt")
+        String GenotypeDosageIndexPath = read_string("GenotypeDosagei.txt")
+        String QTLCovariatesPath = read_string("QtlCovariates.txt")
+        String SampleListPath = read_string("Sample_list.txt")
+        String PhenotypeBedPath = read_string("phenotype_bed.txt")
+        String TensorQTLPermutationsPath = read_string("cis_qtl.txt")
+    }
+}
 
 task PrepInputs {
     input {
@@ -168,39 +193,41 @@ task susieR {
 
 workflow susieR_workflow {
     input {
-        File GenotypeDosages
-        File GenotypeDosageIndex
-        File QTLCovariates
-        File TensorQTLPermutations
-        File SampleList
-        File PhenotypeBed
+        File InputMappingTsv
+        String Ancestry
         Int CisDistance
         File susie_rscript
         Int memory
         Int NumPrempt
         String OutputPrefix
         String PhenotypeID
     }
-    String pipeline_version = "aou_9.0.0"
+    String pipeline_version = "aou_9.1.0"
+
+    call ParseInputMapping {
+        input:
+            InputMappingTsv = InputMappingTsv,
+            Ancestry = Ancestry
+    }
 
     call PrepInputs {
         input:
-            TensorQTLPermutations = TensorQTLPermutations,
+            TensorQTLPermutations = ParseInputMapping.TensorQTLPermutationsPath,
             PhenotypeID = PhenotypeID,
-            GenotypeDosages = GenotypeDosages,
-            GenotypeDosageIndex = GenotypeDosageIndex,
-            PhenotypeBed = PhenotypeBed,
+            GenotypeDosages = ParseInputMapping.GenotypeDosagePath,
+            GenotypeDosageIndex = ParseInputMapping.GenotypeDosageIndexPath,
+            PhenotypeBed = ParseInputMapping.PhenotypeBedPath,
             NumPrempt = NumPrempt
     }
 
     call susieR {
         input:
             GenotypeDosages = PrepInputs.SubsetDosages,
             GenotypeDosageIndex = PrepInputs.SubsetDosagesIndex,
-            QTLCovariates = QTLCovariates,
+            QTLCovariates = ParseInputMapping.QTLCovariatesPath,
             TensorQTLPermutations = PrepInputs.SubsetPermutationPvals,
-            SampleList = SampleList,
-            PhenotypeBed = PhenotypeBed ,
+            SampleList = ParseInputMapping.SampleListPath,
+            PhenotypeBed = ParseInputMapping.PhenotypeBedPath,
             CisDistance = CisDistance,
             OutputPrefix = PhenotypeID,
             susie_rscript = susie_rscript,
@@ -209,12 +236,7 @@ workflow susieR_workflow {
 
         }
 
-    #call MergeSusie {
-    #    input:
-    #        SusieOutput = susieR.SusieParquet,
-    #        OutputPrefix = OutputPrefix 
-    #
-    #} 
+
     output {
         File SusieParquet = susieR.SusieParquet
         File SusielbfParquet = susieR.lbfParquet

pipeline_versions.txt

@@ -9,21 +9,24 @@ Glimpse2LowPassImputation	 1.1.0	2026-08-26
 Glimpse2LowPassImputationBatch	 1.1.0	2026-08-26 
 Glimpse2LowPassImputationQC	 1.1.0	2026-08-26 
 Glimpse2LowPassImputationQuotaConsumed	 1.0.1	2026-08-24 
-Glimpse2SVImputation	 0.0.26	2026-08-28 
-Glimpse2SVImputationBatch	 0.0.19	2026-08-28 
-Glimpse2SVImputationQC	 0.0.2	2026-08-26 
-Glimpse2SVImputationQuotaConsumed	 0.0.2	2026-08-26 
+Glimpse2SVImputation	 1.0.0	2026-09-09 
+Glimpse2SVImputationBatch	 1.0.0	2026-09-09 
+Glimpse2SVImputationQC	 1.0.0	2026-09-09 
+Glimpse2SVImputationQuotaConsumed	 1.0.0	2026-09-09 
 IlluminaGenotypingArray	 1.12.27	2026-01-21 
 Imputation	 1.1.23	2025-10-03 
 ImputationBeagle	 3.0.1	2026-02-23 
 JointGenotyping	 1.7.3	2025-08-11 
+MMIDAS_Analyze	 1.1.2	2026-08-21 
+MMIDAS_DataPrep	 1.0.2	2026-08-11 
+MMIDAS_Train	 1.3.0	2026-08-27 
 MultiSampleSmartSeq2SingleNucleus	 2.2.8	2026-07-10 
-MultilevelHierarchicallyPasteVcfsStreaming	 0.0.10	2026-08-28 
+MultilevelHierarchicallyPasteVcfsStreaming	 1.0.0	2026-09-09 
 Multiome	 7.0.2	2026-07-10 
 Optimus	 9.2.0	2026-07-10 
 PairedTag	 3.0.2	2026-07-10 
 PeakCalling	 1.0.1	2025-08-11 
-PreprocessPLsGVCF	 0.0.15	2026-08-28 
+PreprocessPLsGVCF	 1.0.0	2026-09-09 
 RNAWithUMIsPipeline	 1.0.20	2026-01-21 
 ReblockGVCF	 2.4.4	2026-01-29 
 SlideSeq	 3.6.8	2026-07-10 

pipelines/wdl/glimpse/sv_imputation/Glimpse2SVImputation.changelog.md

@@ -1,3 +1,25 @@
+# 1.0.0
+2026-09-09 (Date of Last Commit)
+
+* Initial version for teaspoons prod release
+
+# 0.0.29
+2026-09-08 (Date of Last Commit)
+
+* Update linked `input_qc_version` to 0.0.3.
+
+# 0.0.28
+2026-09-03 (Date of Last Commit)
+
+* Rename outputs `imputed_vcf` to `imputed_vcfs` and `imputed_vcf_index` to `imputed_vcf_indexes`
+
+# 0.0.27
+2026-08-31 (Date of Last Commit)
+
+* Update `ConvertInputArraysToManifest` output wiring to use `output_gvcf_manifest`.
+* Switch `PreProcessGVCFsBatch` outputs consumed by `Glimpse2SVImputationBatch` from `preprocessed_pls_vcf` to `preprocessed_pls_bcf` naming.
+* Update linked `preprocess_pls_gvcf_pipeline_version` to 0.0.16 and `batch_pipeline_version` to 0.0.20.
+
 # 0.0.26
 2026-08-28 (Date of Last Commit)