Add mapmycells

.dockstore.yml

@@ -164,6 +164,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /all_of_us/hla/MakeTable.wdl
 
+  - name: MapMyCells
+    subclass: WDL
+    primaryDescriptorPath: /pipelines/wdl/mapmycells/MapMyCells.wdl
+
   - name: MitochondriaMerge
     subclass: WDL
     primaryDescriptorPath: /all_of_us/mitochondria/merge/mitochondria_merge.wdl
@@ -360,6 +364,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestJointGenotyping.wdl
 
+  - name: TestMapMyCells
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestMapMyCells.wdl
+
   - name: TestMultiome
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestMultiome.wdl

.github/workflows/test_mapmycells.yml

@@ -0,0 +1,67 @@
+name: Test MapMyCells
+
+# Minimal permissions for the reusable test workflow (Terra/GCP auth needs id-token;
+# it also reads contents and writes to the Actions API).
+permissions:
+  contents: read
+  id-token: write
+  actions: write
+
+# Controls when the workflow will run
+on:
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    paths:
+      # anything in the pipelines folder
+      - 'pipelines/wdl/mapmycells/**'
+      # tasks from the pipeline WDL and their dependencies
+      - 'tasks/wdl/Utilities.wdl'
+      # verification WDL (MapMyCells-specific; kept out of the shared
+      # verification/VerifyTasks.wdl so MapMyCells changes don't trigger every pipeline)
+      - 'verification/VerifyMapMyCells.wdl'
+      # test WDL and its dependencies
+      - 'verification/test-wdls/TestMapMyCells.wdl'
+      - 'tasks/wdl/TerraCopyFilesFromCloudToCloud.wdl'
+      # this file, the subworkflow file, and the firecloud_api script
+      - '.github/workflows/test_mapmycells.yml'
+      - '.github/workflows/warp_test_workflow.yml'
+      - 'scripts/firecloud_api/firecloud_api.py'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+jobs:
+  TestMapMyCells:
+    uses: ./.github/workflows/warp_test_workflow.yml
+    with:
+      pipeline_name: TestMapMyCells
+      dockstore_pipeline_name: MapMyCells
+      pipeline_dir: pipelines/wdl/mapmycells
+      use_call_cache: ${{ github.event.inputs.useCallCache || 'true' }}
+      update_truth: ${{ github.event.inputs.updateTruth || 'false' }}
+      test_type: ${{ github.event.inputs.testType || '' }}
+      truth_branch: ${{ github.event.inputs.truthBranch || 'master' }}
+    secrets:
+      PDT_TESTER_SA_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+      DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}

pipeline_versions.txt

@@ -17,6 +17,7 @@ IlluminaGenotypingArray	 1.12.27	2026-01-21
 Imputation	 1.1.23	2025-10-03 
 ImputationBeagle	 3.0.1	2026-02-23 
 JointGenotyping	 1.7.3	2025-08-11 
+MapMyCells	 0.1.0	2026-09-02 
 MultiSampleSmartSeq2SingleNucleus	 2.2.8	2026-07-10 
 MultilevelHierarchicallyPasteVcfsStreaming	 0.0.11	2026-08-31 
 Multiome	 7.0.2	2026-07-10 

pipelines/wdl/mapmycells/MapMyCells.changelog.md

@@ -0,0 +1,13 @@
+# 0.1.0
+2026-09-02 (Date of Last Commit)
+
+* Added the MapMyCells pipeline, wrapping the Allen Institute `cell_type_mapper` library for cell type label transfer against precomputed reference atlases (Human_MTG, Mouse_WMB, or a Custom taxonomy)
+* Moved the Human_MTG / Mouse_WMB precomputed-stats and marker-gene assets into the docker image (built in warp-tools) instead of referencing `gs://` paths that were never actually populated; the task now selects the in-container path per `reference_atlas` instead of computing a workflow-level `File` from a bucket that didn't have the file
+* Added an `ErrorWithMessage` check that fails fast when `reference_atlas` is `"Custom"` but `custom_precomputed_stats` is not supplied, instead of failing later with an opaque `select_first` error
+* Repinned the `docker` default to the image tag actually published for this branch (`add-mapmycells`); the previous default tag was never built
+* Repointed `gene_mapping_db`'s default (applies to every `reference_atlas`, not just Human_MTG/Mouse_WMB) at a real, now-populated copy of Allen Institute's pre-built gene-symbol-to-Ensembl-ID db at `gs://broad-gotc-test-storage/mapmycells/mmc_gene_mapper.2025-08-04.db` (~15GB); the previous path pointed at a file that never existed. Kept this asset GCS-hosted rather than baked into the docker image -- warp-tools has no precedent for baking reference data this large into an image, and doing so would balloon every task's docker pull by ~15GB regardless of whether that run uses gene mapping. Bumped `disk_size` default 100->150 to cover its localization
+* Overridable per-run via the workflow-level `custom_gene_mapping_db` input
+* Repointed `test_inputs/Plumbing/test_human.json` and `test_mouse.json` at real, existing gene-expression h5ads borrowed from the scANVI pipeline's own test set (`10k_PBMC_gex.h5ad`, `Mouse_Hippocampus_AIT_plumbing_gex.h5ad`) instead of `query.h5ad`/`query_mouse.h5ad`, which never existed; MapMyCells only consumes gene expression, so only the gex file from each scANVI test case is reused, not the ATAC or reference h5ads
+* Added a Citing section to the README crediting the Allen Institute's `cell_type_mapper` (source code + the MapMyCells preprint, Daniel et al. 2026, bioRxiv doi:10.64898/2026.03.06.710160)
+* Moved `test_human.json`/`test_mouse.json` from `test_inputs/Plumbing/` to `test_inputs/Scientific/` -- these become the Scientific tests; new, smaller Plumbing tests to follow separately. No truth re-seed needed for this move: `UpdateTestInputs.py` derives the GCS truth tier from the input filename/content, not from which `test_inputs/` subdirectory the file lives in, and both files' already-seeded truth already happens to sit at the paths this heuristic computes for them (`test_mouse` under `.../truth/plumbing/...`, `test_human` under `.../truth/scientific/...` -- the latter was the "known issue" from the previous entry, now moot since `test_human` actually is a Scientific test)
+* Added new `test_inputs/Plumbing/test_human.json` and `test_mouse.json` using Allen Institute's own MapMyCells example query files (https://brain-map.org/bkp/analyze/mapmycells/files: 1k-cell human MTG example, 10k-cell whole-mouse-brain example) staged at `gs://pd-test-storage-public/MapMyCells/input/plumbing/{human,mouse}/`, matching the `<Pipeline>/input/{plumbing,scientific}/` convention other WARP pipelines use for test data

pipelines/wdl/mapmycells/MapMyCells.wdl

@@ -0,0 +1,174 @@
+version 1.0
+
+import "../../../tasks/wdl/Utilities.wdl" as utils
+
+workflow MapMyCells {
+    meta {
+        description: "Executes the MapMyCells cell_type_mapper for cell type annotation."
+        allowNestedInputs: true
+    }
+
+    String pipeline_version = "0.1.0"
+
+    input {
+        File query_h5ad
+        String input_id
+
+        # Select the reference atlas. Allowed values: "Human_MTG", "Mouse_WMB", "Custom"
+        String reference_atlas = "Human_MTG"
+
+        # If reference_atlas == "Custom", you must provide the following:
+        File? custom_precomputed_stats
+        File? custom_gene_mapping_db
+        File? custom_query_markers
+
+        String algorithm = "hierarchical"
+        String normalization = "raw"
+
+        # Docker image
+        # ponytail: pinned to the warp-tools add-mapmycells branch tag; repin to a
+        # versioned tag once warp-tools/3rd-party-tools/mapmycells/docker_versions.tsv
+        # gets a real bump and this pipeline merges.
+        String docker = "us.gcr.io/broad-gotc-prod/mapmycells:add-mapmycells"
+
+        # Runtime
+        # disk_size headroom accounts for gene_mapping_db (~15GB), localized on every run
+        Int cpu = 8
+        Int memory_gb = 64
+        Int disk_size = 150
+        Int preemptible = 3
+    }
+
+    if (reference_atlas == "Custom" && !defined(custom_precomputed_stats)) {
+        call utils.ErrorWithMessage as ErrorCustomPrecomputedStatsRequired {
+            input:
+                message = "custom_precomputed_stats is required when reference_atlas is 'Custom'."
+        }
+    }
+
+    # Allen Institute's pre-built gene-symbol -> Ensembl-ID mapping db (~15GB; hosted on
+    # GCS, not baked into the docker image -- see warp-tools' mapmycells README for why).
+    # Applies to any reference_atlas; override with custom_gene_mapping_db if desired.
+    File gene_mapping_db = select_first([
+        custom_gene_mapping_db,
+        "gs://broad-gotc-test-storage/mapmycells/mmc_gene_mapper.2025-08-04.db"
+    ])
+
+    call RunMapMyCells {
+        input:
+            query_h5ad               = query_h5ad,
+            input_id                 = input_id,
+            reference_atlas          = reference_atlas,
+            custom_precomputed_stats = custom_precomputed_stats,
+            gene_mapping_db          = gene_mapping_db,
+            custom_query_markers     = custom_query_markers,
+            algorithm                = algorithm,
+            normalization            = normalization,
+            docker                   = docker,
+            cpu                      = cpu,
+            memory_gb                = memory_gb,
+            disk_size                = disk_size,
+            preemptible              = preemptible
+    }
+
+    output {
+        File output_csv = RunMapMyCells.output_csv
+        File output_json = RunMapMyCells.output_json
+    }
+}
+
+task RunMapMyCells {
+    input {
+        File query_h5ad
+        String input_id
+        String reference_atlas
+        File? custom_precomputed_stats
+        File gene_mapping_db
+        File? custom_query_markers
+        String algorithm
+        String normalization
+        String docker
+        Int cpu
+        Int memory_gb
+        Int disk_size
+        Int preemptible
+    }
+
+    # Human_MTG and Mouse_WMB reference-atlas assets are baked into the docker image
+    # (see warp-tools/3rd-party-tools/mapmycells) instead of hosted externally, so they
+    # need no File input / localization of their own -- only their in-container path.
+    command <<<
+        set -euo pipefail
+
+        case "~{reference_atlas}" in
+            Human_MTG)
+                precomputed_stats="/opt/mapmycells/data/precomputed_stats.20231120.sea_ad.MTG.h5"
+                query_markers=""
+                ;;
+            Mouse_WMB)
+                precomputed_stats="/opt/mapmycells/data/precomputed_stats_ABC_revision_230821.h5"
+                query_markers="/opt/mapmycells/data/mouse_markers_230821.json"
+                ;;
+            Custom)
+                precomputed_stats="~{custom_precomputed_stats}"
+                query_markers="~{custom_query_markers}"
+                ;;
+            *)
+                >&2 echo "Error: reference_atlas must be one of Human_MTG, Mouse_WMB, Custom (got '~{reference_atlas}')"
+                exit 1
+                ;;
+        esac
+
+        # Determine the entrypoint based on whether markers are available
+        if [ -n "$query_markers" ]; then
+            python -m cell_type_mapper.cli.from_specified_markers \
+                --query_path ~{query_h5ad} \
+                --extended_result_path ~{input_id}_mapmycells_extended.json \
+                --csv_result_path ~{input_id}_mapmycells_results.csv \
+                --type_assignment.n_processors ~{cpu} \
+                --precomputed_stats.path "$precomputed_stats" \
+                --query_markers.serialized_lookup "$query_markers" \
+                --query_markers.collapse_markers False \
+                --type_assignment.algorithm ~{algorithm} \
+                --type_assignment.normalization ~{normalization} \
+                --gene_mapping.db_path ~{gene_mapping_db}
+        else
+            python -m cell_type_mapper.cli.map_to_on_the_fly_markers \
+                --query_path ~{query_h5ad} \
+                --extended_result_path ~{input_id}_mapmycells_extended.json \
+                --csv_result_path ~{input_id}_mapmycells_results.csv \
+                --n_processors ~{cpu} \
+                --precomputed_stats.path "$precomputed_stats" \
+                --type_assignment.algorithm ~{algorithm} \
+                --type_assignment.normalization ~{normalization} \
+                --gene_mapping.db_path ~{gene_mapping_db} \
+                --query_markers.n_per_utility 15 \
+                --reference_markers.log2_fold_min_th 0.5
+        fi
+    >>>
+
+    output {
+        File output_csv = "~{input_id}_mapmycells_results.csv"
+        File output_json = "~{input_id}_mapmycells_extended.json"
+    }
+
+    runtime {
+        docker: docker
+        cpu: cpu
+        memory: "~{memory_gb} GiB"
+        disks: "local-disk ~{disk_size} HDD"
+        preemptible: preemptible
+    }
+
+    parameter_meta {
+        query_h5ad: "Input AnnData file containing raw transcriptomics counts."
+        input_id: "Prefix for output files."
+        reference_atlas: "Which reference atlas's baked-in assets to use, or 'Custom' to supply your own."
+        custom_precomputed_stats: "HDF5 file containing the precomputed hierarchical taxonomy stats. Required when reference_atlas is 'Custom'."
+        gene_mapping_db: "SQLite database for translating gene symbols to Ensembl IDs, used regardless of reference_atlas. Defaults to Allen Institute's pre-built db (see the workflow-level custom_gene_mapping_db input to override)."
+        custom_query_markers: "Optional JSON file containing predefined marker genes. Only used when reference_atlas is 'Custom'."
+        algorithm: "Type assignment algorithm. Options: 'hierarchical', 'hann'. Default: 'hierarchical'."
+        normalization: "Normalization method to use for mapping. Options: 'raw', 'log2CPM'. Default: 'raw'."
+        preemptible: "Number of times to attempt to run on a preemptible VM."
+    }
+}

pipelines/wdl/mapmycells/README.md

@@ -0,0 +1,51 @@
+# MapMyCells Pipeline
+
+## Summary
+
+MapMyCells is a cloud-optimized WDL workflow that performs highly accurate **cell type label transfer** for single-cell or single-nucleus RNA-seq (scRNA-seq/snRNA-seq) data. The pipeline is powered by the Allen Institute's `cell_type_mapper` library, which relies on an extremely fast hierarchical nearest-centroid algorithm to map query data against massive precomputed reference atlases (e.g., the 4M-cell Allen Brain Cell Atlas).
+
+Unlike iterative machine learning methods (such as scANVI or Seurat), MapMyCells maps queries to a statistically precomputed representation of the reference taxonomy. This gives it two distinct advantages:
+1. It scales instantly and maps hundreds of thousands of cells in minutes on a standard CPU.
+2. It flawlessly resolves exceptionally rare subpopulations and extremely granular hierarchical taxonomies.
+
+## Reference Atlases
+
+The pipeline supports three modes of mapping controlled by the `reference_atlas` input:
+* `"Human_MTG"` (Default) — Maps the query against the Human Middle Temporal Gyrus (MTG) taxonomy.
+* `"Mouse_WMB"` — Maps the query against the massive Whole Mouse Brain (WMB) / Allen Brain Cell (ABC) taxonomy.
+* `"Custom"` — Allows the user to provide their own `.h5` reference statistics and `.json` markers.
+
+## Inputs
+
+The MapMyCells pipeline takes an unannotated `query_h5ad` file and outputs `.csv` and `.json` predictions.
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `MapMyCells.query_h5ad` | File | **Required.** The input AnnData file containing raw scRNA-seq counts. |
+| `MapMyCells.input_id` | String | **Required.** An identifier prepended to all output filenames. |
+| `MapMyCells.reference_atlas` | String | Defines which precomputed atlas is used. Options: `"Human_MTG"`, `"Mouse_WMB"`, `"Custom"`. Default: `"Human_MTG"`. |
+| `MapMyCells.custom_precomputed_stats` | File? | **Required if Custom.** An HDF5 file containing the precomputed hierarchical taxonomy stats. |
+| `MapMyCells.custom_query_markers` | File? | *Optional.* JSON file containing predefined marker genes. Only used when `reference_atlas` is `"Custom"` — Human_MTG and Mouse_WMB use their own baked-in markers (or none, for Human_MTG). |
+| `MapMyCells.custom_gene_mapping_db` | File? | *Optional.* SQLite DB for translating gene symbols to Ensembl IDs. Defaults to Allen Institute's pre-built db (`gs://broad-gotc-test-storage/mapmycells/mmc_gene_mapper.2025-08-04.db`, ~15GB) for every atlas; override only if you need a different mapping. |
+| `MapMyCells.algorithm` | String | Type assignment algorithm. Options: `"hierarchical"`, `"hann"`. Default: `"hierarchical"`. |
+
+## Outputs
+
+The task generates two output files:
+* `~{input_id}_mapmycells_results.csv`: Contains the hierarchical mapping assignment for each cell (class, subclass, supertype, cluster) along with bootstrapping probabilities.
+* `~{input_id}_mapmycells_extended.json`: A verbose JSON output detailing the tree traversal, probabilities, and mapped pathways for each individual cell.
+
+## Usage
+
+You can find example inputs in the `test_inputs/` folder:
+* `test_inputs/Plumbing/test_human.json` / `test_inputs/Plumbing/test_mouse.json` — small smoke tests, using Allen Institute's own [MapMyCells example files](https://brain-map.org/bkp/analyze/mapmycells/files) (1k and 10k cells respectively)
+* `test_inputs/Scientific/test_human.json` / `test_inputs/Scientific/test_mouse.json` — full-scale runs
+
+The Human_MTG and Mouse_WMB reference-atlas assets are baked into the pipeline's docker image (see [warp-tools/3rd-party-tools/mapmycells](https://github.com/broadinstitute/warp-tools/tree/develop/3rd-party-tools/mapmycells)), so you do not need to download the reference atlases or pass them into the workflow yourself. Simply specify the `reference_atlas` and provide your `query_h5ad`!
+
+## Citing
+
+This pipeline wraps the Allen Institute's `cell_type_mapper` library. If you use it in your research, please cite:
+
+* Daniel SF, Lee C, Mollenkopf T, et al. *High-performance mapping of unlabeled cell-by-gene data to reference brain taxonomies.* bioRxiv (2026). doi: [10.64898/2026.03.06.710160](https://doi.org/10.64898/2026.03.06.710160) ([PMID: 41958981](https://pubmed.ncbi.nlm.nih.gov/41958981/))
+* Source code: [github.com/AllenInstitute/cell_type_mapper](https://github.com/AllenInstitute/cell_type_mapper)

pipelines/wdl/mapmycells/test_inputs/Plumbing/test_human.json

@@ -0,0 +1,7 @@
+{
+  "MapMyCells.input_id": "human_mtg_example",
+  "MapMyCells.query_h5ad": "gs://pd-test-storage-public/MapMyCells/input/plumbing/human/10xhumanmtgsea-adccn20230505_example_1kcells_36601genes.h5ad",
+  "MapMyCells.reference_atlas": "Human_MTG",
+  "MapMyCells.cpu": 2,
+  "MapMyCells.memory_gb": 8
+}

pipelines/wdl/mapmycells/test_inputs/Plumbing/test_mouse.json

@@ -0,0 +1,7 @@
+{
+  "MapMyCells.input_id": "mouse_wmb_example",
+  "MapMyCells.query_h5ad": "gs://pd-test-storage-public/MapMyCells/input/plumbing/mouse/wholemousebrain_ccn20230722_example_10kcells_550genes.h5ad",
+  "MapMyCells.reference_atlas": "Mouse_WMB",
+  "MapMyCells.cpu": 2,
+  "MapMyCells.memory_gb": 8
+}

pipelines/wdl/mapmycells/test_inputs/Scientific/test_human.json

@@ -0,0 +1,7 @@
+{
+  "MapMyCells.input_id": "test_human_mtg",
+  "MapMyCells.query_h5ad": "gs://pd-test-storage-public/scANVI/input/scientific/10k_PBMC/10k_PBMC_gex.h5ad",
+  "MapMyCells.reference_atlas": "Human_MTG",
+  "MapMyCells.cpu": 4,
+  "MapMyCells.memory_gb": 32
+}

pipelines/wdl/mapmycells/test_inputs/Scientific/test_mouse.json

@@ -0,0 +1,7 @@
+{
+  "MapMyCells.input_id": "test_mouse_wmb",
+  "MapMyCells.query_h5ad": "gs://pd-test-storage-public/scANVI/input/plumbing/Mouse_Hippocampus_AIT/Mouse_Hippocampus_AIT_plumbing_gex.h5ad",
+  "MapMyCells.reference_atlas": "Mouse_WMB",
+  "MapMyCells.cpu": 8,
+  "MapMyCells.memory_gb": 64
+}