Skip to content

Commit 35a0ef6

Browse files
heylfheylf
committed
adding atera data loader
1 parent 0eea3f1 commit 35a0ef6

4 files changed

Lines changed: 241 additions & 0 deletions

File tree

‎scripts/create_resources/process_10x_atera.sh‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#!/bin/bash
2+
3+
# get the root of the directory
4+
REPO_ROOT=$(git rev-parse --show-toplevel)
5+
6+
# ensure that the command below is run from the root of the repository
7+
cd "$REPO_ROOT"
8+
9+
set -e
10+
11+
publish_dir="s3://openproblems-data/resources/datasets"
12+
13+
cat > /tmp/params.yaml << HERE
14+
param_list:
15+
16+
- id: "10x_xenium/2025_10x_human_breast_cancer_atera"
17+
input: https://s3-us-west-2.amazonaws.com/10x.files/samples/atera/dev/WTA_Preview_FFPE_Breast_Cancer/WTA_Preview_FFPE_Breast_Cancer_xe_outs.zip
18+
dataset_name: "Atera FFPE Human Breast Cancer"
19+
dataset_url: "https://www.10xgenomics.com/datasets/atera-wta-ffpe-human-breast-cancer"
20+
dataset_summary: "Preview dataset showcasing the pre-commercial Atera Whole Transcriptome Assay (WTA) applied to FFPE human breast cancer tissue, profiling 18,028 genes and detecting 170,057 cells."
21+
dataset_description: "This human FFPE breast cancer data showcases results using the pre-commercial version of the Atera Whole Transcriptome Assay (WTA), which is currently under development. The assay is designed to closely match the Chromium Flex Apex assay in terms of content and sensitivity, and includes 18,028 genes. A single 5µm FFPE section of breast cancer tissue (DCIS Grade 3, T1c N0 M0) was analyzed, yielding 170,057 detected cells with a median of 2,116 transcripts per cell and 624,095,990 total high-quality decoded transcripts across 58.9 million µm² of tissue area. Output files are formatted to closely resemble Xenium Onboard Analysis v4 file formats."
22+
dataset_organism: "homo_sapiens"
23+
segmentation_id: [cell, nucleus]
24+
25+
output_dataset: "\$id/dataset.zarr"
26+
output_state: "\$id/state.yaml"
27+
publish_dir: "$publish_dir"
28+
HERE
29+
30+
tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \
31+
--revision build/main \
32+
--pull-latest \
33+
--main-script target/nextflow/datasets/workflows/process_tenx_xenium/main.nf \
34+
--workspace 53907369739130 \
35+
--params-file /tmp/params.yaml \
36+
--config common/nextflow_helpers/labels_tw.config \
37+
--labels datasets,atera

‎scripts/create_test_resources/2026_10x_human_breast_atera.sh‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#!/bin/bash
2+
3+
# get the root of the directory
4+
REPO_ROOT=$(git rev-parse --show-toplevel)
5+
6+
# ensure that the command below is run from the root of the repository
7+
cd "$REPO_ROOT"
8+
9+
set -e
10+
11+
if [ ! -d temp/datasets/10x_xenium/2026_10x_human_breast_atera ]; then
12+
mkdir -p temp/datasets/10x_xenium/2026_10x_human_breast_atera
13+
fi
14+
if [ ! -f temp/datasets/10x_xenium/2026_10x_human_breast_atera/WTA_Preview_FFPE_Breast_Cancer_xe_outs.zip ]; then
15+
wget -O temp/datasets/10x_xenium/2026_10x_human_breast_atera/WTA_Preview_FFPE_Breast_Cancer_xe_outs.zip \
16+
https://s3-us-west-2.amazonaws.com/10x.files/samples/atera/dev/WTA_Preview_FFPE_Breast_Cancer/WTA_Preview_FFPE_Breast_Cancer_xe_outs.zip
17+
fi
18+
19+
cat > /tmp/params.yaml << HERE
20+
param_list:
21+
- id: 2026_10x_human_breast_atera
22+
input: temp/datasets/10x_xenium/2026_10x_human_breast_atera/WTA_Preview_FFPE_Breast_Cancer_xe_outs.zip
23+
segmentation_id:
24+
- cell
25+
- nucleus
26+
dataset_name: "Atera FFPE Human Breast Cancer"
27+
dataset_url: "https://www.10xgenomics.com/datasets/atera-wta-ffpe-human-breast-cancer"
28+
dataset_summary: "Preview dataset showcasing the pre-commercial Atera Whole Transcriptome Assay (WTA) applied to FFPE human breast cancer tissue, profiling 18,028 genes and detecting 170,057 cells."
29+
dataset_description: "This human FFPE breast cancer data showcases results using the pre-commercial version of the Atera Whole Transcriptome Assay (WTA), which is currently under development. The assay is designed to closely match the Chromium Flex Apex assay in terms of content and sensitivity, and includes 18,028 genes. A single 5µm FFPE section of breast cancer tissue (DCIS Grade 3, T1c N0 M0) was analyzed, yielding 170,057 detected cells with a median of 2,116 transcripts per cell and 624,095,990 total high-quality decoded transcripts across 58.9 million µm² of tissue area. Output files are formatted to closely resemble Xenium Onboard Analysis v4 file formats."
30+
dataset_organism: homo_sapiens
31+
crop_region_min_x: 5000
32+
crop_region_max_x: 6000
33+
crop_region_min_y: 5000
34+
crop_region_max_y: 6000
35+
36+
publish_dir: resources_test/common
37+
output_dataset: '\$id/dataset.zarr'
38+
output_state: '\$id/state.yaml'
39+
HERE
40+
41+
# convert to zarr
42+
nextflow run . \
43+
-main-script target/nextflow/datasets/workflows/process_tenx_xenium/main.nf \
44+
-profile docker \
45+
-resume \
46+
-params-file /tmp/params.yaml
47+
48+
# sync to s3
49+
aws s3 sync --profile op \
50+
"resources_test/common/2026_10x_human_breast_atera" \
51+
"s3://openproblems-data/resources_test/common/2026_10x_human_breast_atera" \
52+
--delete --dryrun

‎src/datasets/loaders/tenx_atera/config.vsh.yaml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
name: tenx_atera
2+
namespace: datasets/loaders
3+
4+
argument_groups:
5+
- name: Inputs
6+
arguments:
7+
- type: string
8+
name: --input
9+
required: true
10+
description: A 10x atera directory or zip file or download url
11+
- type: string
12+
name: --segmentation_id
13+
required: true
14+
description: The segmentation identifier
15+
multiple: true
16+
- name: Metadata
17+
arguments:
18+
- type: string
19+
name: --dataset_id
20+
description: "A unique identifier for the dataset"
21+
required: true
22+
- name: --dataset_name
23+
type: string
24+
description: Nicely formatted name.
25+
required: true
26+
- type: string
27+
name: --dataset_url
28+
description: Link to the original source of the dataset.
29+
required: false
30+
- name: --dataset_reference
31+
type: string
32+
description: Bibtex reference of the paper in which the dataset was published.
33+
required: false
34+
- name: --dataset_summary
35+
type: string
36+
description: Short description of the dataset.
37+
required: true
38+
- name: --dataset_description
39+
type: string
40+
description: Long description of the dataset.
41+
required: true
42+
- name: --dataset_organism
43+
type: string
44+
description: The organism of the sample in the dataset.
45+
required: false
46+
- name: Outputs
47+
arguments:
48+
- name: "--output"
49+
__merge__: /src/api/file_common_ist.yaml
50+
direction: output
51+
required: true
52+
53+
resources:
54+
- type: python_script
55+
path: script.py
56+
57+
engines:
58+
- type: docker
59+
image: openproblems/base_python:1
60+
setup:
61+
- type: python
62+
pypi:
63+
- spatialdata-io
64+
- type: native
65+
66+
runners:
67+
- type: executable
68+
- type: nextflow
69+
directives:
70+
label: [midmem, midcpu, midtime]

‎src/datasets/loaders/tenx_atera/script.py‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
import spatialdata as sd
2+
import anndata as ad
3+
from spatialdata_io import xenium
4+
import shutil
5+
import os
6+
import zipfile
7+
import tempfile
8+
9+
## VIASH START
10+
par = {
11+
"input": "https://s3-us-west-2.amazonaws.com/10x.files/samples/atera/dev/WTA_Preview_FFPE_Breast_Cancer/WTA_Preview_FFPE_Breast_Cancer_xe_outs.zip",
12+
"segmentation_id": [
13+
"cell",
14+
"nucleus",
15+
],
16+
"dataset_id": "value",
17+
"dataset_name": "value",
18+
"dataset_url": "value",
19+
"dataset_reference": "value",
20+
"dataset_summary": "value",
21+
"dataset_description": "value",
22+
"dataset_organism": "value",
23+
"output": "temp/datasets/10x_atera/breast/breast.zarr"
24+
}
25+
meta = {
26+
"cpus": 1,
27+
}
28+
29+
## VIASH END
30+
31+
# Download the data if it's a download url, extract the data if it's a zip file
32+
par_input = par["input"]
33+
with tempfile.TemporaryDirectory() as tmpdirname:
34+
if par_input.startswith("http"):
35+
print(f"Downloading data to {tmpdirname}", flush=True)
36+
file_name = par_input.split("/")[-1]
37+
# download the data
38+
os.system(f"wget {par['input']} -O {tmpdirname}/{file_name}")
39+
par_input = tmpdirname + "/" + file_name
40+
41+
if zipfile.is_zipfile(par_input):
42+
print(f"Extracting input zip to {tmpdirname}", flush=True)
43+
with zipfile.ZipFile(par_input, "r") as zip_ref:
44+
zip_ref.extractall(tmpdirname)
45+
par_input = tmpdirname
46+
47+
# read the data
48+
sdata = xenium(
49+
path=par_input,
50+
n_jobs=meta["cpus"] or 1,
51+
cells_boundaries=True,
52+
nucleus_boundaries=True,
53+
morphology_focus=True,
54+
cells_as_circles=False,
55+
)
56+
57+
# remove morphology_focus
58+
_ = sdata.images.pop("morphology_focus")
59+
60+
print("Add uns to table", flush=True)
61+
new_uns = {
62+
"dataset_id": par["dataset_id"],
63+
"dataset_name": par["dataset_name"],
64+
"dataset_url": par["dataset_url"],
65+
"dataset_reference": par["dataset_reference"],
66+
"dataset_summary": par["dataset_summary"],
67+
"dataset_description": par["dataset_description"],
68+
"dataset_organism": par["dataset_organism"],
69+
"segmentation_id": par["segmentation_id"],
70+
}
71+
for key, value in new_uns.items():
72+
sdata.tables["table"].uns[key] = value
73+
74+
print(f"Output: {sdata}", flush=True)
75+
76+
print(f"Writing to '{par['output']}'", flush=True)
77+
if os.path.exists(par["output"]):
78+
shutil.rmtree(par["output"])
79+
80+
print(f"Output: {sdata}")
81+
82+
sdata.write(par["output"])

0 commit comments

Comments
 (0)