Update test resources

CHANGELOG.md

@@ -1,7 +1,16 @@
 # openpipeline_composed x.x.x
 
+## NEW FUNCTIONALITY
+
 * `workflows/single_cell/parallel_annotation`: Add consensus voting based on per-method weighted probabilities via `--run_consensus` flag (default is true) (PR #22).
 
+## MINOR CHANGES
+
+* Migration of test resources to the package-specific `s3://openpipelines-bio/openpipeline_composed/resources_test` bucket (#24):
+
+  - Add `.info.test_resources` to `_viash.yaml` to specify where test resources need to be synced from.
+  - Test resources were regenerated via the scripts in `resources_test_scripts/`.
+
 # openpipeline_composed 0.2.1
 
 ## MINOR CHANGES

_viash.yaml

@@ -27,7 +27,7 @@ repositories:
 info:
   test_resources:
     - type: s3
-      path: s3://openpipelines-bio/openpipeline_incubator/resources_test
+      path: s3://openpipelines-bio/openpipeline_composed/resources_test
       dest: resources_test
 
 config_mods: |

resources_test_scripts/10x_5k_anticmv.sh

@@ -69,102 +69,9 @@ if [[ ! -f "$feature_reference" ]]; then
   wget "https://cf.10xgenomics.com/samples/cell-vdj/6.1.2/5k_human_antiCMV_T_TBNK_connect_Multiplex/5k_human_antiCMV_T_TBNK_connect_Multiplex_count_feature_reference.csv" -O "$feature_reference"
 fi
 
-# download vdj reference if needed
-vdj_ref="$raw_dir/refdata-cellranger-vdj-GRCh38-alts-ensembl-7.0.0.tar.gz"
-if [[ ! -f "$vdj_ref" ]]; then
-  wget "https://cf.10xgenomics.com/supp/cell-vdj/refdata-cellranger-vdj-GRCh38-alts-ensembl-7.0.0.tar.gz" -O "$vdj_ref"
-fi
-
-
-# Run mapping pipeline
-cat > /tmp/params.yaml << HERE
-param_list:
-- id: "$ID"
-  input: "$raw_dir"
-  library_id:
-    - "${orig_sample_id}_GEX_1_subset"
-    - "${orig_sample_id}_AB_subset"
-    - "${orig_sample_id}_VDJ_subset"
-  library_type:
-    - "Gene Expression"
-    - "Antibody Capture"
-    - "VDJ"
-
-gex_reference: "$genome_tar"
-vdj_reference: "$vdj_ref"
-feature_reference: "$feature_reference"
-HERE
-
-nextflow \
-  run https://packages.viash-hub.com/vsh/openpipeline \
-  -r v4.0.4 \
-  -main-script target/nextflow/mapping/cellranger_multi/main.nf \
-  -resume \
-  --publish_dir "${OUT}_v10/processed" \
-  -profile docker,mount_temp \
-  -params-file /tmp/params.yaml \
-  -c ./src/configs/labels_ci.config 
-
-# Convert to h5mu
-cat > /tmp/params.yaml << HERE
-id: "$orig_sample_id"
-input: "$OUT/processed/10x_5k_anticmv.cellranger_multi.output"
-publish_dir: "$OUT/"
-output: "*.h5mu"
-HERE
-
-nextflow \
-  run https://packages.viash-hub.com/vsh/openpipeline \
-  -r v4.0.4 \
-  -main-script target/nextflow/convert/from_cellranger_multi_to_h5mu/main.nf \
-  -resume \
-  -profile docker,mount_temp \
-  -params-file /tmp/params.yaml \
-  -c ./src/configs/labels_ci.config
-
-mv "$OUT/0.h5mu" "$OUT/${orig_sample_id}.h5mu"
-
-
-# run qc workflow
-cat > /tmp/params.yaml << HERE
-id: "$ID"
-input: "$OUT/$orig_sample_id.h5mu"
-var_name_mitochondrial_genes: mitochondrial
-var_name_ribosomal_genes: ribosomal
-publish_dir: "$OUT/"
-output: "${orig_sample_id}_qc.h5mu"
-HERE
-
-nextflow \
-  run https://packages.viash-hub.com/vsh/openpipeline \
-  -r v4.0.4 \
-  -main-script target/nextflow/workflows/qc/qc/main.nf \
-  -resume \
-  -profile docker,mount_temp \
-  -params-file /tmp/params.yaml \
-  -c ./src/configs/labels_ci.config
-
-
-# Run full pipeline
-cat > /tmp/params.yaml << HERE
-id: "$ID"
-input: "$OUT/${orig_sample_id}_qc.h5mu"
-publish_dir: "$OUT/"
-output: "${orig_sample_id}_mms.h5mu"
-HERE
-
-nextflow \
-  run https://packages.viash-hub.com/vsh/openpipeline \
-  -r v4.0.4 \
-  -main-script target/nextflow/workflows/multiomics/process_samples/main.nf \
-  -resume \
-  -profile docker,mount_temp \
-  -params-file /tmp/params.yaml \
-  -c ./src/configs/labels_ci.config
 
 aws s3 sync \
   "$OUT" \
-  s3://openpipelines-bio/openpipeline_incubator/resources_test/"$ID" \
-  --exclude "*.yaml" \
+  s3://openpipelines-bio/openpipeline_composed/resources_test/"$ID" \
   --delete \
   --dryrun

resources_test_scripts/annotation_test_data.sh

@@ -20,29 +20,19 @@ wget "https://zenodo.org/record/7587774/files/TS_Blood_filtered.h5ad?download=1"
 # Download Tabula Sapiens Blood pretrained model from https://doi.org/10.5281/zenodo.7580707
 wget "https://zenodo.org/record/7580707/files/pretrained_models_Blood_ts.tar.gz?download=1" -O "${OUT}/tmp_pretrained_models_Blood_ts.tar.gz"
 
-# Download PopV specific CL ontology files - needed for OnClass
-# OUT_ONTOLOGY="${OUT}/ontology"
-# [ -d "$OUT_ONTOLOGY" ] || mkdir -p "$OUT_ONTOLOGY"
-# wget https://raw.githubusercontent.com/czbiohub/PopV/main/ontology/cl.obo \
-# -O "${OUT_ONTOLOGY}/cl.obo"
-# wget https://raw.githubusercontent.com/czbiohub/PopV/main/ontology/cl.ontology \
-# -O "${OUT_ONTOLOGY}/cl.ontology"
-# wget https://raw.githubusercontent.com/czbiohub/PopV/main/ontology/cl.ontology.nlp.emb \
-# -O "${OUT_ONTOLOGY}/cl.ontology.nlp.emb"
-
 
 # Process Tabula Sapiens Blood reference h5ad
-# (Select one individual and 100 cells per cell type)
+# Select one individual and 100 cells per cell type
+# Add major types
 # normalize and log1p transform data
-# Add treatment and disease columns
 python <<HEREDOC
 import anndata as ad
 import scanpy as sc
 import numpy as np
 
 # Read in data
 ref_adata = ad.read_h5ad("${OUT}/tmp_TS_Blood_filtered.h5ad")
-sub_ref_adata = ref_adata[ref_adata.obs["donor_assay"] == "TSP14_10x 3' v3"] 
+sub_ref_adata = ref_adata[ref_adata.obs["donor_assay"] == "TSP14_10x 3' v3"]
 n=100
 s=sub_ref_adata.obs.groupby('cell_ontology_class').cell_ontology_class.transform('count')
 sub_ref_adata_final = sub_ref_adata[sub_ref_adata.obs[s>=n].groupby('cell_ontology_class').head(n).index]
@@ -58,109 +48,138 @@ sc.pp.log1p(
 )
 sub_ref_adata_final.layers["log_normalized"] = data_for_scanpy.X
 
-# Add treatment and disease columns
-n_cells = sub_ref_adata_final.n_obs
-treatment = np.random.choice(["ctrl", "stim"], size=n_cells, p=[0.5, 0.5])
-disease = np.random.choice(["healthy", "diseased"], size=n_cells, p=[0.5, 0.5])
-sub_ref_adata_final.obs["treatment"] = treatment
-sub_ref_adata_final.obs["disease"] = disease
+# Add a two-level cell-type hierarchy for subtyping tests: every fine cell type
+# (subtype) is nested under exactly one major cell type. The subtyping workflow
+# splits the reference on major_cell_type and subtypes each major independently
+# into its cell_type labels.
+major_cell_type_map = {
+    "classical monocyte": "myeloid",
+    "neutrophil": "myeloid",
+    "erythrocyte": "myeloid",
+    "plasma cell": "lymphoid",
+}
+major_cell_type = sub_ref_adata_final.obs["cell_type"].astype(str).map(major_cell_type_map)
+unmapped = sorted(sub_ref_adata_final.obs["cell_type"].astype(str)[major_cell_type.isna()].unique())
+assert not unmapped, f"cell types missing a major_cell_type mapping: {unmapped}"
+sub_ref_adata_final.obs["major_cell_type"] = major_cell_type.astype("category")
+
+# The var index shares its name ("feature_name") with a var column of the same
+# name. anndata refuses to write a MuData when the index name collides with a
+# differently-typed column, so clear the index name before conversion.
+sub_ref_adata_final.var.index.name = None
 
 # Write out data
 sub_ref_adata_final.write("${OUT}/TS_Blood_filtered.h5ad", compression='gzip')
 HEREDOC
 
 
 echo "> Converting to h5mu"
-viash run src/convert/from_h5ad_to_h5mu/config.vsh.yaml --engine docker -- \
-    --input "${OUT}/TS_Blood_filtered.h5ad" \
-    --output "${OUT}/TS_Blood_filtered.h5mu" \
-    --modality "rna"
-
-rm "${OUT}/tmp_TS_Blood_filtered.h5ad"
-
-echo "> Downloading pretrained CellTypist model and sample test data"
-wget https://celltypist.cog.sanger.ac.uk/models/Pan_Immune_CellTypist/v2/Immune_All_Low.pkl \
-    -O "${OUT}/celltypist_model_Immune_All_Low.pkl"
+nextflow \
+  run https://packages.viash-hub.com/vsh/openpipeline \
+  -r v4.1.1 \
+  -main-script target/nextflow/convert/from_h5ad_to_h5mu/main.nf \
+  -profile docker,mount_temp \
+  -c ./src/configs/labels_ci.config \
+  --publish_dir "${OUT}" \
+  --input "${OUT}/TS_Blood_filtered.h5ad" \
+  --output "TS_Blood_filtered.h5mu" \
+  --modality "rna"
+
+
+echo "> Downloading sample test data"
 wget https://celltypist.cog.sanger.ac.uk/Notebook_demo_data/demo_2000_cells.h5ad \
     -O "${OUT}/demo_2000_cells.h5ad"
-viash run src/convert/from_h5ad_to_h5mu/config.vsh.yaml --engine docker -- \
-    --input "${OUT}/demo_2000_cells.h5ad" \
-    --output "${OUT}/demo_2000_cells.h5mu" \
-    --modality "rna"
-
 
-echo "> Fetching OnClass data and models"
-OUT_ONTOLOGY="${OUT}/ontology"
-[ -d "$OUT_ONTOLOGY" ] || mkdir -p "$OUT_ONTOLOGY"
-wget https://figshare.com/ndownloader/files/28394466 -O "${OUT_ONTOLOGY}/OnClass_data_public_minimal.tar.gz"
-tar -xzvf "${OUT_ONTOLOGY}/OnClass_data_public_minimal.tar.gz" -C "${OUT_ONTOLOGY}" --strip-components=2
-rm "${OUT_ONTOLOGY}/allen.ontology"
-rm "${OUT_ONTOLOGY}/OnClass_data_public_minimal.tar.gz"
-
-wget https://figshare.com/ndownloader/files/28394541 -O "${OUT}/OnClass_models.tar.gz"
-tar -xzvf "${OUT}/OnClass_models.tar.gz" -C "${OUT}" --strip-components=1
-rm "${OUT}/OnClass_models.tar.gz"
-rm "${OUT}/tmp_pretrained_models_Blood_ts.tar.gz"
+echo "> Converting to h5mu"
+nextflow \
+  run https://packages.viash-hub.com/vsh/openpipeline \
+  -r v4.1.1 \
+  -main-script target/nextflow/convert/from_h5ad_to_h5mu/main.nf \
+  -profile docker,mount_temp \
+  -c ./src/configs/labels_ci.config \
+  --publish_dir "${OUT}" \
+  --input "${OUT}/demo_2000_cells.h5ad" \
+  --output "demo_2000_cells.h5mu" \
+  --modality "rna"
 
-find "${OUT}/Pretrained_model" ! -name "example_file_model*" -type f -exec rm -f {} +
-mv "${OUT}/Pretrained_model" "${OUT}/onclass_model"
 
 echo "> Creating simple SCVI model"
-viash run src/integrate/scvi/config.vsh.yaml --engine docker -- \
-    --input "${OUT}/TS_Blood_filtered.h5mu" \
-    --obs_batch "donor_id" \
-    --var_gene_names "ensemblid" \
-    --output "${OUT}/scvi_output.h5mu" \
-    --output_model "${OUT}/scvi_model" \
-    --max_epochs 5 \
-    --n_obs_min_count 10 \
-    --n_var_min_count 10
+nextflow \
+  run https://packages.viash-hub.com/vsh/openpipeline \
+  -r v4.1.1 \
+  -main-script target/nextflow/integrate/scvi/main.nf \
+  -profile docker,mount_temp \
+  -c ./src/configs/labels_ci.config \
+  --publish_dir "${OUT}" \
+  --input "${OUT}/TS_Blood_filtered.h5mu" \
+  --obs_batch "donor_id" \
+  --var_gene_names "ensemblid" \
+  --output "scvi_output.h5mu" \
+  --output_model "scvi_model" \
+  --max_epochs 5 \
+  --n_obs_min_count 10 \
+  --n_var_min_count 10
 
 echo "> Creating SCVI model with covariates"
-viash run src/integrate/scvi/config.vsh.yaml --engine docker -- \
-    --input "${OUT}/TS_Blood_filtered.h5mu" \
-    --obs_batch "donor_id" \
-    --var_gene_names "ensemblid" \
-    --obs_categorical_covariate "assay" \
-    --obs_categorical_covariate "donor_assay" \
-    --output "${OUT}/scvi_covariate_output.h5mu" \
-    --output_model "${OUT}/scvi_covariate_model" \
-    --max_epochs 5 \
-    --n_obs_min_count 10 \
-    --n_var_min_count 10
+nextflow \
+  run https://packages.viash-hub.com/vsh/openpipeline \
+  -r v4.1.1 \
+  -main-script target/nextflow/integrate/scvi/main.nf \
+  -profile docker,mount_temp \
+  -c ./src/configs/labels_ci.config \
+  --publish_dir "${OUT}" \
+  --input "${OUT}/TS_Blood_filtered.h5mu" \
+  --obs_batch "donor_id" \
+  --obs_categorical_covariate "assay;donor_assay" \
+  --var_gene_names "ensemblid" \
+  --output "scvi_covariate_output.h5mu" \
+  --output_model "scvi_covariate_model" \
+  --max_epochs 5 \
+  --n_obs_min_count 10 \
+  --n_var_min_count 10
 
 echo "> Creating simple SCANVI model"
-viash run src/annotate/scanvi/config.vsh.yaml --engine docker -- \
-    --input "${OUT}/TS_Blood_filtered.h5mu" \
-    --var_gene_names "ensemblid" \
-    --obs_labels "cell_ontology_class" \
-    --scvi_model "${OUT}/scvi_model" \
-    --output "${OUT}/scanvi_output.h5mu" \
-    --output_model "${OUT}/scanvi_model" \
-    --max_epochs 5 
+nextflow \
+  run https://packages.viash-hub.com/vsh/openpipeline \
+  -r v4.1.1 \
+  -main-script target/nextflow/annotate/scanvi/main.nf \
+  -profile docker,mount_temp \
+  -c ./src/configs/labels_ci.config \
+  --publish_dir "${OUT}" \
+  --input "${OUT}/TS_Blood_filtered.h5mu" \
+  --scvi_model "${OUT}/scvi_model" \
+  --obs_labels "cell_ontology_class" \
+  --var_gene_names "ensemblid" \
+  --output "scanvi_output.h5mu" \
+  --output_model "scanvi_model" \
+  --max_epochs 5
 
 echo "> Creating SCANVI model with covariates"
-viash run src/annotate/scanvi/config.vsh.yaml --engine docker -- \
-    --input "${OUT}/TS_Blood_filtered.h5mu" \
-    --var_gene_names "ensemblid" \
-    --obs_labels "cell_ontology_class" \
-    --scvi_model "${OUT}/scvi_covariate_model" \
-    --output "${OUT}/scanvi_covariate_output.h5mu" \
-    --output_model "${OUT}/scanvi_covariate_model" \
-    --max_epochs 5 
+nextflow \
+  run https://packages.viash-hub.com/vsh/openpipeline \
+  -r v4.1.1 \
+  -main-script target/nextflow/annotate/scanvi/main.nf \
+  -profile docker,mount_temp \
+  -c ./src/configs/labels_ci.config \
+  --publish_dir "${OUT}" \
+  --input "${OUT}/TS_Blood_filtered.h5mu" \
+  --scvi_model "${OUT}/scvi_covariate_model" \
+  --obs_labels "cell_ontology_class" \
+  --var_gene_names "ensemblid" \
+  --output "scanvi_covariate_output.h5mu" \
+  --output_model "scanvi_covariate_model" \
+  --max_epochs 5
+
 
 rm "${OUT}/scanvi_output.h5mu"
 rm "${OUT}/scanvi_covariate_output.h5mu"
 rm "${OUT}/scvi_output.h5mu"
 rm "${OUT}/scvi_covariate_output.h5mu"
-rm -r "${OUT}/Pretrained_model/"
-
-echo "> Creating Pseudobulk Data for DGEA"
-viash run src/differential_expression/create_pseudobulk/config.vsh.yaml --engine docker -- \
-    --input "${OUT}/TS_Blood_filtered.h5mu" \
-    --obs_grouping "cell_type" \
-    --obs_sample_conditions "donor_id" \
-    --obs_sample_conditions "treatment" \
-    --obs_sample_conditions "disease" \
-    --min_num_cells_per_sample 5 \
-    --output "${OUT}/TS_Blood_filtered_pseudobulk.h5mu" 
+rm -f "${OUT}"/*.h5ad
+rm -f "${OUT}"/*.state.yaml
+
+aws s3 sync \
+  "$OUT" \
+  s3://openpipelines-bio/openpipeline_composed/resources_test/"$ID" \
+  --delete \
+  --dryrun