From 771e4e41200765d1982988c6589e1e42370dccae Mon Sep 17 00:00:00 2001 From: dorien-er Date: Thu, 2 Jul 2026 15:48:55 +0200 Subject: [PATCH 01/13] update test resources --- _viash.yaml | 2 +- resources_test_scripts/10x_5k_anticmv.sh | 95 +------ .../annotation_test_data.sh | 241 +++++++++++------- resources_test_scripts/pbmc_1k_protein_v3.sh | 137 +++------- resources_test_scripts/qc_sample_data.sh | 166 ------------ .../reference_gencodev41.sh | 14 +- .../spatial_qc_sample_data.sh | 37 --- 7 files changed, 183 insertions(+), 509 deletions(-) delete mode 100755 resources_test_scripts/qc_sample_data.sh delete mode 100755 resources_test_scripts/spatial_qc_sample_data.sh diff --git a/_viash.yaml b/_viash.yaml index 3c40013..5e055b4 100644 --- a/_viash.yaml +++ b/_viash.yaml @@ -27,7 +27,7 @@ repositories: info: test_resources: - type: s3 - path: s3://openpipelines-bio/openpipeline_incubator/resources_test + path: s3://openpipelines-bio/openpipeline_composed/resources_test dest: resources_test config_mods: | diff --git a/resources_test_scripts/10x_5k_anticmv.sh b/resources_test_scripts/10x_5k_anticmv.sh index aee5c20..1a72c64 100644 --- a/resources_test_scripts/10x_5k_anticmv.sh +++ b/resources_test_scripts/10x_5k_anticmv.sh @@ -69,102 +69,9 @@ if [[ ! -f "$feature_reference" ]]; then wget "https://cf.10xgenomics.com/samples/cell-vdj/6.1.2/5k_human_antiCMV_T_TBNK_connect_Multiplex/5k_human_antiCMV_T_TBNK_connect_Multiplex_count_feature_reference.csv" -O "$feature_reference" fi -# download vdj reference if needed -vdj_ref="$raw_dir/refdata-cellranger-vdj-GRCh38-alts-ensembl-7.0.0.tar.gz" -if [[ ! -f "$vdj_ref" ]]; then - wget "https://cf.10xgenomics.com/supp/cell-vdj/refdata-cellranger-vdj-GRCh38-alts-ensembl-7.0.0.tar.gz" -O "$vdj_ref" -fi - - -# Run mapping pipeline -cat > /tmp/params.yaml << HERE -param_list: -- id: "$ID" - input: "$raw_dir" - library_id: - - "${orig_sample_id}_GEX_1_subset" - - "${orig_sample_id}_AB_subset" - - "${orig_sample_id}_VDJ_subset" - library_type: - - "Gene Expression" - - "Antibody Capture" - - "VDJ" - -gex_reference: "$genome_tar" -vdj_reference: "$vdj_ref" -feature_reference: "$feature_reference" -HERE - -nextflow \ - run https://packages.viash-hub.com/vsh/openpipeline \ - -r v4.0.4 \ - -main-script target/nextflow/mapping/cellranger_multi/main.nf \ - -resume \ - --publish_dir "${OUT}_v10/processed" \ - -profile docker,mount_temp \ - -params-file /tmp/params.yaml \ - -c ./src/configs/labels_ci.config - -# Convert to h5mu -cat > /tmp/params.yaml << HERE -id: "$orig_sample_id" -input: "$OUT/processed/10x_5k_anticmv.cellranger_multi.output" -publish_dir: "$OUT/" -output: "*.h5mu" -HERE - -nextflow \ - run https://packages.viash-hub.com/vsh/openpipeline \ - -r v4.0.4 \ - -main-script target/nextflow/convert/from_cellranger_multi_to_h5mu/main.nf \ - -resume \ - -profile docker,mount_temp \ - -params-file /tmp/params.yaml \ - -c ./src/configs/labels_ci.config - -mv "$OUT/0.h5mu" "$OUT/${orig_sample_id}.h5mu" - - -# run qc workflow -cat > /tmp/params.yaml << HERE -id: "$ID" -input: "$OUT/$orig_sample_id.h5mu" -var_name_mitochondrial_genes: mitochondrial -var_name_ribosomal_genes: ribosomal -publish_dir: "$OUT/" -output: "${orig_sample_id}_qc.h5mu" -HERE - -nextflow \ - run https://packages.viash-hub.com/vsh/openpipeline \ - -r v4.0.4 \ - -main-script target/nextflow/workflows/qc/qc/main.nf \ - -resume \ - -profile docker,mount_temp \ - -params-file /tmp/params.yaml \ - -c ./src/configs/labels_ci.config - - -# Run full pipeline -cat > /tmp/params.yaml << HERE -id: "$ID" -input: "$OUT/${orig_sample_id}_qc.h5mu" -publish_dir: "$OUT/" -output: "${orig_sample_id}_mms.h5mu" -HERE - -nextflow \ - run https://packages.viash-hub.com/vsh/openpipeline \ - -r v4.0.4 \ - -main-script target/nextflow/workflows/multiomics/process_samples/main.nf \ - -resume \ - -profile docker,mount_temp \ - -params-file /tmp/params.yaml \ - -c ./src/configs/labels_ci.config aws s3 sync \ "$OUT" \ - s3://openpipelines-bio/openpipeline_incubator/resources_test/"$ID" \ - --exclude "*.yaml" \ + s3://openpipelines-bio/openpipeline_composed/resources_test/"$ID" \ --delete \ --dryrun diff --git a/resources_test_scripts/annotation_test_data.sh b/resources_test_scripts/annotation_test_data.sh index e989564..8c2f55c 100644 --- a/resources_test_scripts/annotation_test_data.sh +++ b/resources_test_scripts/annotation_test_data.sh @@ -20,21 +20,11 @@ wget "https://zenodo.org/record/7587774/files/TS_Blood_filtered.h5ad?download=1" # Download Tabula Sapiens Blood pretrained model from https://doi.org/10.5281/zenodo.7580707 wget "https://zenodo.org/record/7580707/files/pretrained_models_Blood_ts.tar.gz?download=1" -O "${OUT}/tmp_pretrained_models_Blood_ts.tar.gz" -# Download PopV specific CL ontology files - needed for OnClass -# OUT_ONTOLOGY="${OUT}/ontology" -# [ -d "$OUT_ONTOLOGY" ] || mkdir -p "$OUT_ONTOLOGY" -# wget https://raw.githubusercontent.com/czbiohub/PopV/main/ontology/cl.obo \ -# -O "${OUT_ONTOLOGY}/cl.obo" -# wget https://raw.githubusercontent.com/czbiohub/PopV/main/ontology/cl.ontology \ -# -O "${OUT_ONTOLOGY}/cl.ontology" -# wget https://raw.githubusercontent.com/czbiohub/PopV/main/ontology/cl.ontology.nlp.emb \ -# -O "${OUT_ONTOLOGY}/cl.ontology.nlp.emb" - # Process Tabula Sapiens Blood reference h5ad -# (Select one individual and 100 cells per cell type) +# Select one individual and 100 cells per cell type +# Add major types # normalize and log1p transform data -# Add treatment and disease columns python < Converting to h5mu" -viash run src/convert/from_h5ad_to_h5mu/config.vsh.yaml --engine docker -- \ - --input "${OUT}/TS_Blood_filtered.h5ad" \ - --output "${OUT}/TS_Blood_filtered.h5mu" \ - --modality "rna" - -rm "${OUT}/tmp_TS_Blood_filtered.h5ad" - -echo "> Downloading pretrained CellTypist model and sample test data" -wget https://celltypist.cog.sanger.ac.uk/models/Pan_Immune_CellTypist/v2/Immune_All_Low.pkl \ - -O "${OUT}/celltypist_model_Immune_All_Low.pkl" -wget https://celltypist.cog.sanger.ac.uk/Notebook_demo_data/demo_2000_cells.h5ad \ - -O "${OUT}/demo_2000_cells.h5ad" -viash run src/convert/from_h5ad_to_h5mu/config.vsh.yaml --engine docker -- \ - --input "${OUT}/demo_2000_cells.h5ad" \ - --output "${OUT}/demo_2000_cells.h5mu" \ - --modality "rna" - - -echo "> Fetching OnClass data and models" -OUT_ONTOLOGY="${OUT}/ontology" -[ -d "$OUT_ONTOLOGY" ] || mkdir -p "$OUT_ONTOLOGY" -wget https://figshare.com/ndownloader/files/28394466 -O "${OUT_ONTOLOGY}/OnClass_data_public_minimal.tar.gz" -tar -xzvf "${OUT_ONTOLOGY}/OnClass_data_public_minimal.tar.gz" -C "${OUT_ONTOLOGY}" --strip-components=2 -rm "${OUT_ONTOLOGY}/allen.ontology" -rm "${OUT_ONTOLOGY}/OnClass_data_public_minimal.tar.gz" - -wget https://figshare.com/ndownloader/files/28394541 -O "${OUT}/OnClass_models.tar.gz" -tar -xzvf "${OUT}/OnClass_models.tar.gz" -C "${OUT}" --strip-components=1 -rm "${OUT}/OnClass_models.tar.gz" -rm "${OUT}/tmp_pretrained_models_Blood_ts.tar.gz" - -find "${OUT}/Pretrained_model" ! -name "example_file_model*" -type f -exec rm -f {} + -mv "${OUT}/Pretrained_model" "${OUT}/onclass_model" +cat > /tmp/from_h5ad_to_h5mu.yaml << HERE +input: "${OUT}/TS_Blood_filtered.h5ad" +output: "TS_Blood_filtered.h5mu" +modality: "rna" +HERE + +nextflow \ + run https://packages.viash-hub.com/vsh/openpipeline \ + -r v4.1.1 \ + -main-script target/nextflow/convert/from_h5ad_to_h5mu/main.nf \ + --publish_dir "${OUT}" \ + -profile docker,mount_temp \ + -params-file /tmp/from_h5ad_to_h5mu.yaml \ + -c ./src/configs/labels_ci.config + + +# echo "> Downloading pretrained CellTypist model and sample test data" +# wget https://celltypist.cog.sanger.ac.uk/models/Pan_Immune_CellTypist/v2/Immune_All_Low.pkl \ +# -O "${OUT}/celltypist_model_Immune_All_Low.pkl" +# wget https://celltypist.cog.sanger.ac.uk/Notebook_demo_data/demo_2000_cells.h5ad \ +# -O "${OUT}/demo_2000_cells.h5ad" + +echo "> Converting to h5mu" +cat > /tmp/from_h5ad_to_h5mu_demo.yaml << HERE +input: "${OUT}/demo_2000_cells.h5ad" +output: "demo_2000_cells.h5mu" +modality: "rna" +HERE + +nextflow \ + run https://packages.viash-hub.com/vsh/openpipeline \ + -r v4.1.1 \ + -main-script target/nextflow/convert/from_h5ad_to_h5mu/main.nf \ + --publish_dir "${OUT}" \ + -profile docker,mount_temp \ + -params-file /tmp/from_h5ad_to_h5mu_demo.yaml \ + -c ./src/configs/labels_ci.config + echo "> Creating simple SCVI model" -viash run src/integrate/scvi/config.vsh.yaml --engine docker -- \ - --input "${OUT}/TS_Blood_filtered.h5mu" \ - --obs_batch "donor_id" \ - --var_gene_names "ensemblid" \ - --output "${OUT}/scvi_output.h5mu" \ - --output_model "${OUT}/scvi_model" \ - --max_epochs 5 \ - --n_obs_min_count 10 \ - --n_var_min_count 10 +cat > /tmp/simple_scvi.yaml << HERE +input: "${OUT}/TS_Blood_filtered.h5mu" +obs_batch: "donor_id" +var_gene_names: "ensemblid" +output: "scvi_output.h5mu" +output_model: "scvi_model" +max_epochs: 5 +n_obs_min_count: 10 +n_var_min_count: 10 +HERE + +nextflow \ + run https://packages.viash-hub.com/vsh/openpipeline \ + -r v4.1.1 \ + -main-script target/nextflow/integrate/scvi/main.nf \ + --publish_dir "${OUT}" \ + -profile docker,mount_temp \ + -params-file /tmp/simple_scvi.yaml \ + -c ./src/configs/labels_ci.config echo "> Creating SCVI model with covariates" -viash run src/integrate/scvi/config.vsh.yaml --engine docker -- \ - --input "${OUT}/TS_Blood_filtered.h5mu" \ - --obs_batch "donor_id" \ - --var_gene_names "ensemblid" \ - --obs_categorical_covariate "assay" \ - --obs_categorical_covariate "donor_assay" \ - --output "${OUT}/scvi_covariate_output.h5mu" \ - --output_model "${OUT}/scvi_covariate_model" \ - --max_epochs 5 \ - --n_obs_min_count 10 \ - --n_var_min_count 10 +cat > /tmp/covariates_scvi.yaml << HERE +input: "${OUT}/TS_Blood_filtered.h5mu" +obs_batch: "donor_id" +obs_categorical_covariate: "assay" +obs_categorical_covariate: "donor_assay" +var_gene_names: "ensemblid" +output: "scvi_covariate_output.h5mu" +output_model: "scvi_covariate_model" +max_epochs: 5 +n_obs_min_count: 10 +n_var_min_count: 10 +HERE + +nextflow \ + run https://packages.viash-hub.com/vsh/openpipeline \ + -r v4.1.1 \ + -main-script target/nextflow/integrate/scvi/main.nf \ + --publish_dir "${OUT}" \ + -profile docker,mount_temp \ + -params-file /tmp/covariates_scvi.yaml \ + -c ./src/configs/labels_ci.config echo "> Creating simple SCANVI model" -viash run src/annotate/scanvi/config.vsh.yaml --engine docker -- \ - --input "${OUT}/TS_Blood_filtered.h5mu" \ - --var_gene_names "ensemblid" \ - --obs_labels "cell_ontology_class" \ - --scvi_model "${OUT}/scvi_model" \ - --output "${OUT}/scanvi_output.h5mu" \ - --output_model "${OUT}/scanvi_model" \ - --max_epochs 5 +cat > /tmp/simple_scanvi.yaml << HERE +input: "${OUT}/TS_Blood_filtered.h5mu" +scvi_model: "${OUT}/scvi_model" +obs_labels: "cell_ontology_class" +var_gene_names: "ensemblid" +output: "scanvi_output.h5mu" +output_model: "scanvi_model" +max_epochs: 5 +HERE + +nextflow \ + run https://packages.viash-hub.com/vsh/openpipeline \ + -r v4.1.1 \ + -main-script target/nextflow/annotate/scanvi/main.nf \ + --publish_dir "${OUT}" \ + -profile docker,mount_temp \ + -params-file /tmp/simple_scanvi.yaml \ + -c ./src/configs/labels_ci.config echo "> Creating SCANVI model with covariates" -viash run src/annotate/scanvi/config.vsh.yaml --engine docker -- \ - --input "${OUT}/TS_Blood_filtered.h5mu" \ - --var_gene_names "ensemblid" \ - --obs_labels "cell_ontology_class" \ - --scvi_model "${OUT}/scvi_covariate_model" \ - --output "${OUT}/scanvi_covariate_output.h5mu" \ - --output_model "${OUT}/scanvi_covariate_model" \ - --max_epochs 5 +cat > /tmp/covariates_scanvi.yaml << HERE +input: "${OUT}/TS_Blood_filtered.h5mu" +scvi_model: "${OUT}/scvi_covariate_model" +obs_labels: "cell_ontology_class" +var_gene_names: "ensemblid" +output: "scanvi_covariate_output.h5mu" +output_model: "scanvi_covariate_model" +max_epochs: 5 +HERE + +nextflow \ + run https://packages.viash-hub.com/vsh/openpipeline \ + -r v4.1.1 \ + -main-script target/nextflow/annotate/scanvi/main.nf \ + --publish_dir "${OUT}" \ + -profile docker,mount_temp \ + -params-file /tmp/covariates_scanvi.yaml \ + -c ./src/configs/labels_ci.config + rm "${OUT}/scanvi_output.h5mu" rm "${OUT}/scanvi_covariate_output.h5mu" rm "${OUT}/scvi_output.h5mu" rm "${OUT}/scvi_covariate_output.h5mu" -rm -r "${OUT}/Pretrained_model/" - -echo "> Creating Pseudobulk Data for DGEA" -viash run src/differential_expression/create_pseudobulk/config.vsh.yaml --engine docker -- \ - --input "${OUT}/TS_Blood_filtered.h5mu" \ - --obs_grouping "cell_type" \ - --obs_sample_conditions "donor_id" \ - --obs_sample_conditions "treatment" \ - --obs_sample_conditions "disease" \ - --min_num_cells_per_sample 5 \ - --output "${OUT}/TS_Blood_filtered_pseudobulk.h5mu" +rm -f "${OUT}"/*.h5ad +rm -f "${OUT}"/*.state.yaml + +aws s3 sync \ + resources_test/annotation_test_data \ + s3://openpipelines-bio/openpipeline_composed/resources_test/annotation_test_data \ + --dryrun --delete \ No newline at end of file diff --git a/resources_test_scripts/pbmc_1k_protein_v3.sh b/resources_test_scripts/pbmc_1k_protein_v3.sh index 98ffd0f..3864efd 100644 --- a/resources_test_scripts/pbmc_1k_protein_v3.sh +++ b/resources_test_scripts/pbmc_1k_protein_v3.sh @@ -9,10 +9,9 @@ REPO_ROOT=$(git rev-parse --show-toplevel) cd "$REPO_ROOT" ID=pbmc_1k_protein_v3 -OUT=resources_test/$ID/$ID +OUT=resources_test/$ID DIR=$(dirname "$OUT") -# ideally, this would be a versioned pipeline run [ -d "$DIR" ] || mkdir -p "$DIR" # dataset page: @@ -26,126 +25,54 @@ wget https://cf.10xgenomics.com/samples/cell-exp/3.0.0/pbmc_1k_protein_v3/pbmc_1 wget https://cf.10xgenomics.com/samples/cell-exp/3.0.0/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5 \ -O "${OUT}_filtered_feature_bc_matrix.h5" -wget https://cf.10xgenomics.com/samples/cell-exp/3.0.0/pbmc_1k_protein_v3/pbmc_1k_protein_v3_raw_feature_bc_matrix.h5 \ - -O "${OUT}_raw_feature_bc_matrix.h5" - -# download counts matrix tar gz file -wget https://cf.10xgenomics.com/samples/cell-exp/3.0.0/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.tar.gz \ - -O "${OUT}_filtered_feature_bc_matrix.tar.gz" - -# extract matrix tar gz -mkdir -p "${OUT}_filtered_feature_bc_matrix" -tar -xvf "${OUT}_filtered_feature_bc_matrix.tar.gz" \ - -C "${OUT}_filtered_feature_bc_matrix" \ - --strip-components 1 -rm "${OUT}_filtered_feature_bc_matrix.tar.gz" - -cat > /tmp/params.yaml << HERE - --input "${OUT}_filtered_feature_bc_matrix.h5" \ - --input_metrics_summary "${OUT}_metrics_summary.csv" \ - --output "${OUT}_filtered_feature_bc_matrix.h5mu" - -param_list: - - id: "$ID" - genome_fasta: "https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_41/GRCh38.primary_assembly.genome.fa.gz" - transcriptome_gtf: "https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_41/gencode.v41.annotation.gtf.gz" - target: ["bd_rhapsody", "cellranger_arc"] - output_fasta: "reference.fa.gz" - output_gtf: "reference.gtf.gz" - non_nuclear_contigs: null - output_cellranger_arc: "reference_cellranger.tar.gz" - output_bd_rhapsody: "reference_bd_rhapsody.tar.gz" - bdrhap_extra_star_params: "--genomeSAindexNbases 12 --genomeSAsparseD 2" - motifs_file: "$motifs_modified" - subset_regex: "chr1" -HERE +# wget https://cf.10xgenomics.com/samples/cell-exp/3.0.0/pbmc_1k_protein_v3/pbmc_1k_protein_v3_raw_feature_bc_matrix.h5 \ +# -O "${OUT}_raw_feature_bc_matrix.h5" + +# # download counts matrix tar gz file +# wget https://cf.10xgenomics.com/samples/cell-exp/3.0.0/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.tar.gz \ +# -O "${OUT}_filtered_feature_bc_matrix.tar.gz" + +# # extract matrix tar gz +# mkdir -p "${OUT}_filtered_feature_bc_matrix" +# tar -xvf "${OUT}_filtered_feature_bc_matrix.tar.gz" \ +# -C "${OUT}_filtered_feature_bc_matrix" \ +# --strip-components 1 +# rm "${OUT}_filtered_feature_bc_matrix.tar.gz" + # convert 10x h5 to h5mu nextflow run https://packages.viash-hub.com/vsh/openpipeline \ -latest \ - -r v4.0.4 \ - -main-script target/docker/convert/from_10xh5_to_h5mu/from_10xh5_to_h5mu \ + -r v4.1.1 \ + -main-script target/nextflow/convert/from_10xh5_to_h5mu/main.nf \ -profile docker \ -c ./src/configs/labels_ci.config \ - -params-file /tmp/params.yaml \ --publish_dir $OUT \ + --input "${OUT}/${ID}_filtered_feature_bc_matrix.h5" \ + --input_metrics_summary "${OUT}/${ID}_metrics_summary.csv" \ + --output "${ID}_filtered_feature_bc_matrix.h5mu" \ -resume -# run single sample -nextflow \ - run . \ - -main-script target/nextflow/workflows/rna/rna_singlesample/main.nf \ - -c src/workflows/utils/labels_ci.config \ - -profile docker \ - --id pbmc_1k_protein_v3_uss \ - --input "${OUT}_filtered_feature_bc_matrix.h5mu" \ - --output "`basename $OUT`_uss.h5mu" \ - --publishDir `dirname $OUT` \ - -resume - -# add the sample ID to the mudata object +# run sample processing nextflow \ - run . \ - -main-script target/nextflow/metadata/add_id/main.nf \ - -c src/workflows/utils/labels_ci.config \ + run https://packages.viash-hub.com/vsh/openpipeline \ + -latest \ + -r v4.1.1 \ + -main-script target/nextflow/workflows/multiomics/process_samples/main.nf \ + -c ./src/configs/labels_ci.config \ -profile docker \ --id pbmc_1k_protein_v3_uss \ - --input "${OUT}_uss.h5mu" \ - --input_id "pbmc_1k_protein_v3_uss" \ - --output "`basename $OUT`_uss_with_id.h5mu" \ - --output_compression "gzip" \ - --publishDir `dirname $OUT` \ - -resume - -# run multisample -nextflow \ - run . \ - -main-script target/nextflow/workflows/rna/rna_multisample/main.nf \ - -c src/workflows/utils/labels_ci.config \ - -profile docker \ - --id pbmc_1k_protein_v3_ums \ - --input "${OUT}_uss_with_id.h5mu" \ - --output "`basename $OUT`_ums.h5mu" \ - --publishDir `dirname $OUT` \ - -resume - -rm "${OUT}_uss_with_id.h5mu" - -# run dimred -nextflow \ - run . \ - -main-script target/nextflow/workflows/multiomics/dimensionality_reduction/main.nf \ - -c src/workflows/utils/labels_ci.config \ - -profile docker \ - --id pbmc_1k_protein_v3_mms \ - --input "${OUT}_ums.h5mu" \ - --output "`basename $OUT`_mms.h5mu" \ - --publishDir `dirname $OUT` \ - --obs_covariates sample_id \ - -resume - -# run integration -nextflow \ - run . \ - -main-script target/nextflow/workflows/integration/harmony_leiden/main.nf \ - -c src/workflows/utils/labels_ci.config \ - -profile docker \ - --id pbmc_1k_protein_v3_mms_integration \ - --input "${OUT}_mms.h5mu" \ - --output "`basename $OUT`_mms.h5mu" \ - --publishDir `dirname $OUT` \ - --obs_covariates sample_id \ + --input "${OUT}/${ID}_filtered_feature_bc_matrix.h5mu" \ + --output "${ID}_mms.h5mu" \ + --publishDir "$OUT" \ -resume -python < /tmp/params_create_h5mu.yaml < /tmp/params_subset.yaml < /tmp/add_metadata_obs.py < /tmp/params_cellbender.yaml < /tmp/qc.yaml < Date: Mon, 6 Jul 2026 10:57:50 +0200 Subject: [PATCH 02/13] add temporary integration test workflow --- .github/workflows/integration-test.yml | 141 +++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 .github/workflows/integration-test.yml diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml new file mode 100644 index 0000000..b71fc5a --- /dev/null +++ b/.github/workflows/integration-test.yml @@ -0,0 +1,141 @@ +name: integration test +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +on: + push: + branches: [update-test-resources] + workflow_dispatch: + inputs: + push_containers: + type: boolean + required: false + default: true + description: Build docker images and push them to the registry + schedule: + - cron: '33 2 * * *' + +jobs: + # Build and create containers + build: + uses: ./.github/workflows/main-build.yml + with: + push_containers: ${{ github.event_name == 'schedule' || inputs.push_containers }} + version: 'integration_build' + target_tag: 'integration_build' + deploy_branch: 'integration_build' + secrets: inherit + + # Synchronize S3 Bucket and create cache for per-component runs + sync_s3: + env: + s3_bucket: s3://openpipelines-data/ + runs-on: ubuntu-latest + + outputs: + cache_key: ${{ steps.cache.outputs.cache_key }} + dest_paths: ${{ steps.cache.outputs.dest_paths }} + + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.' + - uses: data-intuitive/reclaim-the-bytes@v2 + - uses: viash-io/viash-actions/project/sync-and-cache@v6 + id: cache + + # phase 3 + integration_test: + runs-on: ubuntu-latest + needs: [ build, sync_s3 ] + if: "${{ needs.build.outputs.workflow_matrix != '[]' }}" + + strategy: + fail-fast: false + matrix: + component: ${{ fromJson(needs.build.outputs.workflow_matrix) }} + + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.' + - uses: data-intuitive/reclaim-the-bytes@v2 + + - name: Keep symlinks as-is + run: | + git config --global core.symlinks true + + - uses: actions/checkout@v7 + with: + ref: 'integration_build' + + - uses: viash-io/viash-actions/setup@v6 + + - uses: nf-core/setup-nextflow@v3.0.0 + + # use cache + - name: Cache resources data + if: ${{ needs.sync_s3.outputs.cache_key != '' }} + uses: actions/cache/restore@v5 + timeout-minutes: 10 + with: + path: ${{ needs.sync_s3.outputs.dest_paths }} + key: ${{ needs.sync_s3.outputs.cache_key }} + fail-on-cache-miss: true + + - name: Remove unused test resources to save space + shell: bash + run: | + readarray -t resources < <(viash config view --format json "${{ matrix.component.config }}" -c 'del(.dependencies)' | jq -r -c '(.build_info.config | capture("^(?.*\/)").dir) as $dir | .test_resources | map(select(.type == "file")) | map($dir + .path) | unique | .[]') + to_not_remove=() + for resource in "${resources[@]}"; do + if [[ $resource == *"resources_test"* ]]; then + relative_path=${resource#*resources_test/} + relative_path_trailing_slash_removed=${relative_path%/} + to_not_remove+=("-path" "./resources_test/$relative_path_trailing_slash_removed" "-prune" "-o") + fi + done + # Remove last prune and -o + if (( ${#errors[@]} )); then + unset 'to_not_remove[${#to_not_remove[@]}-1]' + unset 'to_not_remove[${#to_not_remove[@]}-1]' + to_not_remove+=( "(" "${to_not_remove[@]}" ")" "-prune" "-o") + fi + echo "Not removing ${to_not_remove[@]}" + find ./resources_test/ "${to_not_remove[@]}" -type f -exec rm {} + + tree ./resources_test/ + + - name: Build test dependencies + run: | + readarray -t test_dependencies < <(viash ns list --format json "${{ matrix.component.config }}" | jq -r -c '(.[].info.test_dependencies // []) | flatten | .[] | (.namespace + "/" + .name)') + for dependency in "${test_dependencies[@]}"; do + echo $dependency + viash ns build -q "$dependency" -c ".version := 'integration_build'" -c ".engines[.type == 'docker'].target_tag := 'integration_build'" --setup alwayspullelsebuild + done + + - name: Remove docker builder cache + run: | + docker builder prune -a -f + + - name: Run integration test + timeout-minutes: 120 + run: | + # check if stub runs work + export NXF_VER=24.10.6 + nextflow run . \ + -main-script "target/${{ matrix.component.scope == 'private' && '_private/' || '' }}nextflow/${{matrix.component.namespace}}/${{matrix.component.name}}/main.nf" \ + -stub -profile no_publish \ + -c src/workflows/utils/integration_tests.config + + # todo: replace with viash test command + export NXF_VER=24.10.6 + nextflow run . \ + -main-script "${{ matrix.component.main_script }}" \ + -entry "${{ matrix.component.entry }}" \ + -profile docker,mount_temp,no_publish \ + -c src/workflows/utils/labels_ci.config \ + -c src/workflows/utils/integration_tests.config From 7b603b6fd9a99a2e4ab14833f4a6193bc4acff99 Mon Sep 17 00:00:00 2001 From: dorien-er Date: Mon, 6 Jul 2026 11:06:05 +0200 Subject: [PATCH 03/13] cleanup --- .../annotation_test_data.sh | 142 +++++++----------- resources_test_scripts/pbmc_1k_protein_v3.sh | 7 +- .../reference_gencodev41.sh | 3 +- 3 files changed, 61 insertions(+), 91 deletions(-) diff --git a/resources_test_scripts/annotation_test_data.sh b/resources_test_scripts/annotation_test_data.sh index 8c2f55c..a15f705 100644 --- a/resources_test_scripts/annotation_test_data.sh +++ b/resources_test_scripts/annotation_test_data.sh @@ -32,7 +32,7 @@ import numpy as np # Read in data ref_adata = ad.read_h5ad("${OUT}/tmp_TS_Blood_filtered.h5ad") -sub_ref_adata = ref_adata[ref_adata.obs["donor_assay"] == "TSP14_10x 3' v3"] +sub_ref_adata = ref_adata[ref_adata.obs["donor_assay"] == "TSP14_10x 3' v3"] n=100 s=sub_ref_adata.obs.groupby('cell_ontology_class').cell_ontology_class.transform('count') sub_ref_adata_final = sub_ref_adata[sub_ref_adata.obs[s>=n].groupby('cell_ontology_class').head(n).index] @@ -65,7 +65,7 @@ sub_ref_adata_final.obs["major_cell_type"] = major_cell_type.astype("category") # The var index shares its name ("feature_name") with a var column of the same # name. anndata refuses to write a MuData when the index name collides with a -# differently-typed column, so clear the index name before conversion. +# differently-typed column, so clear the index name before conversion. sub_ref_adata_final.var.index.name = None # Write out data @@ -74,128 +74,101 @@ HEREDOC echo "> Converting to h5mu" -cat > /tmp/from_h5ad_to_h5mu.yaml << HERE -input: "${OUT}/TS_Blood_filtered.h5ad" -output: "TS_Blood_filtered.h5mu" -modality: "rna" -HERE - nextflow \ run https://packages.viash-hub.com/vsh/openpipeline \ -r v4.1.1 \ -main-script target/nextflow/convert/from_h5ad_to_h5mu/main.nf \ - --publish_dir "${OUT}" \ -profile docker,mount_temp \ - -params-file /tmp/from_h5ad_to_h5mu.yaml \ - -c ./src/configs/labels_ci.config + -c ./src/configs/labels_ci.config \ + --publish_dir "${OUT}" \ + --input "${OUT}/TS_Blood_filtered.h5ad" \ + --output "TS_Blood_filtered.h5mu" \ + --modality "rna" -# echo "> Downloading pretrained CellTypist model and sample test data" -# wget https://celltypist.cog.sanger.ac.uk/models/Pan_Immune_CellTypist/v2/Immune_All_Low.pkl \ -# -O "${OUT}/celltypist_model_Immune_All_Low.pkl" -# wget https://celltypist.cog.sanger.ac.uk/Notebook_demo_data/demo_2000_cells.h5ad \ -# -O "${OUT}/demo_2000_cells.h5ad" +echo "> Downloading sample test data" +wget https://celltypist.cog.sanger.ac.uk/Notebook_demo_data/demo_2000_cells.h5ad \ + -O "${OUT}/demo_2000_cells.h5ad" echo "> Converting to h5mu" -cat > /tmp/from_h5ad_to_h5mu_demo.yaml << HERE -input: "${OUT}/demo_2000_cells.h5ad" -output: "demo_2000_cells.h5mu" -modality: "rna" -HERE - nextflow \ run https://packages.viash-hub.com/vsh/openpipeline \ -r v4.1.1 \ -main-script target/nextflow/convert/from_h5ad_to_h5mu/main.nf \ - --publish_dir "${OUT}" \ -profile docker,mount_temp \ - -params-file /tmp/from_h5ad_to_h5mu_demo.yaml \ - -c ./src/configs/labels_ci.config + -c ./src/configs/labels_ci.config \ + --publish_dir "${OUT}" \ + --input "${OUT}/demo_2000_cells.h5ad" \ + --output "demo_2000_cells.h5mu" \ + --modality "rna" echo "> Creating simple SCVI model" -cat > /tmp/simple_scvi.yaml << HERE -input: "${OUT}/TS_Blood_filtered.h5mu" -obs_batch: "donor_id" -var_gene_names: "ensemblid" -output: "scvi_output.h5mu" -output_model: "scvi_model" -max_epochs: 5 -n_obs_min_count: 10 -n_var_min_count: 10 -HERE - nextflow \ run https://packages.viash-hub.com/vsh/openpipeline \ -r v4.1.1 \ -main-script target/nextflow/integrate/scvi/main.nf \ - --publish_dir "${OUT}" \ -profile docker,mount_temp \ - -params-file /tmp/simple_scvi.yaml \ - -c ./src/configs/labels_ci.config + -c ./src/configs/labels_ci.config \ + --publish_dir "${OUT}" \ + --input "${OUT}/TS_Blood_filtered.h5mu" \ + --obs_batch "donor_id" \ + --var_gene_names "ensemblid" \ + --output "scvi_output.h5mu" \ + --output_model "scvi_model" \ + --max_epochs 5 \ + --n_obs_min_count 10 \ + --n_var_min_count 10 echo "> Creating SCVI model with covariates" -cat > /tmp/covariates_scvi.yaml << HERE -input: "${OUT}/TS_Blood_filtered.h5mu" -obs_batch: "donor_id" -obs_categorical_covariate: "assay" -obs_categorical_covariate: "donor_assay" -var_gene_names: "ensemblid" -output: "scvi_covariate_output.h5mu" -output_model: "scvi_covariate_model" -max_epochs: 5 -n_obs_min_count: 10 -n_var_min_count: 10 -HERE - nextflow \ run https://packages.viash-hub.com/vsh/openpipeline \ -r v4.1.1 \ -main-script target/nextflow/integrate/scvi/main.nf \ - --publish_dir "${OUT}" \ -profile docker,mount_temp \ - -params-file /tmp/covariates_scvi.yaml \ - -c ./src/configs/labels_ci.config + -c ./src/configs/labels_ci.config \ + --publish_dir "${OUT}" \ + --input "${OUT}/TS_Blood_filtered.h5mu" \ + --obs_batch "donor_id" \ + --obs_categorical_covariate "assay;donor_assay" \ + --var_gene_names "ensemblid" \ + --output "scvi_covariate_output.h5mu" \ + --output_model "scvi_covariate_model" \ + --max_epochs 5 \ + --n_obs_min_count 10 \ + --n_var_min_count 10 echo "> Creating simple SCANVI model" -cat > /tmp/simple_scanvi.yaml << HERE -input: "${OUT}/TS_Blood_filtered.h5mu" -scvi_model: "${OUT}/scvi_model" -obs_labels: "cell_ontology_class" -var_gene_names: "ensemblid" -output: "scanvi_output.h5mu" -output_model: "scanvi_model" -max_epochs: 5 -HERE - nextflow \ run https://packages.viash-hub.com/vsh/openpipeline \ -r v4.1.1 \ -main-script target/nextflow/annotate/scanvi/main.nf \ - --publish_dir "${OUT}" \ -profile docker,mount_temp \ - -params-file /tmp/simple_scanvi.yaml \ - -c ./src/configs/labels_ci.config + -c ./src/configs/labels_ci.config \ + --publish_dir "${OUT}" \ + --input "${OUT}/TS_Blood_filtered.h5mu" \ + --scvi_model "${OUT}/scvi_model" \ + --obs_labels "cell_ontology_class" \ + --var_gene_names "ensemblid" \ + --output "scanvi_output.h5mu" \ + --output_model "scanvi_model" \ + --max_epochs 5 echo "> Creating SCANVI model with covariates" -cat > /tmp/covariates_scanvi.yaml << HERE -input: "${OUT}/TS_Blood_filtered.h5mu" -scvi_model: "${OUT}/scvi_covariate_model" -obs_labels: "cell_ontology_class" -var_gene_names: "ensemblid" -output: "scanvi_covariate_output.h5mu" -output_model: "scanvi_covariate_model" -max_epochs: 5 -HERE - nextflow \ run https://packages.viash-hub.com/vsh/openpipeline \ -r v4.1.1 \ -main-script target/nextflow/annotate/scanvi/main.nf \ - --publish_dir "${OUT}" \ -profile docker,mount_temp \ - -params-file /tmp/covariates_scanvi.yaml \ - -c ./src/configs/labels_ci.config + -c ./src/configs/labels_ci.config \ + --publish_dir "${OUT}" \ + --input "${OUT}/TS_Blood_filtered.h5mu" \ + --scvi_model "${OUT}/scvi_covariate_model" \ + --obs_labels "cell_ontology_class" \ + --var_gene_names "ensemblid" \ + --output "scanvi_covariate_output.h5mu" \ + --output_model "scanvi_covariate_model" \ + --max_epochs 5 rm "${OUT}/scanvi_output.h5mu" @@ -206,6 +179,7 @@ rm -f "${OUT}"/*.h5ad rm -f "${OUT}"/*.state.yaml aws s3 sync \ - resources_test/annotation_test_data \ - s3://openpipelines-bio/openpipeline_composed/resources_test/annotation_test_data \ - --dryrun --delete \ No newline at end of file + "$OUT" \ + s3://openpipelines-bio/openpipeline_composed/resources_test/"$ID" \ + --delete \ + --dryrun diff --git a/resources_test_scripts/pbmc_1k_protein_v3.sh b/resources_test_scripts/pbmc_1k_protein_v3.sh index 3864efd..0f2e846 100644 --- a/resources_test_scripts/pbmc_1k_protein_v3.sh +++ b/resources_test_scripts/pbmc_1k_protein_v3.sh @@ -42,12 +42,11 @@ wget https://cf.10xgenomics.com/samples/cell-exp/3.0.0/pbmc_1k_protein_v3/pbmc_1 # convert 10x h5 to h5mu nextflow run https://packages.viash-hub.com/vsh/openpipeline \ - -latest \ -r v4.1.1 \ -main-script target/nextflow/convert/from_10xh5_to_h5mu/main.nf \ -profile docker \ -c ./src/configs/labels_ci.config \ - --publish_dir $OUT \ + --publish_dir "$OUT" \ --input "${OUT}/${ID}_filtered_feature_bc_matrix.h5" \ --input_metrics_summary "${OUT}/${ID}_metrics_summary.csv" \ --output "${ID}_filtered_feature_bc_matrix.h5mu" \ @@ -56,7 +55,6 @@ nextflow run https://packages.viash-hub.com/vsh/openpipeline \ # run sample processing nextflow \ run https://packages.viash-hub.com/vsh/openpipeline \ - -latest \ -r v4.1.1 \ -main-script target/nextflow/workflows/multiomics/process_samples/main.nf \ -c ./src/configs/labels_ci.config \ @@ -64,7 +62,7 @@ nextflow \ --id pbmc_1k_protein_v3_uss \ --input "${OUT}/${ID}_filtered_feature_bc_matrix.h5mu" \ --output "${ID}_mms.h5mu" \ - --publishDir "$OUT" \ + --publish_dir "$OUT" \ -resume # remove all files from the output folder except the final mms output @@ -73,6 +71,5 @@ find "${OUT}" -mindepth 1 ! -name "${ID}_mms.h5mu" -delete aws s3 sync \ "$OUT" \ s3://openpipelines-bio/openpipeline_composed/resources_test/"$ID" \ - --exclude "*.yaml" \ --delete \ --dryrun diff --git a/resources_test_scripts/reference_gencodev41.sh b/resources_test_scripts/reference_gencodev41.sh index c723ac8..08e24ce 100644 --- a/resources_test_scripts/reference_gencodev41.sh +++ b/resources_test_scripts/reference_gencodev41.sh @@ -50,13 +50,12 @@ param_list: HERE nextflow run https://packages.viash-hub.com/vsh/openpipeline \ - -latest \ -r v4.1.1 \ -main-script target/nextflow/workflows/ingestion/make_reference/main.nf \ -profile docker \ -c ./src/configs/labels_ci.config \ -params-file /tmp/params.yaml \ - --publish_dir $OUT \ + --publish_dir "$OUT" \ -resume rm "$motifs_modified" From 39196fc3d276b30bc0075812536f96572c58515a Mon Sep 17 00:00:00 2001 From: dorien-er Date: Mon, 6 Jul 2026 11:10:44 +0200 Subject: [PATCH 04/13] add temporary integration test workflow --- .github/workflows/main-build.yml | 268 +++++++++++++++++++++++++++++++ 1 file changed, 268 insertions(+) create mode 100644 .github/workflows/main-build.yml diff --git a/.github/workflows/main-build.yml b/.github/workflows/main-build.yml new file mode 100644 index 0000000..901a5b7 --- /dev/null +++ b/.github/workflows/main-build.yml @@ -0,0 +1,268 @@ +name: Build +concurrency: + group: ${{ github.workflow }}-${{ inputs.deploy_branch && inputs.deploy_branch || format('{0}_build', github.ref_name) }} + cancel-in-progress: true + +on: + workflow_dispatch: + inputs: + push_containers: + type: boolean + required: false + default: false + description: Build docker images and push them to the registry + version: + type: string + required: false + description: | + Version to tag the build components with (e.i functionality.version). + Defaults to name of the branch that triggered the workflow, suffixed by "_build". + target_tag: + type: string + required: false + description: | + Version tag of containers to use. Defaults to name of the branch that triggered the workflow, + suffixed by "_build". Can be used in combination with 'push_containers' (by unchecking it) + to re-use existing docker images or set the tag for new builds. + + # when used as a subworkflow + workflow_call: + inputs: + push_containers: + type: boolean + required: false + default: false + description: push the containers to the registry + version: + type: string + required: false + description: | + Version to tag the build components with (e.i functionality.version). + Defaults to name of the branch that triggered the workflow, suffixed by "_build". + target_tag: + type: string + required: false + description: | + Version tag of containers to use. Defaults to name of the branch that triggered the workflow, + suffixed by "_build". Can be used in combination with 'push_containers' (by setting it to 'false') + to re-use existing docker images or set the tag for new builds. + deploy_branch: + type: string + required: false + description: | + Branch to deploy the build to. Defaults to name of the branch + that triggered the workflow, suffixed by "_build". + is_workflow_call: + description: | + Flag to destinguish between a 'workflow_call' and 'workflow_dispatch'. + github.event_name can not be used for this because it refers to the event of + the top-level workflow. See https://github.com/actions/runner/issues/3146 + type: boolean + required: false + default: true + outputs: + component_matrix: + description: "A JSON object that can be used to populate a github actions matrix for component jobs." + value: ${{ jobs.build_and_deploy_target_folder.outputs.component_matrix }} + workflow_matrix: + description: "A JSON object that can be used to populate a github actions matrix for workflow jobs." + value: ${{ jobs.build_and_deploy_target_folder.outputs.workflow_matrix}} + secrets: + GTHB_USER: + required: true + GTHB_PAT: + required: true + push: + branches: [ 'main' ] + + +jobs: + # phase 1 + build_and_deploy_target_folder: + name: "Build and push target folder" + runs-on: ubuntu-latest + + outputs: + component_matrix: ${{ steps.set_matrix.outputs.components }} + workflow_matrix: ${{ steps.set_matrix.outputs.workflows }} + + env: + DEPLOY_BRANCH: ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch }} + + steps: + - name: Do not allow manual trigger if the branch is main + # github.event_name is the event of the top-level workflow, so use dummy 'is_workflow_call' input + if: ${{ github.event_name != 'push' && (!inputs.is_workflow_call) && github.ref == 'refs/heads/main' }} + run: | + echo "This workflow should not be triggered with workflow_dispatch from the main branch" + exit 1 + - name: Check input arguments + run: | + input_version="${{ inputs.version || format('{0}_build', github.ref_name) }}" + target_tag="${{ github.event_name == 'push' && (inputs.version || format('{0}_build', github.ref_name)) || inputs.target_tag }}" + should_push="${{inputs.push_containers }}" + if [ "$input_version" != "$target_tag" ] && [ "$should_push" == "true" ]; then + echo "When trying to push new docker images, the tag for the components must be equal to the target_tag for the docker images." + exit 1 + fi + + - name: Keep symlinks as-is + run: | + git config --global core.symlinks true + + - name: Branch to checkout (use existing target branch if it exists) + id: get_checkout_branch + run: | + if ! git ls-remote --heads --exit-code https://github.com/openpipelines-bio/openpipeline_composed.git "$DEPLOY_BRANCH" > /dev/null; then + echo "Remote branch does not exist, fetching current branch and building on top of it" + echo "checkout_branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT" + else + echo "Remote branch exists, checking out existing branch" + echo "checkout_branch=$DEPLOY_BRANCH" >> "$GITHUB_OUTPUT" + fi + + - uses: actions/checkout@v7 + with: + ref: ${{ steps.get_checkout_branch.outputs.checkout_branch }} + fetch-depth: 0 + + - name: Fetch changes from ${{github.ref_name}} + run: | + git fetch origin ${{github.ref_name}} + git checkout -f --no-overlay origin/${{github.ref_name}} -- '.' + + - uses: viash-io/viash-actions/setup@v6 + + - name: Remove target folder from .gitignore + run: | + # allow publishing the target folder + sed -i '/^\/target\/$/d' .gitignore + + - uses: viash-io/viash-actions/ns-build@v6 + with: + config_mod: | + .version := "${{ inputs.version || format('{0}_build', github.ref_name) }}" + .engines[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && (inputs.version || format('{0}_build', github.ref_name)) || inputs.target_tag }}' + parallel: true + query: ^(?!workflows|test_workflows) + + - uses: viash-io/viash-actions/ns-build@v6 + with: + config_mod: .version := "${{ inputs.version || format('{0}_build', github.ref_name) }}" + parallel: true + query: ^workflows + + - name: "Remove test component and workflows from the target folder" + run: | + rm -r target/_test + + - name: Build nextflow schemas + uses: viash-io/viash-actions/pro/build-nextflow-schemas@v6 + with: + target_dir: target/nextflow + viash_pro_token: ${{ secrets.GTHB_PAT }} + tools_version: 'main_build' + enable_dataset_input: true + + - name: Build parameter files + uses: viash-io/viash-actions/pro/build-nextflow-params@v6 + with: + target_dir: target/nextflow + viash_pro_token: ${{ secrets.GTHB_PAT }} + tools_version: 'main_build' + + - name: Deploy to target branch + uses: stefanzweifel/git-auto-commit-action@v7 + with: + create_branch: true + commit_message: "deploy: ${{github.sha}}" + skip_dirty_check: true + branch: ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch }} + + - name: "List components" + id: ns_list + uses: viash-io/viash-actions/ns-list@v6 + with: + engine: docker + src: src + format: json + query_namespace: ^(?!workflows|test_workflows) + + - name: "List workflows" + id: ns_list_workflows + uses: viash-io/viash-actions/ns-list@v6 + with: + src: src + format: json + query_namespace: ^workflows/(?!test_workflows) + + - name: "Parse JSON output from 'viash ns list' as input for matrix." + id: set_matrix + run: | + echo "components=$(jq -c '[ .[] | select(.scope.target != "test") | + { + "name": .name, + "namespace_separator": .engines | map(select(.type == "docker"))[0].namespace_separator, + "namespace": .namespace, + "config": .build_info.config, + "dir": .build_info.config | capture("^(?.*\/)").dir + } + ] | unique' ${{ steps.ns_list.outputs.output_file }} )" >> $GITHUB_OUTPUT + + echo "workflows=$(jq -c '[ .[] | select(.scope.target != "test") | . as $config | (.test_resources // [])[] | select(.type == "nextflow_script", .entrypoint) | + { + "name": $config.name, + "namespace": $config.namespace, + "main_script": (($config.build_info.config | capture("^(?.*\/)").dir) + "/" + .path), + "entry": .entrypoint, + "config": $config.build_info.config, + "scope": $config.scope.target + } + ] | unique' ${{ steps.ns_list_workflows.outputs.output_file }} )" >> $GITHUB_OUTPUT + + # phase 2 + build_and_deploy_docker_containers: + needs: build_and_deploy_target_folder + if: ${{github.event_name == 'push' || inputs.push_containers }} + + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + component: ${{ fromJson(needs.build_and_deploy_target_folder.outputs.component_matrix) }} + + name: "${{matrix.component.name}} - ${{matrix.component.namespace}} - Build and push docker image" + + steps: + # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.' + - uses: data-intuitive/reclaim-the-bytes@v2 + + - uses: actions/checkout@v7 + + - uses: viash-io/viash-actions/setup@v6 + + - name: Build container + uses: viash-io/viash-actions/ns-build@v6 + with: + config_mod: | + .version := "${{ inputs.version || format('{0}_build', github.ref_name) }}" + .engines[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && 'main_build' || inputs.target_tag }}' + engine: docker + src: ${{ matrix.component.dir }} + setup: build + + - name: Login to container registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ secrets.GTHB_USER }} + password: ${{ secrets.GTHB_PAT }} + + - name: Push container + uses: viash-io/viash-actions/ns-build@v6 + with: + config_mod: .version := "${{ inputs.version || format('{0}_build', github.ref_name) }}" + engine: docker + src: ${{ matrix.component.dir }} + setup: push \ No newline at end of file From 213dbf57d0e9f0d75bd341050bb14655437c7340 Mon Sep 17 00:00:00 2001 From: dorien-er Date: Mon, 6 Jul 2026 11:31:04 +0200 Subject: [PATCH 05/13] add temporary integration test workflow --- .github/workflows/main-build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main-build.yml b/.github/workflows/main-build.yml index 901a5b7..090ed5c 100644 --- a/.github/workflows/main-build.yml +++ b/.github/workflows/main-build.yml @@ -144,13 +144,13 @@ jobs: .version := "${{ inputs.version || format('{0}_build', github.ref_name) }}" .engines[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && (inputs.version || format('{0}_build', github.ref_name)) || inputs.target_tag }}' parallel: true - query: ^(?!workflows|test_workflows) - + query: ^(?!single_cell|test_workflows) + - uses: viash-io/viash-actions/ns-build@v6 with: config_mod: .version := "${{ inputs.version || format('{0}_build', github.ref_name) }}" parallel: true - query: ^workflows + query: ^single_cell - name: "Remove test component and workflows from the target folder" run: | @@ -186,7 +186,7 @@ jobs: engine: docker src: src format: json - query_namespace: ^(?!workflows|test_workflows) + query_namespace: ^(?!single_cell|test_workflows) - name: "List workflows" id: ns_list_workflows @@ -194,7 +194,7 @@ jobs: with: src: src format: json - query_namespace: ^workflows/(?!test_workflows) + query_namespace: ^single_cell - name: "Parse JSON output from 'viash ns list' as input for matrix." id: set_matrix From c42f382acb9b6521eef4d96358144ed4db04d5ac Mon Sep 17 00:00:00 2001 From: dorien-er Date: Mon, 6 Jul 2026 13:37:50 +0200 Subject: [PATCH 06/13] update integration test resources --- src/single_cell/cellranger_multi_qc/test.nf | 2 +- .../parallel_annotation/integration_test.sh | 11 +++++++++++ src/single_cell/parallel_annotation/test.nf | 5 ++--- src/single_cell/parallel_integration/test.nf | 2 +- src/single_cell/process_integrate_annotate/test.nf | 2 +- src/test_workflows/assert_h5mu_slots/script.py | 4 +++- 6 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/single_cell/cellranger_multi_qc/test.nf b/src/single_cell/cellranger_multi_qc/test.nf index 04e498d..264c0f5 100644 --- a/src/single_cell/cellranger_multi_qc/test.nf +++ b/src/single_cell/cellranger_multi_qc/test.nf @@ -2,7 +2,7 @@ nextflow.enable.dsl=2 include { cellranger_multi_qc } from params.rootDir + "/target/_private/nextflow/single_cell/cellranger_multi_qc/main.nf" -params.resources_test = "s3://openpipelines-bio/openpipeline_incubator/resources_test/" +params.resources_test = "s3://openpipelines-bio/openpipeline_composed/resources_test/" workflow test_wf { resources_test = file(params.resources_test) diff --git a/src/single_cell/parallel_annotation/integration_test.sh b/src/single_cell/parallel_annotation/integration_test.sh index 30fde28..354dc73 100755 --- a/src/single_cell/parallel_annotation/integration_test.sh +++ b/src/single_cell/parallel_annotation/integration_test.sh @@ -17,3 +17,14 @@ nextflow \ -c src/configs/labels_ci.config \ -c src/configs/integration_tests.config \ --publish_dir test + +nextflow \ + run . \ + -main-script src/single_cell/parallel_annotation/test.nf \ + -entry test_wf_2 \ + -resume \ + -profile docker \ + -c src/configs/labels_ci.config \ + -c src/configs/integration_tests.config \ + --publish_dir test + \ No newline at end of file diff --git a/src/single_cell/parallel_annotation/test.nf b/src/single_cell/parallel_annotation/test.nf index 41f0175..2f2f4f8 100644 --- a/src/single_cell/parallel_annotation/test.nf +++ b/src/single_cell/parallel_annotation/test.nf @@ -2,7 +2,7 @@ nextflow.enable.dsl=2 include { parallel_annotation } from params.rootDir + "/target/nextflow/single_cell/parallel_annotation/main.nf" include { assert_h5mu_slots } from params.rootDir + "/target/_test/nextflow/test_workflows/assert_h5mu_slots/main.nf" -params.resources_test = "s3://openpipelines-bio/openpipeline_incubator/resources_test/" +params.resources_test = "s3://openpipelines-bio/openpipeline_composed/resources_test/" // Default .obs prediction/probability and .obsm embedding slots each method // writes into the merged output. celltypist produces predictions only. @@ -91,9 +91,8 @@ workflow test_wf_2 { [ [ id: "celltypist_model_test", - input: resources_test.resolve("pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu"), + input: resources_test.resolve("annotation_test_data/demo_2000_cells.h5mu"), celltypist_model: resources_test.resolve("annotation_test_data/celltypist_model_Immune_All_Low.pkl"), - input_var_gene_names: "gene_symbol", annotation_methods: "celltypist" ] ]) diff --git a/src/single_cell/parallel_integration/test.nf b/src/single_cell/parallel_integration/test.nf index 6cdb12d..190af5f 100644 --- a/src/single_cell/parallel_integration/test.nf +++ b/src/single_cell/parallel_integration/test.nf @@ -2,7 +2,7 @@ nextflow.enable.dsl=2 include { parallel_integration } from params.rootDir + "/target/nextflow/single_cell/parallel_integration/main.nf" include { assert_h5mu_slots } from params.rootDir + "/target/_test/nextflow/test_workflows/assert_h5mu_slots/main.nf" -params.resources_test = "s3://openpipelines-bio/openpipeline_incubator/resources_test/" +params.resources_test = "s3://openpipelines-bio/openpipeline_composed/resources_test/" // Build the set of slots each selected method is expected to write into the // merged output, using the workflow's default slot names. bbknn does not diff --git a/src/single_cell/process_integrate_annotate/test.nf b/src/single_cell/process_integrate_annotate/test.nf index 6f0617e..20e91c5 100644 --- a/src/single_cell/process_integrate_annotate/test.nf +++ b/src/single_cell/process_integrate_annotate/test.nf @@ -1,7 +1,7 @@ nextflow.enable.dsl=2 include { process_integrate_annotate } from params.rootDir + "/target/_private/nextflow/single_cell/process_integrate_annotate/main.nf" -params.resources_test = "s3://openpipelines-bio/openpipeline_incubator/resources_test/" +params.resources_test = "s3://openpipelines-bio/openpipeline_composed/resources_test/" workflow test_wf { resources_test = file(params.resources_test) diff --git a/src/test_workflows/assert_h5mu_slots/script.py b/src/test_workflows/assert_h5mu_slots/script.py index c8418b7..92403bb 100644 --- a/src/test_workflows/assert_h5mu_slots/script.py +++ b/src/test_workflows/assert_h5mu_slots/script.py @@ -36,7 +36,9 @@ missing = {} for slot, present_keys in present.items(): - expected = par.get(slot) or [] + # A multiple-value argument passed an empty list arrives as [""] from Viash; + # drop empty strings so an empty selection means "no expected keys". + expected = [key for key in (par.get(slot) or []) if key] absent = [key for key in expected if key not in present_keys] if absent: missing[slot] = (absent, sorted(present_keys)) From 0bbf70b0ec14a6b76e06b2c9bac44aa9ff16e8ab Mon Sep 17 00:00:00 2001 From: dorien-er Date: Mon, 6 Jul 2026 17:20:22 +0200 Subject: [PATCH 07/13] update integration test workflow --- .github/workflows/integration-test.yml | 8 ++++---- .github/workflows/main-build.yml | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index b71fc5a..74777c7 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -30,7 +30,7 @@ jobs: # Synchronize S3 Bucket and create cache for per-component runs sync_s3: env: - s3_bucket: s3://openpipelines-data/ + s3_bucket: s3://openpipelines-bio/openpipeline_composed runs-on: ubuntu-latest outputs: @@ -129,7 +129,7 @@ jobs: nextflow run . \ -main-script "target/${{ matrix.component.scope == 'private' && '_private/' || '' }}nextflow/${{matrix.component.namespace}}/${{matrix.component.name}}/main.nf" \ -stub -profile no_publish \ - -c src/workflows/utils/integration_tests.config + -c src/configs/integration_tests.config # todo: replace with viash test command export NXF_VER=24.10.6 @@ -137,5 +137,5 @@ jobs: -main-script "${{ matrix.component.main_script }}" \ -entry "${{ matrix.component.entry }}" \ -profile docker,mount_temp,no_publish \ - -c src/workflows/utils/labels_ci.config \ - -c src/workflows/utils/integration_tests.config + -c src/configs/labels_ci.config \ + -c src/configs/integration_tests.config diff --git a/.github/workflows/main-build.yml b/.github/workflows/main-build.yml index 090ed5c..91ded01 100644 --- a/.github/workflows/main-build.yml +++ b/.github/workflows/main-build.yml @@ -154,7 +154,9 @@ jobs: - name: "Remove test component and workflows from the target folder" run: | - rm -r target/_test + # test-scoped components (e.g. test_workflows) are excluded from the build queries, + # so target/_test may not exist; -f avoids failing the build when it is absent. + rm -rf target/_test - name: Build nextflow schemas uses: viash-io/viash-actions/pro/build-nextflow-schemas@v6 From 91dcceb7a6fb8d420bf741c82a1a70b9a8a4037c Mon Sep 17 00:00:00 2001 From: dorien-er Date: Mon, 6 Jul 2026 17:46:38 +0200 Subject: [PATCH 08/13] fix: yield valid boolean for push_containers on push trigger The build reusable-workflow call passed `github.event_name == 'schedule' || inputs.push_containers` to a boolean input. On a push event `inputs` is empty, so the expression resolved to null and failed workflow-call validation at startup, which is why the build job never ran and integration_test was skipped. Treat push like schedule so the value is always a real boolean. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 74777c7..5515b71 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -21,7 +21,7 @@ jobs: build: uses: ./.github/workflows/main-build.yml with: - push_containers: ${{ github.event_name == 'schedule' || inputs.push_containers }} + push_containers: ${{ github.event_name == 'schedule' || github.event_name == 'push' || inputs.push_containers }} version: 'integration_build' target_tag: 'integration_build' deploy_branch: 'integration_build' From cdb6312c9ee3c0575edd5ddbc586cf047ae38f74 Mon Sep 17 00:00:00 2001 From: dorien-er Date: Mon, 6 Jul 2026 19:24:16 +0200 Subject: [PATCH 09/13] fix: use target_tag input for container build when called as subworkflow Phase 2 built the docker image with tag 'main_build' whenever the top-level event was a push, but the push step tags it with the version input ('integration_build'), so the build/push tags diverged and the push failed. github.event_name reflects the top-level event, which is 'push' when integration-test triggers main-build from a branch push. Guard the 'main_build' tag with !is_workflow_call so a called build uses the provided target_tag, keeping direct pushes to main unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/main-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main-build.yml b/.github/workflows/main-build.yml index 91ded01..4952c47 100644 --- a/.github/workflows/main-build.yml +++ b/.github/workflows/main-build.yml @@ -249,7 +249,7 @@ jobs: with: config_mod: | .version := "${{ inputs.version || format('{0}_build', github.ref_name) }}" - .engines[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && 'main_build' || inputs.target_tag }}' + .engines[.type == 'docker'].target_tag := '${{ (github.event_name == 'push' && !inputs.is_workflow_call) && 'main_build' || inputs.target_tag }}' engine: docker src: ${{ matrix.component.dir }} setup: build From eceb1a48b38be0c3692e9da97fb58343012e8d10 Mon Sep 17 00:00:00 2001 From: dorien-er Date: Tue, 7 Jul 2026 09:47:41 +0200 Subject: [PATCH 10/13] remove github workflows --- .github/workflows/integration-test.yml | 141 ------------- .github/workflows/main-build.yml | 270 ------------------------- 2 files changed, 411 deletions(-) delete mode 100644 .github/workflows/integration-test.yml delete mode 100644 .github/workflows/main-build.yml diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml deleted file mode 100644 index 5515b71..0000000 --- a/.github/workflows/integration-test.yml +++ /dev/null @@ -1,141 +0,0 @@ -name: integration test -concurrency: - group: ${{ github.workflow }} - cancel-in-progress: false - -on: - push: - branches: [update-test-resources] - workflow_dispatch: - inputs: - push_containers: - type: boolean - required: false - default: true - description: Build docker images and push them to the registry - schedule: - - cron: '33 2 * * *' - -jobs: - # Build and create containers - build: - uses: ./.github/workflows/main-build.yml - with: - push_containers: ${{ github.event_name == 'schedule' || github.event_name == 'push' || inputs.push_containers }} - version: 'integration_build' - target_tag: 'integration_build' - deploy_branch: 'integration_build' - secrets: inherit - - # Synchronize S3 Bucket and create cache for per-component runs - sync_s3: - env: - s3_bucket: s3://openpipelines-bio/openpipeline_composed - runs-on: ubuntu-latest - - outputs: - cache_key: ${{ steps.cache.outputs.cache_key }} - dest_paths: ${{ steps.cache.outputs.dest_paths }} - - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.' - - uses: data-intuitive/reclaim-the-bytes@v2 - - uses: viash-io/viash-actions/project/sync-and-cache@v6 - id: cache - - # phase 3 - integration_test: - runs-on: ubuntu-latest - needs: [ build, sync_s3 ] - if: "${{ needs.build.outputs.workflow_matrix != '[]' }}" - - strategy: - fail-fast: false - matrix: - component: ${{ fromJson(needs.build.outputs.workflow_matrix) }} - - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.' - - uses: data-intuitive/reclaim-the-bytes@v2 - - - name: Keep symlinks as-is - run: | - git config --global core.symlinks true - - - uses: actions/checkout@v7 - with: - ref: 'integration_build' - - - uses: viash-io/viash-actions/setup@v6 - - - uses: nf-core/setup-nextflow@v3.0.0 - - # use cache - - name: Cache resources data - if: ${{ needs.sync_s3.outputs.cache_key != '' }} - uses: actions/cache/restore@v5 - timeout-minutes: 10 - with: - path: ${{ needs.sync_s3.outputs.dest_paths }} - key: ${{ needs.sync_s3.outputs.cache_key }} - fail-on-cache-miss: true - - - name: Remove unused test resources to save space - shell: bash - run: | - readarray -t resources < <(viash config view --format json "${{ matrix.component.config }}" -c 'del(.dependencies)' | jq -r -c '(.build_info.config | capture("^(?.*\/)").dir) as $dir | .test_resources | map(select(.type == "file")) | map($dir + .path) | unique | .[]') - to_not_remove=() - for resource in "${resources[@]}"; do - if [[ $resource == *"resources_test"* ]]; then - relative_path=${resource#*resources_test/} - relative_path_trailing_slash_removed=${relative_path%/} - to_not_remove+=("-path" "./resources_test/$relative_path_trailing_slash_removed" "-prune" "-o") - fi - done - # Remove last prune and -o - if (( ${#errors[@]} )); then - unset 'to_not_remove[${#to_not_remove[@]}-1]' - unset 'to_not_remove[${#to_not_remove[@]}-1]' - to_not_remove+=( "(" "${to_not_remove[@]}" ")" "-prune" "-o") - fi - echo "Not removing ${to_not_remove[@]}" - find ./resources_test/ "${to_not_remove[@]}" -type f -exec rm {} + - tree ./resources_test/ - - - name: Build test dependencies - run: | - readarray -t test_dependencies < <(viash ns list --format json "${{ matrix.component.config }}" | jq -r -c '(.[].info.test_dependencies // []) | flatten | .[] | (.namespace + "/" + .name)') - for dependency in "${test_dependencies[@]}"; do - echo $dependency - viash ns build -q "$dependency" -c ".version := 'integration_build'" -c ".engines[.type == 'docker'].target_tag := 'integration_build'" --setup alwayspullelsebuild - done - - - name: Remove docker builder cache - run: | - docker builder prune -a -f - - - name: Run integration test - timeout-minutes: 120 - run: | - # check if stub runs work - export NXF_VER=24.10.6 - nextflow run . \ - -main-script "target/${{ matrix.component.scope == 'private' && '_private/' || '' }}nextflow/${{matrix.component.namespace}}/${{matrix.component.name}}/main.nf" \ - -stub -profile no_publish \ - -c src/configs/integration_tests.config - - # todo: replace with viash test command - export NXF_VER=24.10.6 - nextflow run . \ - -main-script "${{ matrix.component.main_script }}" \ - -entry "${{ matrix.component.entry }}" \ - -profile docker,mount_temp,no_publish \ - -c src/configs/labels_ci.config \ - -c src/configs/integration_tests.config diff --git a/.github/workflows/main-build.yml b/.github/workflows/main-build.yml deleted file mode 100644 index 4952c47..0000000 --- a/.github/workflows/main-build.yml +++ /dev/null @@ -1,270 +0,0 @@ -name: Build -concurrency: - group: ${{ github.workflow }}-${{ inputs.deploy_branch && inputs.deploy_branch || format('{0}_build', github.ref_name) }} - cancel-in-progress: true - -on: - workflow_dispatch: - inputs: - push_containers: - type: boolean - required: false - default: false - description: Build docker images and push them to the registry - version: - type: string - required: false - description: | - Version to tag the build components with (e.i functionality.version). - Defaults to name of the branch that triggered the workflow, suffixed by "_build". - target_tag: - type: string - required: false - description: | - Version tag of containers to use. Defaults to name of the branch that triggered the workflow, - suffixed by "_build". Can be used in combination with 'push_containers' (by unchecking it) - to re-use existing docker images or set the tag for new builds. - - # when used as a subworkflow - workflow_call: - inputs: - push_containers: - type: boolean - required: false - default: false - description: push the containers to the registry - version: - type: string - required: false - description: | - Version to tag the build components with (e.i functionality.version). - Defaults to name of the branch that triggered the workflow, suffixed by "_build". - target_tag: - type: string - required: false - description: | - Version tag of containers to use. Defaults to name of the branch that triggered the workflow, - suffixed by "_build". Can be used in combination with 'push_containers' (by setting it to 'false') - to re-use existing docker images or set the tag for new builds. - deploy_branch: - type: string - required: false - description: | - Branch to deploy the build to. Defaults to name of the branch - that triggered the workflow, suffixed by "_build". - is_workflow_call: - description: | - Flag to destinguish between a 'workflow_call' and 'workflow_dispatch'. - github.event_name can not be used for this because it refers to the event of - the top-level workflow. See https://github.com/actions/runner/issues/3146 - type: boolean - required: false - default: true - outputs: - component_matrix: - description: "A JSON object that can be used to populate a github actions matrix for component jobs." - value: ${{ jobs.build_and_deploy_target_folder.outputs.component_matrix }} - workflow_matrix: - description: "A JSON object that can be used to populate a github actions matrix for workflow jobs." - value: ${{ jobs.build_and_deploy_target_folder.outputs.workflow_matrix}} - secrets: - GTHB_USER: - required: true - GTHB_PAT: - required: true - push: - branches: [ 'main' ] - - -jobs: - # phase 1 - build_and_deploy_target_folder: - name: "Build and push target folder" - runs-on: ubuntu-latest - - outputs: - component_matrix: ${{ steps.set_matrix.outputs.components }} - workflow_matrix: ${{ steps.set_matrix.outputs.workflows }} - - env: - DEPLOY_BRANCH: ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch }} - - steps: - - name: Do not allow manual trigger if the branch is main - # github.event_name is the event of the top-level workflow, so use dummy 'is_workflow_call' input - if: ${{ github.event_name != 'push' && (!inputs.is_workflow_call) && github.ref == 'refs/heads/main' }} - run: | - echo "This workflow should not be triggered with workflow_dispatch from the main branch" - exit 1 - - name: Check input arguments - run: | - input_version="${{ inputs.version || format('{0}_build', github.ref_name) }}" - target_tag="${{ github.event_name == 'push' && (inputs.version || format('{0}_build', github.ref_name)) || inputs.target_tag }}" - should_push="${{inputs.push_containers }}" - if [ "$input_version" != "$target_tag" ] && [ "$should_push" == "true" ]; then - echo "When trying to push new docker images, the tag for the components must be equal to the target_tag for the docker images." - exit 1 - fi - - - name: Keep symlinks as-is - run: | - git config --global core.symlinks true - - - name: Branch to checkout (use existing target branch if it exists) - id: get_checkout_branch - run: | - if ! git ls-remote --heads --exit-code https://github.com/openpipelines-bio/openpipeline_composed.git "$DEPLOY_BRANCH" > /dev/null; then - echo "Remote branch does not exist, fetching current branch and building on top of it" - echo "checkout_branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT" - else - echo "Remote branch exists, checking out existing branch" - echo "checkout_branch=$DEPLOY_BRANCH" >> "$GITHUB_OUTPUT" - fi - - - uses: actions/checkout@v7 - with: - ref: ${{ steps.get_checkout_branch.outputs.checkout_branch }} - fetch-depth: 0 - - - name: Fetch changes from ${{github.ref_name}} - run: | - git fetch origin ${{github.ref_name}} - git checkout -f --no-overlay origin/${{github.ref_name}} -- '.' - - - uses: viash-io/viash-actions/setup@v6 - - - name: Remove target folder from .gitignore - run: | - # allow publishing the target folder - sed -i '/^\/target\/$/d' .gitignore - - - uses: viash-io/viash-actions/ns-build@v6 - with: - config_mod: | - .version := "${{ inputs.version || format('{0}_build', github.ref_name) }}" - .engines[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && (inputs.version || format('{0}_build', github.ref_name)) || inputs.target_tag }}' - parallel: true - query: ^(?!single_cell|test_workflows) - - - uses: viash-io/viash-actions/ns-build@v6 - with: - config_mod: .version := "${{ inputs.version || format('{0}_build', github.ref_name) }}" - parallel: true - query: ^single_cell - - - name: "Remove test component and workflows from the target folder" - run: | - # test-scoped components (e.g. test_workflows) are excluded from the build queries, - # so target/_test may not exist; -f avoids failing the build when it is absent. - rm -rf target/_test - - - name: Build nextflow schemas - uses: viash-io/viash-actions/pro/build-nextflow-schemas@v6 - with: - target_dir: target/nextflow - viash_pro_token: ${{ secrets.GTHB_PAT }} - tools_version: 'main_build' - enable_dataset_input: true - - - name: Build parameter files - uses: viash-io/viash-actions/pro/build-nextflow-params@v6 - with: - target_dir: target/nextflow - viash_pro_token: ${{ secrets.GTHB_PAT }} - tools_version: 'main_build' - - - name: Deploy to target branch - uses: stefanzweifel/git-auto-commit-action@v7 - with: - create_branch: true - commit_message: "deploy: ${{github.sha}}" - skip_dirty_check: true - branch: ${{ !inputs.deploy_branch && format('{0}_build', github.ref_name) || inputs.deploy_branch }} - - - name: "List components" - id: ns_list - uses: viash-io/viash-actions/ns-list@v6 - with: - engine: docker - src: src - format: json - query_namespace: ^(?!single_cell|test_workflows) - - - name: "List workflows" - id: ns_list_workflows - uses: viash-io/viash-actions/ns-list@v6 - with: - src: src - format: json - query_namespace: ^single_cell - - - name: "Parse JSON output from 'viash ns list' as input for matrix." - id: set_matrix - run: | - echo "components=$(jq -c '[ .[] | select(.scope.target != "test") | - { - "name": .name, - "namespace_separator": .engines | map(select(.type == "docker"))[0].namespace_separator, - "namespace": .namespace, - "config": .build_info.config, - "dir": .build_info.config | capture("^(?.*\/)").dir - } - ] | unique' ${{ steps.ns_list.outputs.output_file }} )" >> $GITHUB_OUTPUT - - echo "workflows=$(jq -c '[ .[] | select(.scope.target != "test") | . as $config | (.test_resources // [])[] | select(.type == "nextflow_script", .entrypoint) | - { - "name": $config.name, - "namespace": $config.namespace, - "main_script": (($config.build_info.config | capture("^(?.*\/)").dir) + "/" + .path), - "entry": .entrypoint, - "config": $config.build_info.config, - "scope": $config.scope.target - } - ] | unique' ${{ steps.ns_list_workflows.outputs.output_file }} )" >> $GITHUB_OUTPUT - - # phase 2 - build_and_deploy_docker_containers: - needs: build_and_deploy_target_folder - if: ${{github.event_name == 'push' || inputs.push_containers }} - - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - component: ${{ fromJson(needs.build_and_deploy_target_folder.outputs.component_matrix) }} - - name: "${{matrix.component.name}} - ${{matrix.component.namespace}} - Build and push docker image" - - steps: - # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.' - - uses: data-intuitive/reclaim-the-bytes@v2 - - - uses: actions/checkout@v7 - - - uses: viash-io/viash-actions/setup@v6 - - - name: Build container - uses: viash-io/viash-actions/ns-build@v6 - with: - config_mod: | - .version := "${{ inputs.version || format('{0}_build', github.ref_name) }}" - .engines[.type == 'docker'].target_tag := '${{ (github.event_name == 'push' && !inputs.is_workflow_call) && 'main_build' || inputs.target_tag }}' - engine: docker - src: ${{ matrix.component.dir }} - setup: build - - - name: Login to container registry - uses: docker/login-action@v4 - with: - registry: ghcr.io - username: ${{ secrets.GTHB_USER }} - password: ${{ secrets.GTHB_PAT }} - - - name: Push container - uses: viash-io/viash-actions/ns-build@v6 - with: - config_mod: .version := "${{ inputs.version || format('{0}_build', github.ref_name) }}" - engine: docker - src: ${{ matrix.component.dir }} - setup: push \ No newline at end of file From 70917207068c48708f9c9b45a7c53da5af76e667 Mon Sep 17 00:00:00 2001 From: dorien-er Date: Tue, 7 Jul 2026 10:34:49 +0200 Subject: [PATCH 11/13] update changelog --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b393fc..7364ba2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,16 @@ # openpipeline_composed x.x.x +## NEW FUNCTIONALITY + * `workflows/single_cell/parallel_annotation`: Add consensus voting based on per-method weighted probabilities via `--run_consensus` flag (default is true) (PR #22). +## MINOR CHANGES + +* Migration of test resources to the package-specific `s3://openpipelines-bio/openpipeline_composed/resources_test` bucket (#24): + + - Add `.info.test_resources` to `_viash.yaml` to specify where test resources need to be synced from. + - Test resources were regenerated via the scripts in `resources_test_scripts/`. + # openpipeline_composed 0.2.1 ## MINOR CHANGES From c58216caf5e559d6db4f3821e9ff24930dfdc1a7 Mon Sep 17 00:00:00 2001 From: Dorien <41797896+dorien-er@users.noreply.github.com> Date: Tue, 7 Jul 2026 13:48:13 +0200 Subject: [PATCH 12/13] Update CHANGELOG.md Co-authored-by: Jakub Majercik <57993790+jakubmajercik@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7364ba2..ad873ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ ## MINOR CHANGES -* Migration of test resources to the package-specific `s3://openpipelines-bio/openpipeline_composed/resources_test` bucket (#24): +* Migration of test resources to the package-specific `s3://openpipelines-bio/openpipeline_composed/resources_test` bucket (PR #24): - Add `.info.test_resources` to `_viash.yaml` to specify where test resources need to be synced from. - Test resources were regenerated via the scripts in `resources_test_scripts/`. From c0cc51a22fd602bc1d0580f2cb91cbaa4acd5db9 Mon Sep 17 00:00:00 2001 From: Dorien <41797896+dorien-er@users.noreply.github.com> Date: Tue, 7 Jul 2026 13:48:20 +0200 Subject: [PATCH 13/13] Update resources_test_scripts/reference_gencodev41.sh Co-authored-by: Jakub Majercik <57993790+jakubmajercik@users.noreply.github.com> --- resources_test_scripts/reference_gencodev41.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources_test_scripts/reference_gencodev41.sh b/resources_test_scripts/reference_gencodev41.sh index 08e24ce..8b33250 100644 --- a/resources_test_scripts/reference_gencodev41.sh +++ b/resources_test_scripts/reference_gencodev41.sh @@ -65,7 +65,7 @@ rm -f "${OUT}"/*.state.yaml rm -f "${OUT}"/reference.*.gz aws s3 sync \ - resources_test/reference_gencodev41_chr1 \ + "$OUT"\ s3://openpipelines-bio/openpipeline_composed/resources_test/reference_gencodev41_chr1 \ --delete \ --dryrun