Overhaul Rosalind as deterministic analyzer platform (#94) #167
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| jobs: | |
| macos-smoke: | |
| name: macOS smoke | |
| runs-on: macos-latest | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable | |
| with: | |
| toolchain: stable | |
| - name: Build and exercise reference-pack CLI | |
| run: | | |
| cargo build --locked --bin rosalind | |
| target/debug/rosalind reference build \ | |
| --fasta examples/data/illumina_toy/reference.fa \ | |
| --output "$RUNNER_TEMP/reference.rref" | |
| target/debug/rosalind reference inspect \ | |
| --reference-pack "$RUNNER_TEMP/reference.rref" --json | |
| advisory-audit: | |
| name: Rust advisory audit | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable | |
| with: | |
| toolchain: stable | |
| - name: Install pinned cargo-audit | |
| run: cargo install cargo-audit --version 0.22.1 --locked | |
| - name: Check RustSec advisories | |
| run: cargo audit | |
| rust: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - name: Cache cargo artifacts | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo- | |
| - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable | |
| with: | |
| toolchain: stable | |
| - name: Cargo fmt | |
| run: cargo fmt --all -- --check | |
| - name: Cargo test | |
| run: cargo test --workspace | |
| - name: Synthetic chr20 GIAB smoke | |
| run: | | |
| benchmarks/giab/smoke.sh | |
| python3 -m unittest discover -s benchmarks/giab -p 'test_*.py' | |
| - name: Build examples | |
| run: cargo build --examples | |
| - name: Build docs | |
| run: cargo doc --workspace --no-deps | |
| - name: Compile receipt parser fuzz targets | |
| run: cargo check --manifest-path fuzz/Cargo.toml --bins | |
| - name: Package publishable crates and embedded assets | |
| run: | | |
| cargo package -p rosalind-build-info --allow-dirty | |
| cargo package -p rosalind-receipt --allow-dirty | |
| cargo package -p rosalind-bio --allow-dirty \ | |
| --config 'patch.crates-io.rosalind-receipt.path="crates/receipt"' \ | |
| --config 'patch.crates-io.rosalind-build-info.path="crates/build-info"' | |
| cargo package -p rosalind-bio --allow-dirty --list \ | |
| --config 'patch.crates-io.rosalind-receipt.path="crates/receipt"' \ | |
| --config 'patch.crates-io.rosalind-build-info.path="crates/build-info"' > package-files.txt | |
| grep -q '^web/verify/pkg/rosalind_verify_bg.wasm$' package-files.txt | |
| grep -q '^assets/demo/reference.fa$' package-files.txt | |
| grep -q '^assets/scaffold/Cargo.toml.template$' package-files.txt | |
| python: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - name: Cache cargo artifacts | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo- | |
| - name: Cache pip | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('.github/workflows/ci.yml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable | |
| with: | |
| toolchain: stable | |
| - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install Python feature dependencies | |
| run: pip install numpy pyarrow | |
| - name: Build the rosalind binary | |
| run: cargo build --release --bin rosalind | |
| - name: Exercise the Python boundary (bit-reproducible feature substrate) | |
| # Runs the documented entry point (python/rosalind.py) end-to-end: builds a | |
| # toy index + BAM, extracts features twice, and proves the inputs are | |
| # byte-identical with matching BLAKE3 receipts. | |
| run: python3 examples/reproducible_features_demo.py target/release/rosalind | |
| cli-e2e: | |
| runs-on: ubuntu-latest | |
| env: | |
| PYO3_PYTHON: python3 | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - name: Cache cargo artifacts | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo- | |
| - name: Cache pip | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('.github/workflows/ci.yml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | |
| with: | |
| python-version: '3.11' | |
| - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable | |
| with: | |
| toolchain: stable | |
| - name: Verify the committed toy reference is intact (pinned hash) | |
| # A real anti-corruption gate, not the old self-referential check: pin the | |
| # COMMITTED reference.fa (the one the release bundle + every other job | |
| # ships) to a known hash. The previous step regenerated reference.fa at | |
| # 1 Mbp without --length, overwriting the committed 4 kbp fixture and | |
| # checking freshly-generated files against freshly-generated sums (a | |
| # tautology that also tested a different reference than ships). | |
| run: | | |
| echo "ce8a6df801e836f4eda0b7661a68dcd7b7a9ab247c66d4582a89dbc207554efa examples/data/illumina_toy/reference.fa" \ | |
| | sha256sum --check | |
| - name: Generate reads from the committed toy reference | |
| # --reference simulates reads FROM the committed reference (does not invent | |
| # or overwrite one), so cli-e2e exercises exactly the fixture that ships. | |
| run: python3 scripts/generate_toy_data.py examples/data/illumina_toy --reference examples/data/illumina_toy/reference.fa | |
| - name: Align to SAM | |
| run: | | |
| cargo run --release -- align \ | |
| --reference examples/data/illumina_toy/reference.fa \ | |
| --reads examples/data/illumina_toy/reads_R1.fastq \ | |
| --format sam \ | |
| --output examples/data/illumina_toy/alignments.sam | |
| - name: Align to BAM | |
| run: | | |
| cargo run --release -- align \ | |
| --reference examples/data/illumina_toy/reference.fa \ | |
| --reads examples/data/illumina_toy/reads_R1.fastq \ | |
| --format bam \ | |
| --force \ | |
| --output examples/data/illumina_toy/alignments.bam | |
| - name: Call variants | |
| run: | | |
| cargo run --release -- variants \ | |
| --reference examples/data/illumina_toy/reference.fa \ | |
| --alignments examples/data/illumina_toy/alignments.sam \ | |
| --mapq-threshold 5 \ | |
| --output examples/data/illumina_toy/variants.vcf | |
| - name: Sanity-check outputs | |
| run: | | |
| test -s examples/data/illumina_toy/alignments.sam | |
| test -s examples/data/illumina_toy/alignments.bam | |
| grep -q '^#CHROM' examples/data/illumina_toy/variants.vcf | |
| grep -q -v '^#' examples/data/illumina_toy/variants.vcf || echo \"warning: no variant lines emitted\" | |
| - name: Build index for the bounded contract path | |
| run: | | |
| cargo run --release -- index \ | |
| --reference examples/data/illumina_toy/reference.fa \ | |
| --output examples/data/illumina_toy/reference.idx | |
| - name: Sort the BAM for the contract path | |
| run: | | |
| cargo run --release -- sort \ | |
| --input examples/data/illumina_toy/alignments.bam \ | |
| --output examples/data/illumina_toy/sorted.bam | |
| - name: Contract gate — fits / verify / refuse (bounded --index path) | |
| run: | | |
| set -e | |
| # (1) Generous budget fits -> exit 0 + "contract: OK". | |
| cargo run --release -- variants --index examples/data/illumina_toy/reference.idx \ | |
| --alignments examples/data/illumina_toy/sorted.bam \ | |
| --memory-budget-mb 4096 --enforce \ | |
| -o examples/data/illumina_toy/contract.vcf 2> ok.log | |
| grep -q "contract: OK" ok.log | |
| # (2) Verify the receipt without re-running -> exit 0. | |
| cargo run --release -- verify \ | |
| --manifest examples/data/illumina_toy/contract.vcf.manifest.json | grep -q "verify: OK" | |
| # (3) 1 MiB budget refuses up front -> exit 3, and writes no VCF. | |
| rm -f examples/data/illumina_toy/refused.vcf | |
| set +e | |
| cargo run --release -- variants --index examples/data/illumina_toy/reference.idx \ | |
| --alignments examples/data/illumina_toy/sorted.bam \ | |
| --memory-budget-mb 1 --enforce \ | |
| -o examples/data/illumina_toy/refused.vcf 2> refuse.log | |
| code=$? | |
| set -e | |
| test "$code" -eq 3 | |
| grep -q "REFUSE" refuse.log | |
| test ! -f examples/data/illumina_toy/refused.vcf | |
| - name: Reproduce fence — a stranger re-derives the result on this runner | |
| # The contract.vcf + its receipt were produced above; reproduce re-derives it | |
| # byte-for-byte from content-located inputs on THIS runner (a different machine | |
| # than the author's), writes a chainable certificate, and a byte-changed input | |
| # is correctly reported INCONCLUSIVE (exit 7) — never a false DIVERGED. | |
| run: | | |
| set -e | |
| D=examples/data/illumina_toy | |
| cargo run --release -- reproduce \ | |
| --manifest $D/contract.vcf.manifest.json \ | |
| --inputs $D/ | tee reproduce.log | |
| grep -q "REPRODUCED" reproduce.log | |
| test -f $D/contract.vcf.manifest.json.repro.json | |
| # Negative: a byte-changed input can no longer be content-located -> exit 7. | |
| mkdir -p tamper && cp $D/reference.idx tamper/ && cp $D/sorted.bam tamper/ | |
| printf 'x' >> tamper/sorted.bam | |
| set +e | |
| cargo run --release -- reproduce \ | |
| --manifest $D/contract.vcf.manifest.json --inputs tamper/ > /dev/null 2>&1 | |
| code=$? | |
| set -e | |
| test "$code" -eq 7 | |
| - name: Emit a self-hosted reproducibility badge | |
| run: | | |
| D=examples/data/illumina_toy | |
| cargo run --release -- badge \ | |
| --manifest $D/contract.vcf.manifest.json \ | |
| --repro $D/contract.vcf.manifest.json.repro.json \ | |
| -o badge.json | |
| cat badge.json | |
| - name: Upload the reproducibility badge | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: rosalind-reproducibility-badge | |
| path: badge.json | |
| if-no-files-found: ignore | |
| - name: Exercise the rosalind-budget Action (local, from-source binary) | |
| uses: ./ | |
| with: | |
| index: examples/data/illumina_toy/reference.idx | |
| alignments: examples/data/illumina_toy/sorted.bam | |
| budget-mb: 4096 | |
| binary-path: target/release/rosalind | |
| output: examples/data/illumina_toy/action.vcf | |
| - name: Build the preferred analysis reference for Action compatibility | |
| run: | | |
| target/release/rosalind reference build \ | |
| --fasta examples/data/illumina_toy/reference.fa \ | |
| --output examples/data/illumina_toy/action.rref | |
| - name: Exercise reference-pack and Arrow Action inputs | |
| uses: ./ | |
| with: | |
| reference-pack: examples/data/illumina_toy/action.rref | |
| alignments: examples/data/illumina_toy/sorted.bam | |
| analyzer: features | |
| format: arrow-ipc | |
| budget-mb: 4096 | |
| binary-path: target/release/rosalind | |
| output: examples/data/illumina_toy/action.arrow | |
| artifact-name: rosalind-reference-pack-arrow | |
| msrv: | |
| name: MSRV (1.83) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - name: Cache cargo artifacts | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ${{ runner.os }}-cargo-msrv-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo-msrv- | |
| - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable | |
| with: | |
| toolchain: 1.83.0 | |
| - name: cargo check on the declared MSRV | |
| run: cargo check --all-targets | |
| clippy: | |
| name: Clippy (-D warnings) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - name: Cache cargo artifacts | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ${{ runner.os }}-cargo-clippy-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo-clippy- | |
| - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable | |
| with: | |
| toolchain: stable | |
| components: clippy | |
| - name: Clippy as a gate | |
| run: cargo clippy --all-targets -- -D warnings | |
| benchmarks: | |
| name: Claims harness | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - name: Cache cargo artifacts | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo- | |
| - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable | |
| with: | |
| toolchain: stable | |
| - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Run the claims harness (a standing regression gate on the contract + reproducibility) | |
| run: bash benchmarks/run.sh | |
| studio-assets: | |
| name: Receipt Studio generated assets | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable | |
| with: | |
| toolchain: 1.83.0 | |
| targets: wasm32-unknown-unknown | |
| - uses: taiki-e/install-action@c7eb1735f09259a5035e8e5d44b1406b1cddc0fb # v2 | |
| with: | |
| tool: wasm-pack@0.14.0 | |
| - name: Rebuild committed client-only WASM | |
| env: | |
| RUSTUP_TOOLCHAIN: 1.83.0 | |
| run: scripts/build-wasm-verifier.sh | |
| - name: Generated assets are reproducible | |
| run: git diff --exit-code -- web/verify/pkg crates/receipt-wasm/Cargo.lock | |
| - name: Upload generated assets on mismatch | |
| if: failure() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: receipt-studio-generated-${{ github.run_id }} | |
| path: web/verify/pkg | |
| if-no-files-found: error | |
| - name: WASM unit tests | |
| run: cargo test --manifest-path crates/receipt-wasm/Cargo.toml | |
| release-automation: | |
| name: Maintainer automation dry run | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable | |
| - name: Install pinned workflow linters | |
| run: | | |
| curl -fsSLO https://github.com/rhysd/actionlint/releases/download/v1.7.7/actionlint_1.7.7_linux_amd64.tar.gz | |
| echo '023070a287cd8cccd71515fedc843f1985bf96c436b7effaecce67290e7e0757 actionlint_1.7.7_linux_amd64.tar.gz' | sha256sum --check | |
| tar -xzf actionlint_1.7.7_linux_amd64.tar.gz actionlint | |
| sudo install actionlint /usr/local/bin/actionlint | |
| curl -fsSLO https://github.com/koalaman/shellcheck/releases/download/v0.10.0/shellcheck-v0.10.0.linux.x86_64.tar.xz | |
| echo '6c881ab0698e4e6ea235245f22832860544f17ba386442fe7e9d629f8cbedf87 shellcheck-v0.10.0.linux.x86_64.tar.xz' | sha256sum --check | |
| tar -xJf shellcheck-v0.10.0.linux.x86_64.tar.xz | |
| sudo install shellcheck-v0.10.0/shellcheck /usr/local/bin/shellcheck | |
| python3 -m pip install 'jsonschema==4.23.0' | |
| - name: Maintainer CLI fake-remote integration suite | |
| run: cargo test -p xtask --locked | |
| - name: Validate schemas, workflows, and shell entry points | |
| run: | | |
| set +e | |
| cargo xtask doctor --json > maintainer-report.json | |
| doctor_code=$? | |
| set -e | |
| if [ "$doctor_code" -ne 0 ] && [ "$doctor_code" -ne 3 ]; then exit "$doctor_code"; fi | |
| cargo xtask partners init --persona analyzer-builder --output partner-packet | |
| python3 - <<'PY' | |
| import json | |
| from pathlib import Path | |
| from jsonschema import Draft202012Validator, validate | |
| for path in Path("release/schemas").glob("*.json"): | |
| Draft202012Validator.check_schema(json.load(open(path))) | |
| validate(json.load(open("maintainer-report.json")), json.load(open("release/schemas/maintainer-report-v1.schema.json"))) | |
| validate(json.load(open("partner-packet/feedback.json")), json.load(open("release/schemas/design-partner-v1.schema.json"))) | |
| PY | |
| scripts/verify-release-automation.py | |
| actionlint | |
| shellcheck scripts/release-publish.sh scripts/post-publish-smoke.sh \ | |
| scripts/stage-release.sh benchmarks/giab/prepare.sh benchmarks/giab/run.sh \ | |
| benchmarks/giab/happy/run.sh assets/scaffold/scripts/contract-check.sh | |
| bash -n scripts/release-publish.sh scripts/post-publish-smoke.sh \ | |
| scripts/stage-release.sh benchmarks/giab/prepare.sh benchmarks/giab/run.sh \ | |
| benchmarks/giab/happy/run.sh assets/scaffold/scripts/contract-check.sh |