Skip to content

Commit 0f04ea7

Browse files
logannyelogannye
authored
Overhaul Rosalind as deterministic analyzer platform (#94)
1 parent 5fc6674 commit 0f04ea7

78 files changed

Lines changed: 6775 additions & 1310 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/ci.yml‎

Lines changed: 48 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,36 @@ on:
66
pull_request:
77

88
jobs:
9+
macos-smoke:
10+
name: macOS smoke
11+
runs-on: macos-latest
12+
steps:
13+
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
14+
- uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable
15+
with:
16+
toolchain: stable
17+
- name: Build and exercise reference-pack CLI
18+
run: |
19+
cargo build --locked --bin rosalind
20+
target/debug/rosalind reference build \
21+
--fasta examples/data/illumina_toy/reference.fa \
22+
--output "$RUNNER_TEMP/reference.rref"
23+
target/debug/rosalind reference inspect \
24+
--reference-pack "$RUNNER_TEMP/reference.rref" --json
25+
26+
advisory-audit:
27+
name: Rust advisory audit
28+
runs-on: ubuntu-latest
29+
steps:
30+
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
31+
- uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable
32+
with:
33+
toolchain: stable
34+
- name: Install pinned cargo-audit
35+
run: cargo install cargo-audit --version 0.22.1 --locked
36+
- name: Check RustSec advisories
37+
run: cargo audit
38+
939
rust:
1040
runs-on: ubuntu-latest
1141
steps:
@@ -78,8 +108,8 @@ jobs:
78108
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
79109
with:
80110
python-version: '3.11'
81-
- name: Install numpy
82-
run: pip install numpy
111+
- name: Install Python feature dependencies
112+
run: pip install numpy pyarrow
83113
- name: Build the rosalind binary
84114
run: cargo build --release --bin rosalind
85115
- name: Exercise the Python boundary (bit-reproducible feature substrate)
@@ -238,6 +268,22 @@ jobs:
238268
budget-mb: 4096
239269
binary-path: target/release/rosalind
240270
output: examples/data/illumina_toy/action.vcf
271+
- name: Build the preferred analysis reference for Action compatibility
272+
run: |
273+
target/release/rosalind reference build \
274+
--fasta examples/data/illumina_toy/reference.fa \
275+
--output examples/data/illumina_toy/action.rref
276+
- name: Exercise reference-pack and Arrow Action inputs
277+
uses: ./
278+
with:
279+
reference-pack: examples/data/illumina_toy/action.rref
280+
alignments: examples/data/illumina_toy/sorted.bam
281+
analyzer: features
282+
format: arrow-ipc
283+
budget-mb: 4096
284+
binary-path: target/release/rosalind
285+
output: examples/data/illumina_toy/action.arrow
286+
artifact-name: rosalind-reference-pack-arrow
241287

242288
msrv:
243289
name: MSRV (1.83)

‎.github/workflows/container.yml‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
name: Rosalind OCI image
2+
3+
on:
4+
workflow_dispatch:
5+
release:
6+
types: [published]
7+
8+
permissions:
9+
contents: read
10+
packages: write
11+
id-token: write
12+
attestations: write
13+
14+
jobs:
15+
image:
16+
runs-on: ubuntu-latest
17+
environment: release
18+
steps:
19+
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
20+
- uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
21+
- uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
22+
with:
23+
registry: ghcr.io
24+
username: ${{ github.actor }}
25+
password: ${{ secrets.GITHUB_TOKEN }}
26+
- id: build
27+
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
28+
with:
29+
context: .
30+
file: containers/rosalind/Dockerfile
31+
platforms: linux/amd64
32+
push: true
33+
tags: ghcr.io/${{ github.repository }}:${{ github.event.release.tag_name || github.sha }}
34+
provenance: mode=max
35+
sbom: true
36+
- name: Offline non-root smoke
37+
run: |
38+
image="ghcr.io/${{ github.repository }}@${{ steps.build.outputs.digest }}"
39+
docker pull "$image"
40+
test "$(docker run --rm --network none --entrypoint id "$image" -u)" = 10001
41+
docker run --rm --network none "$image" --version
42+
- uses: actions/attest-build-provenance@e8998f949152b193b063cb0ec769d69d929409be # v2
43+
with:
44+
subject-name: ghcr.io/${{ github.repository }}
45+
subject-digest: ${{ steps.build.outputs.digest }}
46+
push-to-registry: true
47+
- name: Emit immutable lock candidate
48+
env:
49+
DIGEST: ${{ steps.build.outputs.digest }}
50+
RELEASE: ${{ github.event.release.tag_name || github.sha }}
51+
run: |
52+
python3 - <<'PY'
53+
import json, os
54+
lock = json.load(open("integrations/container-lock.json"))
55+
lock.update(digest=os.environ["DIGEST"], release=os.environ["RELEASE"], status="published")
56+
open("container-lock.generated.json", "w").write(json.dumps(lock, indent=2, sort_keys=True) + "\n")
57+
PY
58+
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
59+
with:
60+
name: rosalind-container-lock
61+
path: container-lock.generated.json

‎.github/workflows/fuzz.yml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
name: Scheduled fuzz smoke
2+
3+
on:
4+
schedule:
5+
- cron: '17 4 * * 1'
6+
workflow_dispatch:
7+
8+
permissions:
9+
contents: read
10+
11+
jobs:
12+
receipt-replay-parsers:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
16+
- uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable
17+
with:
18+
toolchain: nightly
19+
components: llvm-tools-preview
20+
- name: Install pinned cargo-fuzz
21+
run: cargo install cargo-fuzz --version 0.12.0 --locked
22+
- name: Fuzz canonical receipt parser
23+
run: cargo fuzz run canonical_json -- -max_total_time=120
24+
- name: Fuzz tokenized replay parser
25+
run: cargo fuzz run command_argv -- -max_total_time=120
26+
- name: Fuzz historical command parser
27+
run: cargo fuzz run legacy_command -- -max_total_time=120

‎.github/workflows/giab.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,9 +126,13 @@ jobs:
126126
benchmarks/giab/data/results/credibility.md
127127
benchmarks/giab/data/results/metrics-all.json
128128
benchmarks/giab/data/results/metrics-pass.json
129+
benchmarks/giab/data/results/reference-verify.json
129130
benchmarks/giab/data/results/happy/
131+
benchmarks/giab/data/results/GRCh38.chr20.rref
132+
benchmarks/giab/data/results/GRCh38.chr20.rref.manifest.json
130133
benchmarks/giab/data/results/HG002.rosalind.chr20.vcf
131134
benchmarks/giab/data/results/HG002.rosalind.chr20.vcf.manifest.json
135+
benchmarks/giab/data/results/verify.json
132136
- name: Preserve benchmark comparison outcome
133137
if: always()
134138
run: test '${{ steps.benchmark.outputs.exit_code }}' = 0

‎.github/workflows/integrations.yml‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
name: Workflow integrations
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches: [main]
7+
8+
permissions:
9+
contents: read
10+
11+
jobs:
12+
snakemake:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
16+
- uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable
17+
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
18+
with:
19+
python-version: '3.11'
20+
- name: Install pinned workflow tools
21+
run: |
22+
python -m pip install snakemake==9.8.1
23+
sudo apt-get update
24+
sudo apt-get install -y samtools
25+
- name: Prepare indexed toy BAM and CLI
26+
run: |
27+
cargo build --locked --bin rosalind
28+
samtools view -b -F 4 examples/data/illumina_toy/alignments.bam |
29+
samtools sort -o examples/data/illumina_toy/sorted.bam
30+
samtools index examples/data/illumina_toy/sorted.bam
31+
echo "$PWD/target/debug" >> "$GITHUB_PATH"
32+
- name: Lint, dry-run, and execute local fan-in
33+
run: |
34+
snakemake --lint --snakefile integrations/snakemake/Snakefile \
35+
--configfile integrations/snakemake/tests/config.yaml
36+
snakemake --dry-run --cores 1 --snakefile integrations/snakemake/Snakefile \
37+
--configfile integrations/snakemake/tests/config.yaml
38+
snakemake --cores 1 --snakefile integrations/snakemake/Snakefile \
39+
--configfile integrations/snakemake/tests/config.yaml
40+
test -s results/toy/merged.arrow
41+
test -s results/toy/merged.arrow.manifest.json
42+
test -s results/toy/verify.json
43+
44+
nextflow:
45+
runs-on: ubuntu-latest
46+
steps:
47+
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
48+
- uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable
49+
- uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4
50+
with:
51+
distribution: temurin
52+
java-version: '21'
53+
- uses: nf-core/setup-nextflow@98e3b21a1c45e26529208849ec20e5cfb7086040 # v2
54+
with:
55+
version: '25.04.8'
56+
- name: Parse and execute the local shard graph
57+
run: |
58+
sudo apt-get update
59+
sudo apt-get install -y samtools
60+
cargo build --locked --bin rosalind
61+
mkdir -p "$RUNNER_TEMP/nextflow-inputs"
62+
samtools view -b -F 4 examples/data/illumina_toy/alignments.bam |
63+
samtools sort -o "$RUNNER_TEMP/nextflow-inputs/sample.bam"
64+
samtools index "$RUNNER_TEMP/nextflow-inputs/sample.bam"
65+
target/debug/rosalind reference build \
66+
--fasta examples/data/illumina_toy/reference.fa \
67+
--output "$RUNNER_TEMP/nextflow-inputs/reference.rref"
68+
nextflow -c integrations/nextflow/examples/local/nextflow.config \
69+
config integrations/nextflow/examples/local/main.nf >/tmp/nextflow-config.txt
70+
PATH="$PWD/target/debug:$PATH" nextflow run integrations/nextflow/examples/local/main.nf \
71+
-work-dir "$RUNNER_TEMP/nextflow-work" \
72+
-with-trace "$RUNNER_TEMP/nextflow-trace.txt" \
73+
-c integrations/nextflow/examples/local/nextflow.test.config \
74+
--reference "$RUNNER_TEMP/nextflow-inputs/reference.rref" \
75+
--bam "$RUNNER_TEMP/nextflow-inputs/sample.bam" \
76+
--bai "$RUNNER_TEMP/nextflow-inputs/sample.bam.bai" \
77+
--shard_count 2 \
78+
--image local/rosalind@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
79+
awk -F '\t' 'NR > 1 && $5 != "COMPLETED" { exit 1 } END { if (NR != 8) exit 1 }' \
80+
"$RUNNER_TEMP/nextflow-trace.txt"

‎.github/workflows/platform-benchmark.yml‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
name: Platform benchmark
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
dataset:
7+
description: toy or hg002-chr20
8+
required: true
9+
default: toy
10+
type: choice
11+
options: [toy, hg002-chr20]
12+
schedule:
13+
- cron: '43 9 15 * *'
14+
15+
permissions:
16+
contents: read
17+
18+
concurrency:
19+
group: platform-benchmark-${{ inputs.dataset || 'toy' }}
20+
cancel-in-progress: false
21+
22+
jobs:
23+
compare:
24+
runs-on: ubuntu-latest
25+
timeout-minutes: 360
26+
steps:
27+
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
28+
- name: Select and prepare data
29+
id: data
30+
env:
31+
DATASET: ${{ inputs.dataset || 'toy' }}
32+
run: |
33+
if [ "$DATASET" = hg002-chr20 ]; then
34+
available_kb=$(df -Pk . | awk 'NR==2 {print $4}')
35+
test "$available_kb" -ge 12582912
36+
sudo apt-get update
37+
sudo apt-get install -y samtools
38+
benchmarks/giab/prepare.sh
39+
{
40+
echo "reference=$PWD/benchmarks/giab/data/prepared/GRCh38.chr20.fa"
41+
echo "bam=$PWD/benchmarks/giab/data/prepared/HG002.chr20.bam"
42+
echo "results=$PWD/benchmarks/platform/results-hg002"
43+
} >> "$GITHUB_OUTPUT"
44+
else
45+
{
46+
echo "reference=$PWD/examples/data/illumina_toy/reference.fa"
47+
echo "bam=$PWD/examples/data/illumina_toy/alignments.bam"
48+
echo "results=$PWD/benchmarks/platform/results-toy"
49+
} >> "$GITHUB_OUTPUT"
50+
fi
51+
- name: Run the digest-pinned comparison and memory probes
52+
env:
53+
REFERENCE: ${{ steps.data.outputs.reference }}
54+
BAM: ${{ steps.data.outputs.bam }}
55+
RESULTS: ${{ steps.data.outputs.results }}
56+
run: benchmarks/platform/run.sh "$REFERENCE" "$BAM" "$RESULTS"
57+
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
58+
if: always()
59+
with:
60+
name: platform-${{ inputs.dataset || 'toy' }}-${{ github.sha }}
61+
retention-days: 90
62+
if-no-files-found: error
63+
path: |
64+
benchmarks/platform/results-toy/
65+
benchmarks/platform/results-hg002/

0 commit comments

Comments
 (0)