From 24443c6f059916f359deb002752426b8de9374ed Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 16:43:59 -0700 Subject: [PATCH 01/32] =?UTF-8?q?docs(spec):=20Phase=20C=20=E2=80=94=20mem?= =?UTF-8?q?ory=20as=20a=20verifiable=20contract=20(variants=20path)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design for the Phase-C contract increment on the bounded `variants --index` path: declare → predict → honor-or-refuse → verify. Decomposed C1 (working-set soundness fix + deterministic depth cap + per-call VCF row flush + decode-then- move reference) → C2 (`rosalind plan` + `--enforce`) → C3 (`rosalind verify` + receipt-on-stdout + CI contract gate). Index build stays record-only; somatic and graceful-degrade/spill out of scope (Phase D/E). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../2026-06-01-phase-c-contract-design.md | 300 ++++++++++++++++++ 1 file changed, 300 insertions(+) create mode 100644 docs/superpowers/specs/2026-06-01-phase-c-contract-design.md diff --git a/docs/superpowers/specs/2026-06-01-phase-c-contract-design.md b/docs/superpowers/specs/2026-06-01-phase-c-contract-design.md new file mode 100644 index 0000000..9dfbd8e --- /dev/null +++ b/docs/superpowers/specs/2026-06-01-phase-c-contract-design.md @@ -0,0 +1,300 @@ +# Phase C — memory as a verifiable contract (variants path) (design) + +**Status:** Spec for review — 2026-06-01. The Phase-C sub-stage that turns Rosalind's +already-real *memory receipt* into a *contract* on the bounded whole-genome variants path: +**declare → predict → honor-or-refuse → verify.** Builds directly on the Phase-B4 caller +(`rosalind variants --index`, merged — PR #19). Under the contract thesis in +[`docs/OPEN_PROBLEMS.md`](../../OPEN_PROBLEMS.md) and the strategy reframe +(the contract is the shippable breakthrough; √t is the future Phase-D knob). + +## 1. The capability we are shooting for + +> **You declare a RAM budget; `rosalind plan` tells you *before you commit a byte* whether your +> whole-genome germline call will fit; the run *honors* that budget — fitting cleanly or refusing +> cleanly, never silently OOM-killed mid-job; and `rosalind verify` re-checks a deterministic, +> BLAKE3-stamped receipt proving the realized peak landed inside your budget and the VCF came from +> exactly these inputs.** + +Today (post-B4) the engine already *measures* and *records* peak memory — `peak_rss_bytes()` is a real +`getrusage` signal (`util/rss.rs`), and the manifest carries `peak_rss_bytes` + `max_working_set_bytes` +as BLAKE3-stamped canonical JSON (`provenance/mod.rs`, written at `main.rs:1083-1094`). But the budget +check is **record-only** (`main.rs:1109-1120` computes `budget.admits(peak_rss)` then prints +*"record-only, run completed"*), there is **no pre-run prediction** for the streaming path, the +**stdout path writes no manifest** (`main.rs:1097-1101`), and the reported `max_working_set_bytes` is a +**real under-count** of true peak. Phase C closes exactly those four gaps. No incumbent ships this +four-property contract; it is unique *without* √t (which extends the same contract down to index +construction in Phase D). + +## 2. The contract math — the four working-set terms + +For the bounded whole-genome variants drive (`call/whole_genome.rs`), true peak working set is the sum +of four terms. Phase C makes each one either *exactly known* or *hard-bounded*, so a prediction can be a +true upper bound and the realized high-water can be measured exactly. + +| Term | Bytes | Knowable a priori? | How Phase C bounds it | +|---|---|---|---| +| **Reference** (current contig) | `largest_contig_len` | ✅ from index header | decode-then-**move** into the `Arc` — eliminates the `buf`+`Arc` double-copy at `whole_genome.rs:63-64` (**2× → 1×**) | +| **Active read set** | `D × max_read_len × per_base_cost` | ⚠️ depth **capped**, read-len **assumed** | **hard cap at `--max-depth D`** (deterministic); read length is the one residual assumption | +| **Resident VCF rows** | O(1) (writer buffer) | ✅ constant | **stream each call to the writer as produced** — no genome-wide row Vec; the only resident "rows" memory is the `BufWriter`'s fixed buffer | +| **Fixed overhead** | const | ✅ | constant | + +Two numbers fall out: + +- **Predicted** (`rosalind plan`, pre-run, from index header + declared `D` + assumed max-read-len `L`): + a true upper bound *modulo* the read-length assumption. +- **Realized** (the receipt's high-water): **exact** — sampled from a *corrected* accountant that + includes all four terms. + +**The honest guarantee.** Depth is hard-capped (guaranteed), rows are streamed (guaranteed small), +reference is exactly the largest contig (1× after the move fix). The only residual assumption is max +read length — and `--enforce` **fails loud post-run if realized > budget** (a clean non-zero exit, never +a silent overrun). So the brand is **"fits-or-tells-you up front, and proves the realized peak with a +receipt"** — explicitly **not** "never refuses" (graceful degrade/spill needs the Phase-D ladder and is +out of scope here; without it `--enforce` can only refuse cleanly). + +## 3. Scope + +**In (the variants `--index` path only):** +- **C1 — soundness fix.** Make `max_working_set_bytes` a true conservative upper bound; deterministic + `--max-depth` cap; per-call streaming VCF flush; decode-then-move reference (2×→1×). *Output-preserving + by default* (cap defaults off in C1). +- **C2 — `rosalind plan` + `--enforce`.** A pre-run streaming estimator; the `plan` subcommand; + honor-or-refuse enforcement (refuse-up-front-on-predicted / fail-loud-post-run-on-realized). The + `--max-depth` default (1000) lands here. +- **C3 — `rosalind verify` + receipt-on-stdout + CI contract gate.** A parser for the canonical manifest; + the `verify` subcommand; always-persist-a-receipt; the CI contract suite. + +**Out (deferred, by design):** +- **Index-build enforcement.** The build is O(reference); `rosalind index --memory-budget-mb` / + `plan --reference` stay **advisory/record-only** (enforcement waits for Phase-D sublinear construction). +- **Somatic.** `call/pipeline.rs` collects both column streams into `Vec`s — region-bounded, not + whole-genome-bounded. A region-bound refactor is out of scope; the bounded contract is scoped to the + germline `--index` path and the docs say so. +- **Graceful degrade / external-memory spill.** The Phase-D √t/spill ladder. `--enforce` here refuses + cleanly; it does not degrade. +- **On-demand `ref_base`** (shrinking the reference term to O(pileup window)) → Phase D/E. +- **Deterministic multithreading / thread-invariance.** The engine is single-threaded; we do not claim + thread-invariance (it would be vacuously true) until a parallel path + gate exist (Phase E). + +## 4. Decomposition (one spec → three green sub-stages, one PR each) + +Mirrors the B3/B4 pattern. Each sub-stage lands green (`cargo test`, `cargo fmt --check`, 0 warnings) +with its own plan + PR. + +``` +Phase C — memory as a verifiable contract (variants path) +├─ C1 soundness fix [the hard correctness core] +├─ C2 rosalind plan + --enforce +└─ C3 rosalind verify + stdout receipt + CI contract gate +``` + +## 5. C1 — the soundness fix (the hard correctness core) + +The prerequisite for everything: until the modeled number is a true upper bound, any `plan` lies and the +brand inverts on first contact. C1 is **output-preserving by default** (the cap defaults to `None`); it +changes only the *accounting*, the *reference copy count*, and the *row-flush plumbing*. + +### 5.1 Corrected working-set accountant (`pileup/engine.rs:135`) +`PileupEngine::current_working_set()` today sums only `ref_to_read.len()*16 + 64` per active read + 256. +It omits (a) the engine's own `self.reference` bytes and (b) each active read's `seq`/`qual` bytes. Fix it +to count what is actually resident: +- `self.reference.len()` (the decoded contig the engine holds), +- per active read: `ref_to_read.len()*16` (map) **+ `seq.len()` + `qual.len()`** (the byte buffers) + a + small per-read constant, +- fixed engine overhead. + +This is the single load-bearing correctness change. (It only ever *grows* the reported number — it +cannot newly pass a budget it failed before.) + +### 5.2 Deterministic depth cap (`pileup/engine.rs`) +Add `PileupParams.max_depth: Option` (**default `None` in C1** — no behavior change; C2 sets the CLI +default). When `Some(D)` and the active set already holds `D` reads, **drop the incoming read** and count +it under a new `SkipCounts.over_max_depth`. Because the source is `(contig,pos)`-sorted (and `SliceSource` +sorts on construction), "the first `D` reads to overlap a position" is deterministic ⇒ capped output is +deterministic and order-independent. Output changes *only* at sites deeper than `D` (the artifact +pileups) — standard caller behavior. Applied in `advance_to`/`ingest`. + +### 5.3 Incremental VCF writer (`io/vcf.rs`) +Split the monolithic `write_germline_vcf(writer, contigs, sample, &rows)` into: +- `write_germline_header(writer, contigs, sample)` — header + `##contig` lines (once), +- `write_germline_row(writer, &row)` — one record. + +`write_germline_vcf` is retained as a thin wrapper (header + loop) for back-compat / the single-contig +path. Rows are produced in `(contig,pos)` order, so streaming them is **byte-identical** to the batch +write — the `golden_vcf` + `determinism` gates stay green. + +### 5.4 Streaming sink + true high-water (`call/whole_genome.rs:47`) +Change the drive from returning `(Vec, WorkingSet)` to driving a **row sink** and returning the +*true* high-water working set: +```rust +pub fn call_germline_whole_genome( + source: S, + ref_view: &ReferenceView, + contigs: &ContigSet, + pileup_params: PileupParams, + germline_params: &GermlineParams, + on_row: &mut dyn FnMut((Locus, u8, GermlineCall)) -> Result<(), CoreError>, +) -> Result +``` +- **decode-then-move:** `decode_window(.., &mut buf)` then `Arc::<[u8]>::from(std::mem::take(&mut buf))` + (moves the allocation; no second copy). Reference term 2× → 1×. +- each emitted `(Locus, ref_base, call)` is passed straight to `on_row` (no genome-wide `rows` Vec). +- the returned `WorkingSet` is `max over contigs of (engine peak working set + resident-row bytes for the + in-flight write batch)` — a sound upper bound on realized peak. + +**API-shape decision:** a **sink callback**, not an `Iterator`. It is bounded, trivial to retrofit, and a +clean substrate primitive in its own right (a consumer passes *any* sink: stream-to-VCF, collect, +featurize-into-tensors). An `Iterator` is more idiomatic but fights the borrow checker across the +per-contig reference-decode / contig-switch state machine; a `.calls()` iterator adapter over this sink is +an easy future addition for the front-door cookbook. + +`main.rs::run_variants_index` (`main.rs:1044-1101`) calls `write_germline_header` once, then passes a sink +that `write_germline_row`s into the (file or stdout) writer; the returned `WorkingSet` feeds the receipt. + +### 5.5 C1 tests (the proof) +- **Real-RSS soundness gate** (`tests/`): on a generated growing input with an explicit `--max-depth D`, + assert realized `peak_rss` is flat as the BAM grows **and** the realized working-set ≤ the modeled + bound. +- **Depth-cap determinism**: capped output byte-identical across shuffled input order; sites > `D` are the + only ones changed; `over_max_depth` counted. +- **Accountant completeness**: `current_working_set` includes reference + seq/qual (unit test on a known + active set). +- **VCF byte-identity**: header-split + streamed rows == the old batch write (extend `golden_vcf`). + +## 6. C2 — `rosalind plan` + `--enforce` + +### 6.1 Streaming estimator (`src/call/plan.rs`, new — pure, unit-tested) +A sibling to `genomics/index/report.rs::estimate_build_working_set`, for the call path: +```rust +pub fn estimate_variants_working_set(largest_contig_len: u64, max_depth: u32, max_read_len: u32) -> WorkingSet +``` +`predicted = largest_contig_len (ref 1×) + D·L·per_base_cost (active) + writer_buf_const + fixed`, using the +*same* per-base/per-read constants as the corrected C1 accountant (single source of truth — a shared +`const`/helper so the estimate and the realized accountant cannot drift). Needs **only** the index header +(`contigs.iter().map(|c| c.length).max()`) plus declared `D`/`L` — it never opens the BAM. Plus a pure +renderer `render_variants_plan(estimate_breakdown, budget) -> String` producing the multi-line +`[FITS]`/`[REFUSE]` breakdown. + +### 6.2 `rosalind plan` subcommand (new `Commands::Plan`) +- `plan --index [--max-depth D] [--max-read-len L] [--budget-mb B]` → predicts the **variants** peak + (the flagship): reference / active / rows / fixed → predicted peak / budget → `[FITS]`/`[REFUSE]`. +- `plan --reference [--budget-mb B]` → reuses `estimate_build_working_set` for the **build** + (advisory; build is O(reference) and Phase-D enforces). `--index` XOR `--reference`. +- Output is pure-rendered + deterministic (testable, like `render_plan_line`); exit 0 always (planning is + advisory; refusal is `variants --enforce`). + +### 6.3 `--enforce` on `variants --index` (new flag on `Commands::Variants`) +Replaces the record-only verdict (`main.rs:1109-1120`) when set; both checks require `--memory-budget-mb`: +- **Pre-run:** `estimate_variants_working_set(largest_contig, D, L)` > budget → **refuse**, exit **3**, + actionable stderr ("declared B MiB; predicted peak P MiB @ max-depth D / max-read-len L; raise + --memory-budget-mb, lower --max-depth, or drop --enforce"). No work performed. +- **Post-run:** realized `peak_rss` > budget → write the VCF **and** the manifest (recording the + violation), then **fail loud**, exit **4**. (The artifact + the proof-of-overrun are preserved; the + exit code signals the violated contract.) +- **Without `--enforce`:** today's record-only behavior, exit 0 (back-compat). + +A small `ContractStatus` / exit-code helper centralizes the codes (`0` ok, `3` predicted-over-refused, +`4` realized-over-completed) and the honest messages. + +### 6.4 Depth-cap default lands here +C2 sets the CLI `--max-depth` **default to 1000** (always applied ⇒ the engine is bounded by default); +`--max-depth 0` = uncapped opt-out (and then `plan` cannot promise a hard bound — say so). `--max-read-len` +defaults to a short-read value (e.g. 250) with a flag to raise it for long reads. This is the one +deliberate default-output change in Phase C; it is documented (changes calls only at >1000× artifact +sites) and motivated by the contract surface. + +### 6.5 C2 tests +`estimate_variants_working_set` monotonic + no-overflow; the breakdown renderer `[FITS]`/`[REFUSE]`; +`plan --index` / `plan --reference` subprocess tests; `--enforce` refuses up front (exit 3) on a tiny +budget with `--max-depth`; `--enforce` fails post-run (exit 4) on a budget below realized peak; +record-only path unchanged without `--enforce`. + +## 7. C3 — `rosalind verify` + receipt-on-stdout + CI contract gate + +### 7.1 Persist a receipt on stdout runs (`main.rs:1097-1101`) +Today the stdout branch writes no manifest, so "every run emits a receipt" is false on the default path. +Fix: stdout output → write a `rosalind.variants.manifest.json` sidecar in the cwd + announce it on stderr; +`--manifest ` redirects (works for both stdout and file output). Document the cwd-sidecar behavior. + +### 7.2 Self-describing manifest params +Extend the params written at `main.rs:1083-1094` with `memory_budget_mb` (if declared), `contract_verdict` +(`within`/`over`/`unset`), `enforced` (bool), `max_depth`, `max_read_len`. A receipt then carries +everything `verify` needs without the user re-supplying flags. + +### 7.3 Canonical-manifest parser (`provenance/mod.rs`) +Add `RunManifest::from_canonical_json(&str) -> Result` — a small parser for the +**fixed** canonical shape we emit (not a general JSON parser), keeping deps lean (no `serde_json` in a +public surface) and matching the hand-rolled writer. Guarded by a `serialize → parse → serialize == identity` +property test over varied manifests. + +### 7.4 `rosalind verify` subcommand (new `Commands::Verify`) +`verify --manifest [--budget-mb B]` — read-and-assert, **no re-run**: +- parse the manifest; re-hash each listed input/output file (BLAKE3, streamed via `blake3_file`) and + compare to the recorded digest → detects drift / proves "this VCF came from exactly these inputs"; +- re-check recorded `peak_rss_bytes` ≤ recorded `memory_budget_mb` (or the supplied `--budget-mb`); +- exit 0 if all hold; non-zero with a per-check report otherwise. Missing files / hash mismatch / over + budget each produce a clear, distinct message. + +### 7.5 CI contract suite +Extend `tests/rss_budget.rs` into a subprocess contract suite (style of `tests/index_cli.rs`): +1. predicted envelope ≥ realized peak on a generated growing input; +2. realized working-set flat as the BAM grows (bounded-by-coverage); +3. `--enforce` refuses up front (exit 3); +4. `--enforce` fails post-run on a tiny budget (exit 4); +5. `verify` round-trips a real run's manifest and catches a tampered output hash. + +Scope = add these to the existing suite (which CI already runs). A full CI-matrix overhaul (macOS runners, +clippy gate) stays Phase-E; clippy remains a Phase-E call under MSRV 1.72. + +## 8. Full CLI surface after Phase C + +``` +rosalind plan (--index | --reference ) [--max-depth D] [--max-read-len L] [--budget-mb B] +rosalind variants --index --alignments + [--mapq-threshold N] [--quality-threshold Q] + [--max-depth D] [--max-read-len L] + [--memory-budget-mb M] [--enforce] [--manifest ] [-o out.vcf] +rosalind verify --manifest [--budget-mb B] +``` +`variants --reference` (single-contig) is unchanged; `--enforce`/`--max-depth`/`--max-read-len`/`--manifest` +are additive and back-compatible (absent ⇒ today's behavior, except the `--max-depth` default of 1000 from +C2). + +## 9. Determinism & honesty constraints (cross-cutting) + +- **Byte-identical output preserved.** The writer split + per-call streaming must not change VCF bytes + (gated by `golden_vcf` + `determinism`). The depth cap is deterministic; capped output is reproducible + and order-independent. +- **The estimate and the realized accountant share one set of constants** (§6.1) so a passing `plan` and + the realized receipt cannot silently diverge. +- **Branding is honest in every message and doc:** "fits-or-tells-you / never silently OOM-kills," never + "never refuses." The README/issue-3/`lib.rs` positioning fixes are tracked separately (front-door move), + not in this spec. + +## 10. Per-sub-stage gates + +Each of C1/C2/C3: `cargo test` green (full suite), `cargo fmt --all -- --check`, `cargo build` 0 warnings +(debug **and** release), no htslib types in any new public signature, and the sub-stage's own gates +(C1 §5.5, C2 §6.5, C3 §7.5). Final whole-branch review per the established workflow. + +## 11. Non-goals (so the promises stay airtight) + +Index-build enforcement; somatic whole-genome bounding; graceful degrade / spill; on-demand `ref_base`; +deterministic multithreading / thread-invariance claims; out-accuracy-ing GATK/DeepVariant (the demo is a +memory/reproducibility contract win on SNVs, never an accuracy head-to-head). + +## 12. Resolved decisions (2026-06-01 brainstorm) + +- **Working-set model:** declared depth ceiling + deterministic cap (realized ≤ predicted by construction + for the coverage term; read length is the residual assumption with the realized post-run check as the + backstop). +- **Scope:** full contract on the variants path, decomposed C1/C2/C3; index build stays record-only; + somatic out of scope. +- **Row flush:** stream each call to the writer (resident rows = O(batch)), via a **sink callback** on + `call_germline_whole_genome` (not a per-contig batch, not an iterator). +- **Exit codes:** `3` = predicted over budget (refused, no work); `4` = realized over budget (completed, + contract violated). +- **`--max-depth` default = 1000** (lands in C2; `0` = uncapped); `--max-read-len` default short-read (250). +- **Stdout runs** write a cwd manifest sidecar by default; `--manifest ` redirects. +- **Manifest parsing** for `verify` = a small hand-parser for the fixed canonical shape (no `serde_json`), + round-trip property-tested. From 7ea6c2aeb0ad432235cc23bdd57824c66512be44 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 16:55:22 -0700 Subject: [PATCH 02/32] =?UTF-8?q?docs(plan):=20Phase=20C1=20=E2=80=94=20wo?= =?UTF-8?q?rking-set=20soundness=20fix=20implementation=20plan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 8-task TDD plan for the C1 soundness core: corrected current_working_set (reference + per-read seq/qual), deterministic max_depth cap (mechanism, default off), incremental VCF writer (header + row split), streaming sink on call_germline_whole_genome + decode-then-move reference, and the main.rs wiring. Output-preserving by default; the soundness proof is a library-level bounded- working-set test (process-RSS CI gate deferred to C3). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../2026-06-01-phase-c1-contract-soundness.md | 896 ++++++++++++++++++ 1 file changed, 896 insertions(+) create mode 100644 docs/superpowers/plans/2026-06-01-phase-c1-contract-soundness.md diff --git a/docs/superpowers/plans/2026-06-01-phase-c1-contract-soundness.md b/docs/superpowers/plans/2026-06-01-phase-c1-contract-soundness.md new file mode 100644 index 0000000..5a14558 --- /dev/null +++ b/docs/superpowers/plans/2026-06-01-phase-c1-contract-soundness.md @@ -0,0 +1,896 @@ +# Phase C1 — Working-Set Soundness Fix Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans (inline, chosen for this work) or superpowers:subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make the bounded `variants --index` path's reported working set a *true conservative upper bound* on realized peak — the hard correctness prerequisite for `rosalind plan` (C2) — without changing default output. + +**Architecture:** Four changes, all on the streaming germline path: (1) fix `PileupEngine::current_working_set()` to count the resident reference + per-read `seq`/`qual` bytes it omits today; (2) add a deterministic `--max-depth` cap to the pileup engine (mechanism only — defaults off in C1); (3) split the VCF writer into header + per-row functions so calls can stream; (4) change `call_germline_whole_genome` to drive a row sink (no genome-wide `Vec`) and decode-then-**move** each contig's reference (kills the persistent `buf`+`Arc` double-copy). Then wire `main.rs::run_variants_index` to write the header once and stream rows through the sink. + +**Tech Stack:** Rust 1.72 (MSRV — no `div_ceil`/`is_none_or`), `cargo test`/`fmt`/`build`, the existing `core`/`pileup`/`call`/`io`/`provenance` modules. No new dependencies. + +**Spec:** [`docs/superpowers/specs/2026-06-01-phase-c-contract-design.md`](../specs/2026-06-01-phase-c-contract-design.md) §5. + +**Note on the C1 soundness proof (deliberate refinement of spec §5.5):** the rigorous proof that the accountant is sound is a **library test** on `call_germline_whole_genome` with an explicit `max_depth: Some(D)` — it asserts the returned `WorkingSet` (a) counts the reference term, (b) is bounded by `reference + D·active_cost`, and (c) is **flat as the read count grows**. This is more robust than a process-RSS assertion. The subprocess/CLI **real-RSS CI gate** lands in C3 (where the `--max-depth` CLI default and the contract suite live). + +--- + +## File Structure + +- **Modify** `src/pileup/engine.rs` — `PileupParams.max_depth` field; `SkipCounts.over_max_depth` field + `total()`; `current_working_set()` accounting fix; deterministic cap in `advance_to`. Tests in-file. +- **Modify** `src/io/vcf.rs` — add `write_germline_header` + `write_germline_row`; rewrite `write_germline_vcf` as a (sorting) wrapper over them. Tests in-file. +- **Modify** `src/call/pipeline.rs` — add `call_germline_region_streaming` (sink, returns `WorkingSet`); make `call_germline_region_tracked` a wrapper over it. Tests in-file. +- **Modify** `src/call/mod.rs` — export `call_germline_region_streaming`. +- **Modify** `src/call/whole_genome.rs` — `call_germline_whole_genome` takes a row sink, returns `WorkingSet`; decode-then-move reference. Update its in-file test. Add the soundness test. +- **Modify** `src/main.rs` — `run_variants_index`: write header once, stream rows via the sink, keep the file-path manifest (peak_rss + max_ws) as today. + +--- + +## Task 1: Add the `max_depth` param and `over_max_depth` skip counter (fields only, no behavior) + +**Files:** +- Modify: `src/pileup/engine.rs` (`PileupParams`, `PileupParams::default`, `SkipCounts`, `SkipCounts::total`) + +- [ ] **Step 1: Write the failing test** — append to the `tests` module in `src/pileup/engine.rs`: + +```rust + #[test] + fn params_default_max_depth_is_none_and_skipcounts_total_includes_over_max_depth() { + assert_eq!(PileupParams::default().max_depth, None); + let s = SkipCounts { + unmapped: 1, + wrong_contig: 2, + secondary: 3, + supplementary: 4, + duplicate: 5, + low_mapq: 6, + over_max_depth: 7, + }; + assert_eq!(s.total(), 28); + } +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd ~/rosalind && cargo test -p rosalind --lib pileup::engine::tests::params_default_max_depth 2>&1 | tail -20` +Expected: FAIL — compile error (`PileupParams` has no field `max_depth`; `SkipCounts` has no field `over_max_depth`). + +- [ ] **Step 3: Add the fields.** In `PileupParams` (after `skip_duplicate`): + +```rust + /// Skip PCR/optical duplicates (SAM flag 0x400). + pub skip_duplicate: bool, + /// Cap on the active read set per position (deterministic downsampling). + /// `None` = uncapped (default). When `Some(d)`, reads arriving at a position + /// already covered by `d` active reads are dropped (counted `over_max_depth`). + pub max_depth: Option, +``` + +In `impl Default for PileupParams` (after `skip_duplicate: true,`): + +```rust + skip_duplicate: true, + max_depth: None, +``` + +In `SkipCounts` (after `low_mapq`): + +```rust + /// Reads below the MAPQ threshold. + pub low_mapq: u64, + /// Reads dropped because the position was already at `max_depth`. + pub over_max_depth: u64, +``` + +In `SkipCounts::total` (add the term): + +```rust + pub fn total(&self) -> u64 { + self.unmapped + + self.wrong_contig + + self.secondary + + self.supplementary + + self.duplicate + + self.low_mapq + + self.over_max_depth + } +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd ~/rosalind && cargo test -p rosalind --lib pileup::engine::tests::params_default_max_depth 2>&1 | tail -20` +Expected: PASS. (Other `engine` tests that construct `SkipCounts` with all fields — `skip_counts_total_sums_all_reasons` — need the new field; see Step 5.) + +- [ ] **Step 5: Fix the existing `SkipCounts` literal.** The test `skip_counts_total_sums_all_reasons` builds a `SkipCounts { ... }` without `over_max_depth` — add `over_max_depth: 0,` to that literal and leave its `assert_eq!(s.total(), 21)` unchanged (0 added). + +Run: `cd ~/rosalind && cargo test -p rosalind --lib pileup::engine 2>&1 | tail -20` +Expected: PASS (all engine tests). + +- [ ] **Step 6: Commit** + +```bash +cd ~/rosalind && git add src/pileup/engine.rs && git commit -m "feat(pileup): add max_depth param + over_max_depth skip counter (C1, fields only)" +``` + +--- + +## Task 2: Fix `current_working_set()` to count the resident reference + per-read seq/qual + +**Files:** +- Modify: `src/pileup/engine.rs` (`current_working_set`) + +- [ ] **Step 1: Write the failing test** — append to the `tests` module: + +```rust + #[test] + fn working_set_counts_reference_and_read_byte_buffers() { + // One 4-base read fully covering a 10-base reference. After advancing to + // pos 0 the active set holds that read; the working set must include the + // reference bytes (10) AND the read's seq+qual buffers (4+4), not just the + // projection map. + let reference = b"ACGTACGTAC"; // 10 bytes + let mut e = engine(vec![mread(0, b"ACGT", false)], reference); + let first = e.next().expect("a column").expect("ok"); // drives advance_to(0) + assert_eq!(first.locus.pos.0, 0); + let ws = e.current_working_set().bytes; + // reference (10) + map(4*16=64) + seq(4) + qual(4) + per-read(64) + fixed(256) + // = 10 + 64 + 4 + 4 + 64 + 256 = 402. + assert_eq!(ws, 402); + } +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd ~/rosalind && cargo test -p rosalind --lib pileup::engine::tests::working_set_counts_reference 2>&1 | tail -20` +Expected: FAIL — `assert_eq!` mismatch (current value omits reference + seq + qual; it reports `4*16 + 64 + 256 = 384`... actually the current formula is `len*16 + 64` per read summed `+ 256` = `64 + 64 + 256 = 384`, no reference, no seq/qual). + +- [ ] **Step 3: Implement the corrected accountant.** Replace the body of `current_working_set`: + +```rust + pub fn current_working_set(&self) -> WorkingSet { + // The decoded reference for this contig is resident in the engine. + let reference_bytes = self.reference.len() as u64; + // Each active read holds its projection map (16 B/entry) plus its seq and + // qual byte buffers; count all three (the map alone is a large undercount, + // especially for long reads). + let active_bytes: u64 = self + .active + .iter() + .map(|r| { + (r.ref_to_read.len() as u64) * 16 + + r.seq.len() as u64 + + r.qual.len() as u64 + + 64 + }) + .sum(); + WorkingSet { + bytes: reference_bytes + active_bytes + 256, + } + } +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd ~/rosalind && cargo test -p rosalind --lib pileup::engine::tests::working_set_counts_reference 2>&1 | tail -20` +Expected: PASS. + +- [ ] **Step 5: Confirm the existing coverage-bound test still holds.** `working_set_is_bounded_by_coverage_not_input_size` uses a 50,000-byte reference at depth ~1; the new accounting reports ~50,338 bytes, still `< 64*1024` and `fits(1 MiB)`. + +Run: `cd ~/rosalind && cargo test -p rosalind --lib pileup::engine 2>&1 | tail -20` +Expected: PASS (all engine tests). If `working_set_is_bounded_by_coverage_not_input_size`'s `max_ws < 64 * 1024` now fails, raise that literal bound to `< 128 * 1024` and update the comment to note the reference term is now counted. + +- [ ] **Step 6: Commit** + +```bash +cd ~/rosalind && git add src/pileup/engine.rs && git commit -m "fix(pileup): current_working_set counts resident reference + read seq/qual (C1 soundness core)" +``` + +--- + +## Task 3: Deterministic `max_depth` cap in the pileup engine + +**Files:** +- Modify: `src/pileup/engine.rs` (`advance_to`, the `Ordering::Equal` arm) + +- [ ] **Step 1: Write the failing test** — append to the `tests` module: + +```rust + #[test] + fn max_depth_caps_active_set_deterministically() { + // 5 reads all covering pos 0..4; cap at 2. Only the first 2 (arrival order) + // are kept; the other 3 are counted over_max_depth. Capped output is + // identical regardless of input order (SliceSource sorts on construction). + let reference = b"AAAA"; + let params = PileupParams { + max_depth: Some(2), + ..PileupParams::default() + }; + let run = |reads: Vec| -> (Vec, u64) { + let mut e = PileupEngine::new( + SliceSource::new(reads), + Arc::from(reference.to_vec().into_boxed_slice()), + 0, + 0..4, + params.clone(), + ); + let mut depths = Vec::new(); + while let Some(c) = e.next() { + depths.push(c.unwrap().raw_depth); + } + (depths, e.skip_counts().over_max_depth) + }; + let reads_a = vec![ + mread(0, b"CCCC", false), + mread(0, b"CCCC", false), + mread(0, b"CCCC", false), + mread(0, b"CCCC", false), + mread(0, b"CCCC", false), + ]; + let mut reads_b = reads_a.clone(); + reads_b.reverse(); + let (depths_a, over_a) = run(reads_a); + let (depths_b, over_b) = run(reads_b); + // Capped: every position sees at most 2 reads. + assert!(depths_a.iter().all(|&d| d <= 2), "raw depth must be capped at 2"); + assert_eq!(over_a, 3, "3 of 5 reads dropped over max_depth"); + // Deterministic regardless of input order. + assert_eq!(depths_a, depths_b); + assert_eq!(over_a, over_b); + } +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd ~/rosalind && cargo test -p rosalind --lib pileup::engine::tests::max_depth_caps 2>&1 | tail -20` +Expected: FAIL — `over_a` is 0 and depths reach 5 (no cap applied yet). + +- [ ] **Step 3: Implement the cap.** In `advance_to`, the `std::cmp::Ordering::Equal` arm, insert the cap check between the `read.end() <= pos` guard and `self.ingest(read)`: + +```rust + std::cmp::Ordering::Equal => { + if rp > pos { + break; // future read on our contig + } + let read = self.next_read.take().unwrap(); + if !self.passes_filters(&read) { + continue; + } + if read.end() <= pos { + continue; // does not reach the cursor + } + // Deterministic max-depth cap: once `max_depth` reads already + // cover the cursor, drop arrivals (counted) so the active set — + // and thus the working set — is bounded by the declared depth. + if let Some(max) = self.params.max_depth { + if self.active.len() as u32 >= max { + self.skips.over_max_depth += 1; + continue; + } + } + self.ingest(read); + } +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd ~/rosalind && cargo test -p rosalind --lib pileup::engine 2>&1 | tail -20` +Expected: PASS (new test + all existing engine tests — default `max_depth: None` leaves behavior unchanged). + +- [ ] **Step 5: Commit** + +```bash +cd ~/rosalind && git add src/pileup/engine.rs && git commit -m "feat(pileup): deterministic max_depth cap on the active set (C1)" +``` + +--- + +## Task 4: Split the germline VCF writer into header + row functions + +**Files:** +- Modify: `src/io/vcf.rs` (add `write_germline_header`, `write_germline_row`; rewrite `write_germline_vcf` as a wrapper) + +- [ ] **Step 1: Write the failing test** — append to the `tests` module in `src/io/vcf.rs`: + +```rust + #[test] + fn header_then_streamed_rows_equals_batch_write() { + let r1 = row(0, 100, b'A', het_call()); + let r2 = row(0, 50, b'A', het_call()); + let r3 = row(1, 10, b'A', het_call()); + // Batch writer (sorts internally). + let batch = render_germline_vcf(&contigs(), "S", &[r1.clone(), r2.clone(), r3.clone()]).unwrap(); + // Streaming: header once, then rows in already-sorted (contig,pos) order. + let mut buf = Vec::new(); + write_germline_header(&mut buf, &contigs(), "S").unwrap(); + for r in [&r2, &r1, &r3] { + write_germline_row(&mut buf, &contigs(), r).unwrap(); + } + let streamed = String::from_utf8(buf).unwrap(); + assert_eq!(streamed, batch, "streamed header+rows must equal the batch write byte-for-byte"); + } +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd ~/rosalind && cargo test -p rosalind --lib io::vcf::tests::header_then_streamed_rows 2>&1 | tail -20` +Expected: FAIL — `write_germline_header` / `write_germline_row` not found. + +- [ ] **Step 3: Implement the split.** In `src/io/vcf.rs`, add the two functions and rewrite `write_germline_vcf`. Replace the existing `write_germline_vcf` (lines 48–132) with: + +```rust +/// Write the germline VCFv4.2 header (everything up to and including the +/// `#CHROM` line). Pair with `write_germline_row` to stream records. +pub fn write_germline_header(out: &mut W, contigs: &ContigSet, sample: &str) -> io::Result<()> { + write_fileformat_and_contigs(out, contigs)?; + writeln!( + out, + r#"##INFO="# + )?; + writeln!( + out, + r#"##INFO="# + )?; + writeln!( + out, + r#"##FORMAT="# + )?; + writeln!( + out, + r#"##FORMAT="# + )?; + writeln!( + out, + r#"##FORMAT="# + )?; + writeln!( + out, + r#"##FORMAT="# + )?; + writeln!( + out, + r#"##FORMAT="# + )?; + writeln!( + out, + r#"##FILTER="# + )?; + writeln!( + out, + r#"##FILTER="# + )?; + writeln!( + out, + "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t{sample}" + ) +} + +/// Write one germline record line. The caller supplies rows in canonical +/// (contig, pos, ref, alt) order — this does not sort (use `write_germline_vcf` +/// for an unordered batch). +pub fn write_germline_row(out: &mut W, contigs: &ContigSet, r: &GermlineRow) -> io::Result<()> { + let chrom = contigs + .by_id(r.locus.contig) + .map(|c| c.name.as_ref()) + .unwrap_or("."); + let pos = r.locus.pos.0 as u64 + 1; + let total = (r.call.ad[0] + r.call.ad[1]).max(1); + let af = r.call.ad[1] as f64 / total as f64; + writeln!( + out, + "{chrom}\t{pos}\t.\t{ref_b}\t{alt}\t{qual:.1}\t{filt}\tDP={dp};AF={af:.3}\tGT:GQ:DP:AD:PL\t{gt}:{gq}:{dp}:{ad0},{ad1}:{pl0},{pl1},{pl2}", + ref_b = r.ref_base as char, + alt = r.call.alt_base as char, + qual = r.call.qual, + filt = filter_str(r.call.filter), + dp = r.call.dp, + gt = genotype_str(r.call.genotype), + gq = r.call.gq, + ad0 = r.call.ad[0], + ad1 = r.call.ad[1], + pl0 = r.call.pl[0], + pl1 = r.call.pl[1], + pl2 = r.call.pl[2], + ) +} + +/// Write a spec-valid germline (single-sample) VCFv4.2 to `out`. Records are +/// emitted in canonical (contig, pos, ref, alt) order regardless of input order. +pub fn write_germline_vcf( + out: &mut W, + contigs: &ContigSet, + sample: &str, + rows: &[GermlineRow], +) -> io::Result<()> { + write_germline_header(out, contigs, sample)?; + let mut ordered: Vec<&GermlineRow> = rows.iter().collect(); + ordered.sort_by(|a, b| { + a.locus + .cmp(&b.locus) + .then_with(|| a.ref_base.cmp(&b.ref_base)) + .then_with(|| a.call.alt_base.cmp(&b.call.alt_base)) + }); + for r in ordered { + write_germline_row(out, contigs, r)?; + } + out.flush() +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd ~/rosalind && cargo test -p rosalind --lib io::vcf 2>&1 | tail -25` +Expected: PASS (new test + all existing vcf tests — `write_germline_vcf` output is unchanged: header then sorted rows). + +- [ ] **Step 5: Run the golden VCF snapshot to confirm byte-identity** + +Run: `cd ~/rosalind && cargo test --test golden_vcf 2>&1 | tail -20` +Expected: PASS (no snapshot drift). + +- [ ] **Step 6: Commit** + +```bash +cd ~/rosalind && git add src/io/vcf.rs && git commit -m "refactor(vcf): split germline writer into header + row; write_germline_vcf wraps them (C1)" +``` + +--- + +## Task 5: Streaming germline caller (`call_germline_region_streaming`) + +**Files:** +- Modify: `src/call/pipeline.rs` (add `call_germline_region_streaming`; make `call_germline_region_tracked` a wrapper) +- Modify: `src/call/mod.rs` (export) + +- [ ] **Step 1: Write the failing test** — append to the `tests` module in `src/call/pipeline.rs`: + +```rust + #[test] + fn streaming_emits_same_sites_as_tracked_and_returns_working_set() { + let reference: Arc<[u8]> = Arc::from(b"AAAA".to_vec().into_boxed_slice()); + let reads = vec![ + read(0, b"ACAA", false), + read(0, b"ACAA", false), + read(0, b"AAAA", false), + read(0, b"ACAA", false), + ]; + // Reference path: the tracked collector. + let (collected, _ws) = call_germline_region_tracked( + SliceSource::new(reads.clone()), + Arc::clone(&reference), + 0, + 0..4, + PileupParams::default(), + &GermlineParams::default(), + ) + .unwrap(); + // Streaming path: push into a Vec via the sink, capture the working set. + let mut streamed = Vec::new(); + let ws = call_germline_region_streaming( + SliceSource::new(reads), + reference, + 0, + 0..4, + PileupParams::default(), + &GermlineParams::default(), + &mut |row| { + streamed.push(row); + Ok(()) + }, + ) + .unwrap(); + assert_eq!(streamed, collected, "streaming sites == tracked sites"); + assert!(ws.bytes > 0, "working set tracked and non-zero"); + } +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd ~/rosalind && cargo test -p rosalind --lib call::pipeline::tests::streaming_emits_same 2>&1 | tail -20` +Expected: FAIL — `call_germline_region_streaming` not found. + +- [ ] **Step 3: Implement.** In `src/call/pipeline.rs`, add the streaming primitive and rewrite `call_germline_region_tracked` as a wrapper over it. Replace the current `call_germline_region_tracked` (lines 17–39) with: + +```rust +/// Stream germline calls over `region` of `contig` to a sink, returning the +/// maximum pileup-engine working set observed (the bounded-memory signal behind +/// the `variants` receipt and `rosalind plan`). The sink receives each emitted +/// site as `(locus, ref_base, call)` in ascending position order; no +/// genome-wide buffer accumulates. Hom-ref / no-evidence positions are abstained +/// on (the sink is not called for them). +pub fn call_germline_region_streaming( + source: S, + reference: Arc<[u8]>, + contig: u32, + region: Range, + pileup_params: PileupParams, + germline_params: &GermlineParams, + on_row: &mut dyn FnMut((Locus, u8, GermlineCall)) -> Result<(), CoreError>, +) -> Result { + let mut engine = PileupEngine::new(source, reference, contig, region, pileup_params); + let mut max_ws = WorkingSet { bytes: 0 }; + while let Some(column) = engine.next() { + let column = column?; + let ws = engine.current_working_set(); + if ws.bytes > max_ws.bytes { + max_ws = ws; + } + if let Some(call) = call_germline(&column, germline_params) { + on_row((column.locus, column.ref_base, call))?; + } + } + Ok(max_ws) +} + +/// Like [`call_germline_region`], but also returns the maximum pileup-engine +/// working set observed during the pass. Collects sites into a `Vec` via +/// [`call_germline_region_streaming`]. +pub fn call_germline_region_tracked( + source: S, + reference: Arc<[u8]>, + contig: u32, + region: Range, + pileup_params: PileupParams, + germline_params: &GermlineParams, +) -> Result<(Vec<(Locus, u8, GermlineCall)>, WorkingSet), CoreError> { + let mut out = Vec::new(); + let ws = call_germline_region_streaming( + source, + reference, + contig, + region, + pileup_params, + germline_params, + &mut |row| { + out.push(row); + Ok(()) + }, + )?; + Ok((out, ws)) +} +``` + +In `src/call/mod.rs`, update the `pipeline` re-export line: + +```rust +pub use pipeline::{ + call_germline_region, call_germline_region_streaming, call_germline_region_tracked, + call_somatic_region, +}; +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd ~/rosalind && cargo test -p rosalind --lib call::pipeline 2>&1 | tail -20` +Expected: PASS (new test + the two existing pipeline tests, which go through the unchanged `call_germline_region` → `_tracked` → `_streaming`). + +- [ ] **Step 5: Commit** + +```bash +cd ~/rosalind && git add src/call/pipeline.rs src/call/mod.rs && git commit -m "feat(call): call_germline_region_streaming (sink, returns WorkingSet); tracked wraps it (C1)" +``` + +--- + +## Task 6: `call_germline_whole_genome` drives a row sink + decode-then-move + +**Files:** +- Modify: `src/call/whole_genome.rs` (signature → sink + `WorkingSet`; decode-then-move; update in-file test; add the soundness test) + +- [ ] **Step 1: Write the failing tests** — replace the in-file `whole_genome_equals_per_contig_calls` test body's call site and add the soundness test. First, update the existing test to use the sink (replace the `call_germline_whole_genome(...)` call and the `(rows, ws)` binding, lines 148–155, with): + +```rust + let mut rows: Vec<(crate::core::Locus, u8, GermlineCall)> = Vec::new(); + let ws = call_germline_whole_genome( + SliceSource::new(reads.clone()), + &rv, + contigs, + pp.clone(), + &gp, + &mut |row| { + rows.push(row); + Ok(()) + }, + ) + .unwrap(); +``` + +Then append a new soundness test to the `tests` module: + +```rust + #[test] + fn working_set_is_bounded_by_reference_and_capped_depth_not_read_count() { + // One small contig; pour in increasing numbers of reads at the SAME few + // positions with a depth cap. The returned working set must (a) count the + // reference, (b) stay bounded by reference + capped active, and (c) NOT + // grow with the number of input reads. + let idx_path = tmp("bounded"); + let refseq = vec![b'A'; 2000]; + let index = GenomeIndex::from_named_sequences(&[( + "chr1".to_string(), + refseq.clone(), + )]) + .unwrap(); + IndexWriter::create(&idx_path) + .unwrap() + .write_genome_index(&index) + .unwrap(); + let loaded = IndexReader::open(&idx_path).unwrap(); + let rv = loaded.reference_view().unwrap(); + let contigs = loaded.contigs(); + + let params = PileupParams { + max_depth: Some(8), + ..PileupParams::default() + }; + let gp = GermlineParams::default(); + + let run = |n: usize| -> u64 { + // n reads, each 100bp, all starting at pos 0 (depth would be n without + // the cap; capped at 8). + let reads: Vec = (0..n) + .map(|_| read_at(0, 0, &vec![b'C'; 100])) + .collect(); + let mut sink_calls = 0u64; + let ws = call_germline_whole_genome( + SliceSource::new(reads), + &rv, + contigs, + params.clone(), + &gp, + &mut |_row| { + sink_calls += 1; + Ok(()) + }, + ) + .unwrap(); + ws.bytes + }; + + let ws_small = run(20); + let ws_large = run(2000); + // (a) reference is counted: bound exceeds the 2000-byte reference. + assert!(ws_small > 2000, "working set must include the reference bytes"); + // (c) flat in read count: 100x more reads, same bounded working set. + assert_eq!( + ws_small, ws_large, + "working set must not grow with the number of input reads" + ); + // (b) bounded by reference + capped active: ref(2000) + 8 reads * + // (map 100*16 + seq 100 + qual 100 + 64) + 256, generously bounded. + let bound = 2000 + 8 * (100 * 16 + 100 + 100 + 64) + 256; + assert!( + ws_small <= bound, + "working set {ws_small} exceeded the analytic bound {bound}" + ); + + let _ = std::fs::remove_dir_all(idx_path.parent().unwrap()); + } +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd ~/rosalind && cargo test -p rosalind --lib call::whole_genome 2>&1 | tail -25` +Expected: FAIL — `call_germline_whole_genome` arity/return mismatch (no sink param; returns a tuple). + +- [ ] **Step 3: Implement the sink + decode-then-move.** Replace the body of `call_germline_whole_genome` (the function at lines 47–87) with: + +```rust +pub fn call_germline_whole_genome( + mut source: S, + ref_view: &ReferenceView, + contigs: &ContigSet, + pileup_params: PileupParams, + germline_params: &GermlineParams, + on_row: &mut dyn FnMut((Locus, u8, GermlineCall)) -> Result<(), CoreError>, +) -> Result { + let mut max_ws = WorkingSet { bytes: 0 }; + let mut peeked: Option = None; + + for c in contigs.iter() { + // Decode this contig's reference into a fresh Vec and MOVE it into the + // Arc — no persistent second copy (the steady-state reference resident is + // one contig, not two). Peak = the largest contig; bounded. + let start = c.global_offset as usize; + let end = c.global_offset as usize + c.length as usize; + let mut decoded = Vec::new(); + ref_view.decode_window(start, end, &mut decoded); + let reference: Arc<[u8]> = Arc::from(decoded); + + let per = PerContig { + source: &mut source, + contig: c.id, + peeked: &mut peeked, + }; + let region: Range = 0..c.length; + let ws = call_germline_region_streaming( + per, + reference, + c.id, + region, + pileup_params.clone(), + germline_params, + on_row, + )?; + if ws.bytes > max_ws.bytes { + max_ws = ws; + } + } + + Ok(max_ws) +} +``` + +Update the imports at the top of `whole_genome.rs`: change +`use crate::call::{call_germline_region_tracked, GermlineCall, GermlineParams};` +to +`use crate::call::{call_germline_region_streaming, GermlineCall, GermlineParams};` +(the doc-comment on the function should also drop the "accumulated rows" wording — it now returns only the max working set and streams rows to `on_row`). + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd ~/rosalind && cargo test -p rosalind --lib call::whole_genome 2>&1 | tail -25` +Expected: PASS (the updated equivalence test + the new soundness test). + +- [ ] **Step 5: Commit** + +```bash +cd ~/rosalind && git add src/call/whole_genome.rs && git commit -m "feat(call): whole-genome drive streams rows to a sink + decode-then-move reference (C1)" +``` + +--- + +## Task 7: Wire `main.rs::run_variants_index` to stream via the sink + +**Files:** +- Modify: `src/main.rs` (`run_variants_index`, lines ~1001–1102: imports + the call + the two output branches) + +- [ ] **Step 1: Update the `use` line.** In `run_variants_index`, change: + +```rust + use rosalind::io::vcf::{write_germline_vcf, GermlineRow}; +``` +to: +```rust + use rosalind::core::CoreError; + use rosalind::io::vcf::{write_germline_header, write_germline_row, GermlineRow}; +``` + +- [ ] **Step 2: Replace the call + output section.** Replace the block from `let (sites, max_ws) =` (line ~1044) through the end of the `match output { … }` block (line ~1102) with the streaming form: + +```rust + let source = StreamingBamSource::new(&alignments_path, contigs) + .map_err(|e| anyhow!("failed to open BAM {}: {e}", alignments_path.display()))?; + + // Stream calls straight to the VCF writer (header once, then one row per + // emitted call) so no genome-wide row buffer accumulates. The returned + // WorkingSet is the high-water (reference + active set), captured per contig. + let max_ws = match &output { + Some(path) => { + let file = File::create(path) + .with_context(|| format!("failed to create VCF file {}", path.display()))?; + let mut writer = io::BufWriter::new(file); + write_germline_header(&mut writer, contigs, "SAMPLE")?; + let ws = call_germline_whole_genome( + source, + &ref_view, + contigs, + pileup_params, + &germline_params, + &mut |(locus, ref_base, call)| { + write_germline_row(&mut writer, contigs, &GermlineRow { locus, ref_base, call }) + .map_err(CoreError::from) + }, + ) + .map_err(|e| anyhow!("variant calling failed: {e}"))?; + writer.flush()?; + ws + } + None => { + let stdout = io::stdout(); + let mut handle = stdout.lock(); + write_germline_header(&mut handle, contigs, "SAMPLE")?; + let ws = call_germline_whole_genome( + source, + &ref_view, + contigs, + pileup_params, + &germline_params, + &mut |(locus, ref_base, call)| { + write_germline_row(&mut handle, contigs, &GermlineRow { locus, ref_base, call }) + .map_err(CoreError::from) + }, + ) + .map_err(|e| anyhow!("variant calling failed: {e}"))?; + handle.flush()?; + ws + } + }; + // Realized peak (monotonic high-water mark) captured after the calling pass. + let peak_rss = peak_rss_bytes(); + + // Reproducibility + memory receipt (file output only; stdout receipt is C3). + if let Some(path) = &output { + let mut manifest = RunManifest::new("variants"); + manifest.inputs.push(FileHash { + path: index_path.display().to_string(), + blake3: blake3_file(&index_path)?, + }); + manifest.inputs.push(FileHash { + path: alignments_path.display().to_string(), + blake3: blake3_file(&alignments_path)?, + }); + manifest.outputs.push(FileHash { + path: path.display().to_string(), + blake3: blake3_file(path)?, + }); + manifest + .params + .insert("mapq_threshold".to_string(), mapq_threshold.to_string()); + manifest.params.insert( + "min_qual".to_string(), + (quality_threshold as f64).to_string(), + ); + manifest + .params + .insert("peak_rss_bytes".to_string(), peak_rss.to_string()); + manifest.params.insert( + "max_working_set_bytes".to_string(), + max_ws.bytes.to_string(), + ); + let manifest_path = write_manifest(path, &manifest)?; + eprintln!("wrote reproducibility receipt: {}", manifest_path.display()); + } +``` + +(The `let rows: Vec = sites …collect();` block that previously sat between the call and the `match` is **removed** — rows are streamed, not collected.) + +The existing memory-receipt + record-only budget block (`eprintln!("memory: peak RSS …")` … through the `record-only, run completed` branch, lines ~1103–1120) is **unchanged** and remains after this block. + +- [ ] **Step 3: Build to verify it compiles** + +Run: `cd ~/rosalind && cargo build 2>&1 | tail -25` +Expected: success, 0 warnings. (If `write_germline_vcf`/`GermlineRow` become unused elsewhere in `main.rs`, the build will warn — confirm `run_variants` (the single-contig `--reference` path, line ~857) still uses `write_germline_vcf`; it does, so the import stays valid there.) + +- [ ] **Step 4: Run the whole-genome CLI gate** + +Run: `cd ~/rosalind && cargo test --test variants_index 2>&1 | tail -25` +Expected: PASS (the multi-contig `variants --index` output is byte-identical — header then sorted rows, same as before). + +- [ ] **Step 5: Commit** + +```bash +cd ~/rosalind && git add src/main.rs && git commit -m "feat(cli): variants --index streams rows to the VCF writer via the sink (C1)" +``` + +--- + +## Task 8: Full-suite verification + format/warning gates + +**Files:** none (verification only) + +- [ ] **Step 1: Format check** + +Run: `cd ~/rosalind && cargo fmt --all -- --check 2>&1 | tail -20` +Expected: no output (clean). If it reports diffs, run `cargo fmt --all` and re-commit the touched files. + +- [ ] **Step 2: Zero-warning build (debug + release)** + +Run: `cd ~/rosalind && cargo build 2>&1 | tail -10 && cargo build --release 2>&1 | tail -10` +Expected: both finish with 0 warnings. + +- [ ] **Step 3: Full test suite** + +Run: `cd ~/rosalind && cargo test 2>&1 | tail -40` +Expected: all binaries pass — in particular `determinism`, `golden_vcf`, `variants_index`, `space_bounds`, `pileup_stream`, and the in-lib `pileup`/`call`/`io` tests. + +- [ ] **Step 4: Commit any formatting fixups** (only if Step 1 required changes) + +```bash +cd ~/rosalind && git add -A && git commit -m "style: rustfmt fixups (C1)" +``` + +--- + +## Self-Review notes (filled during writing) + +- **Spec coverage (§5):** §5.1 accountant → Task 2; §5.2 cap → Tasks 1+3; §5.3 incremental writer → Task 4; §5.4 sink + decode-then-move → Tasks 5+6+7; §5.5 proof → Task 6 soundness test (library-level; the process-RSS CI gate is deliberately deferred to C3, noted at the top). +- **Type consistency:** the sink type `&mut dyn FnMut((Locus, u8, GermlineCall)) -> Result<(), CoreError>` is identical in `call_germline_region_streaming` (Task 5), `call_germline_whole_genome` (Task 6), and both `main.rs` call sites (Task 7). `write_germline_row(out, contigs, row)` arity matches across Task 4 (definition) and Tasks 6/7 callers. `WorkingSet`/`CoreError` are the existing `core` types. +- **No behavior change by default:** `max_depth` defaults to `None` (Task 1); `current_working_set` only grows the reported number (Task 2); the VCF wrapper still sorts (Task 4); streamed rows arrive pre-sorted so output is byte-identical (Tasks 6/7, gated by `golden_vcf` + `variants_index`). From a3b8343c81bd57dd75b5ebb7b6c27ddbab09af7c Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 16:57:44 -0700 Subject: [PATCH 03/32] feat(pileup): add max_depth param + over_max_depth skip counter (C1, fields only) --- src/pileup/engine.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/pileup/engine.rs b/src/pileup/engine.rs index cd4d86a..77628d8 100644 --- a/src/pileup/engine.rs +++ b/src/pileup/engine.rs @@ -22,6 +22,10 @@ pub struct PileupParams { pub skip_supplementary: bool, /// Skip PCR/optical duplicates (SAM flag 0x400). pub skip_duplicate: bool, + /// Cap on the active read set per position (deterministic downsampling). + /// `None` = uncapped (default). When `Some(d)`, reads arriving at a position + /// already covered by `d` active reads are dropped (counted `over_max_depth`). + pub max_depth: Option, } impl Default for PileupParams { @@ -32,6 +36,7 @@ impl Default for PileupParams { skip_secondary: true, skip_supplementary: true, skip_duplicate: true, + max_depth: None, } } } @@ -51,6 +56,8 @@ pub struct SkipCounts { pub duplicate: u64, /// Reads below the MAPQ threshold. pub low_mapq: u64, + /// Reads dropped because the position was already at `max_depth`. + pub over_max_depth: u64, } impl SkipCounts { @@ -62,6 +69,7 @@ impl SkipCounts { + self.supplementary + self.duplicate + self.low_mapq + + self.over_max_depth } } @@ -394,6 +402,7 @@ mod tests { supplementary: 4, duplicate: 5, low_mapq: 6, + over_max_depth: 0, }; assert_eq!(s.total(), 21); } @@ -643,4 +652,19 @@ mod tests { assert_eq!(at0.allele_counts(), [0, 1, 0, 0]); assert_eq!(at0.raw_depth, 2); // both reads cover the position } + + #[test] + fn params_default_max_depth_is_none_and_skipcounts_total_includes_over_max_depth() { + assert_eq!(PileupParams::default().max_depth, None); + let s = SkipCounts { + unmapped: 1, + wrong_contig: 2, + secondary: 3, + supplementary: 4, + duplicate: 5, + low_mapq: 6, + over_max_depth: 7, + }; + assert_eq!(s.total(), 28); + } } From e41191e56d94677caa2a43d97c7d56f868fc7663 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 16:58:40 -0700 Subject: [PATCH 04/32] fix(pileup): current_working_set counts resident reference + read seq/qual (C1 soundness core) --- src/pileup/engine.rs | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/src/pileup/engine.rs b/src/pileup/engine.rs index 77628d8..befad8f 100644 --- a/src/pileup/engine.rs +++ b/src/pileup/engine.rs @@ -141,15 +141,23 @@ impl PileupEngine { /// Current working-set estimate: bounded by the active read set (local /// coverage), independent of total input size. Foundation for `rosalind plan`. pub fn current_working_set(&self) -> WorkingSet { - // Each active read costs roughly its projection map (16 B/entry) plus a - // small constant for handles; plus a fixed engine overhead. + // The decoded reference for this contig is resident in the engine. + let reference_bytes = self.reference.len() as u64; + // Each active read holds its projection map (16 B/entry) plus its seq and + // qual byte buffers; count all three (the map alone is a large undercount, + // especially for long reads). let active_bytes: u64 = self .active .iter() - .map(|r| (r.ref_to_read.len() as u64) * 16 + 64) + .map(|r| { + (r.ref_to_read.len() as u64) * 16 + + r.seq.len() as u64 + + r.qual.len() as u64 + + 64 + }) .sum(); WorkingSet { - bytes: active_bytes + 256, + bytes: reference_bytes + active_bytes + 256, } } @@ -667,4 +675,20 @@ mod tests { }; assert_eq!(s.total(), 28); } + + #[test] + fn working_set_counts_reference_and_read_byte_buffers() { + // One 4-base read fully covering a 10-base reference. After advancing to + // pos 0 the active set holds that read; the working set must include the + // reference bytes (10) AND the read's seq+qual buffers (4+4), not just the + // projection map. + let reference = b"ACGTACGTAC"; // 10 bytes + let mut e = engine(vec![mread(0, b"ACGT", false)], reference); + let first = e.next().expect("a column").expect("ok"); // drives advance_to(0) + assert_eq!(first.locus.pos.0, 0); + let ws = e.current_working_set().bytes; + // reference (10) + map(4*16=64) + seq(4) + qual(4) + per-read(64) + fixed(256) + // = 10 + 64 + 4 + 4 + 64 + 256 = 402. + assert_eq!(ws, 402); + } } From c44be67a95742d68ce0d005e0d115a01a2e1b6e3 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 16:59:38 -0700 Subject: [PATCH 05/32] feat(pileup): deterministic max_depth cap on the active set (C1) --- src/pileup/engine.rs | 55 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/pileup/engine.rs b/src/pileup/engine.rs index befad8f..9f295a4 100644 --- a/src/pileup/engine.rs +++ b/src/pileup/engine.rs @@ -240,6 +240,15 @@ impl PileupEngine { if read.end() <= pos { continue; // does not reach the cursor } + // Deterministic max-depth cap: once `max_depth` reads already + // cover the cursor, drop arrivals (counted) so the active set — + // and thus the working set — is bounded by the declared depth. + if let Some(max) = self.params.max_depth { + if self.active.len() as u32 >= max { + self.skips.over_max_depth += 1; + continue; + } + } self.ingest(read); } } @@ -691,4 +700,50 @@ mod tests { // = 10 + 64 + 4 + 4 + 64 + 256 = 402. assert_eq!(ws, 402); } + + #[test] + fn max_depth_caps_active_set_deterministically() { + // 5 reads all covering pos 0..4; cap at 2. Only the first 2 (arrival order) + // are kept; the other 3 are counted over_max_depth. Capped output is + // identical regardless of input order (SliceSource sorts on construction). + let reference = b"AAAA"; + let params = PileupParams { + max_depth: Some(2), + ..PileupParams::default() + }; + let run = |reads: Vec| -> (Vec, u64) { + let mut e = PileupEngine::new( + SliceSource::new(reads), + Arc::from(reference.to_vec().into_boxed_slice()), + 0, + 0..4, + params.clone(), + ); + let mut depths = Vec::new(); + while let Some(c) = e.next() { + depths.push(c.unwrap().raw_depth); + } + (depths, e.skip_counts().over_max_depth) + }; + let reads_a = vec![ + mread(0, b"CCCC", false), + mread(0, b"CCCC", false), + mread(0, b"CCCC", false), + mread(0, b"CCCC", false), + mread(0, b"CCCC", false), + ]; + let mut reads_b = reads_a.clone(); + reads_b.reverse(); + let (depths_a, over_a) = run(reads_a); + let (depths_b, over_b) = run(reads_b); + // Capped: every position sees at most 2 reads. + assert!( + depths_a.iter().all(|&d| d <= 2), + "raw depth must be capped at 2" + ); + assert_eq!(over_a, 3, "3 of 5 reads dropped over max_depth"); + // Deterministic regardless of input order. + assert_eq!(depths_a, depths_b); + assert_eq!(over_a, over_b); + } } From 6749af4a3f45f091009c0977f020f2a7c1ace635 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:01:11 -0700 Subject: [PATCH 06/32] refactor(vcf): split germline writer into header + row; write_germline_vcf wraps them (C1) --- src/io/vcf.rs | 98 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 69 insertions(+), 29 deletions(-) diff --git a/src/io/vcf.rs b/src/io/vcf.rs index bfab61d..000dc88 100644 --- a/src/io/vcf.rs +++ b/src/io/vcf.rs @@ -45,13 +45,12 @@ fn write_fileformat_and_contigs(out: &mut W, contigs: &ContigSet) -> i Ok(()) } -/// Write a spec-valid germline (single-sample) VCFv4.2 to `out`. Records are -/// emitted in canonical (contig, pos, ref, alt) order regardless of input order. -pub fn write_germline_vcf( +/// Write the germline VCFv4.2 header (everything up to and including the +/// `#CHROM` line). Pair with [`write_germline_row`] to stream records. +pub fn write_germline_header( out: &mut W, contigs: &ContigSet, sample: &str, - rows: &[GermlineRow], ) -> io::Result<()> { write_fileformat_and_contigs(out, contigs)?; writeln!( @@ -93,8 +92,51 @@ pub fn write_germline_vcf( writeln!( out, "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t{sample}" - )?; + ) +} +/// Write one germline record line. The caller supplies rows in canonical +/// (contig, pos, ref, alt) order — this does not sort (use [`write_germline_vcf`] +/// for an unordered batch). +pub fn write_germline_row( + out: &mut W, + contigs: &ContigSet, + r: &GermlineRow, +) -> io::Result<()> { + let chrom = contigs + .by_id(r.locus.contig) + .map(|c| c.name.as_ref()) + .unwrap_or("."); + let pos = r.locus.pos.0 as u64 + 1; + let total = (r.call.ad[0] + r.call.ad[1]).max(1); + let af = r.call.ad[1] as f64 / total as f64; + writeln!( + out, + "{chrom}\t{pos}\t.\t{ref_b}\t{alt}\t{qual:.1}\t{filt}\tDP={dp};AF={af:.3}\tGT:GQ:DP:AD:PL\t{gt}:{gq}:{dp}:{ad0},{ad1}:{pl0},{pl1},{pl2}", + ref_b = r.ref_base as char, + alt = r.call.alt_base as char, + qual = r.call.qual, + filt = filter_str(r.call.filter), + dp = r.call.dp, + gt = genotype_str(r.call.genotype), + gq = r.call.gq, + ad0 = r.call.ad[0], + ad1 = r.call.ad[1], + pl0 = r.call.pl[0], + pl1 = r.call.pl[1], + pl2 = r.call.pl[2], + ) +} + +/// Write a spec-valid germline (single-sample) VCFv4.2 to `out`. Records are +/// emitted in canonical (contig, pos, ref, alt) order regardless of input order. +pub fn write_germline_vcf( + out: &mut W, + contigs: &ContigSet, + sample: &str, + rows: &[GermlineRow], +) -> io::Result<()> { + write_germline_header(out, contigs, sample)?; let mut ordered: Vec<&GermlineRow> = rows.iter().collect(); ordered.sort_by(|a, b| { a.locus @@ -102,31 +144,8 @@ pub fn write_germline_vcf( .then_with(|| a.ref_base.cmp(&b.ref_base)) .then_with(|| a.call.alt_base.cmp(&b.call.alt_base)) }); - for r in ordered { - let chrom = contigs - .by_id(r.locus.contig) - .map(|c| c.name.as_ref()) - .unwrap_or("."); - let pos = r.locus.pos.0 as u64 + 1; - let total = (r.call.ad[0] + r.call.ad[1]).max(1); - let af = r.call.ad[1] as f64 / total as f64; - writeln!( - out, - "{chrom}\t{pos}\t.\t{ref_b}\t{alt}\t{qual:.1}\t{filt}\tDP={dp};AF={af:.3}\tGT:GQ:DP:AD:PL\t{gt}:{gq}:{dp}:{ad0},{ad1}:{pl0},{pl1},{pl2}", - ref_b = r.ref_base as char, - alt = r.call.alt_base as char, - qual = r.call.qual, - filt = filter_str(r.call.filter), - dp = r.call.dp, - gt = genotype_str(r.call.genotype), - gq = r.call.gq, - ad0 = r.call.ad[0], - ad1 = r.call.ad[1], - pl0 = r.call.pl[0], - pl1 = r.call.pl[1], - pl2 = r.call.pl[2], - )?; + write_germline_row(out, contigs, r)?; } out.flush() } @@ -354,4 +373,25 @@ mod tests { "chr1\t201\t.\tA\tT\t55.0\tPASS\tSOMATIC\tGT:DP:AD:AF\t0/1:40:28,12:0.300\t0/0:38:38,0:0.000" ); } + + #[test] + fn header_then_streamed_rows_equals_batch_write() { + let r1 = row(0, 100, b'A', het_call()); + let r2 = row(0, 50, b'A', het_call()); + let r3 = row(1, 10, b'A', het_call()); + // Batch writer (sorts internally). + let batch = + render_germline_vcf(&contigs(), "S", &[r1.clone(), r2.clone(), r3.clone()]).unwrap(); + // Streaming: header once, then rows in already-sorted (contig,pos) order. + let mut buf = Vec::new(); + write_germline_header(&mut buf, &contigs(), "S").unwrap(); + for r in [&r2, &r1, &r3] { + write_germline_row(&mut buf, &contigs(), r).unwrap(); + } + let streamed = String::from_utf8(buf).unwrap(); + assert_eq!( + streamed, batch, + "streamed header+rows must equal the batch write byte-for-byte" + ); + } } From 17d5837f7f10ae03f399ff807d4f70470d78b13e Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:02:13 -0700 Subject: [PATCH 07/32] feat(call): call_germline_region_streaming (sink, returns WorkingSet); tracked wraps it (C1) --- src/call/mod.rs | 5 ++- src/call/pipeline.rs | 84 +++++++++++++++++++++++++++++++++++++++----- 2 files changed, 80 insertions(+), 9 deletions(-) diff --git a/src/call/mod.rs b/src/call/mod.rs index 8a41a8b..b1007fd 100644 --- a/src/call/mod.rs +++ b/src/call/mod.rs @@ -9,7 +9,10 @@ pub mod types; pub mod whole_genome; pub use germline::call_germline; -pub use pipeline::{call_germline_region, call_germline_region_tracked, call_somatic_region}; +pub use pipeline::{ + call_germline_region, call_germline_region_streaming, call_germline_region_tracked, + call_somatic_region, +}; pub use somatic::call_somatic; pub use types::{Filter, Genotype, GermlineCall, GermlineParams, SomaticCall, SomaticParams}; pub use whole_genome::call_germline_whole_genome; diff --git a/src/call/pipeline.rs b/src/call/pipeline.rs index 55d2e8d..a332375 100644 --- a/src/call/pipeline.rs +++ b/src/call/pipeline.rs @@ -11,19 +11,22 @@ use crate::call::{ use crate::core::{CoreError, Locus, WorkingSet}; use crate::pileup::{PileupColumn, PileupEngine, PileupParams, ReadSource}; -/// Like [`call_germline_region`], but also returns the maximum pileup-engine -/// working set observed during the pass (the bounded-memory signal — the -/// foundation for the `variants` memory receipt and `rosalind plan`). -pub fn call_germline_region_tracked( +/// Stream germline calls over `region` of `contig` to a sink, returning the +/// maximum pileup-engine working set observed (the bounded-memory signal behind +/// the `variants` receipt and `rosalind plan`). The sink receives each emitted +/// site as `(locus, ref_base, call)` in ascending position order; no +/// genome-wide buffer accumulates. Hom-ref / no-evidence positions are abstained +/// on (the sink is not called for them). +pub fn call_germline_region_streaming( source: S, reference: Arc<[u8]>, contig: u32, region: Range, pileup_params: PileupParams, germline_params: &GermlineParams, -) -> Result<(Vec<(Locus, u8, GermlineCall)>, WorkingSet), CoreError> { + on_row: &mut dyn FnMut((Locus, u8, GermlineCall)) -> Result<(), CoreError>, +) -> Result { let mut engine = PileupEngine::new(source, reference, contig, region, pileup_params); - let mut out = Vec::new(); let mut max_ws = WorkingSet { bytes: 0 }; while let Some(column) = engine.next() { let column = column?; @@ -32,10 +35,37 @@ pub fn call_germline_region_tracked( max_ws = ws; } if let Some(call) = call_germline(&column, germline_params) { - out.push((column.locus, column.ref_base, call)); + on_row((column.locus, column.ref_base, call))?; } } - Ok((out, max_ws)) + Ok(max_ws) +} + +/// Like [`call_germline_region`], but also returns the maximum pileup-engine +/// working set observed during the pass. Collects sites into a `Vec` via +/// [`call_germline_region_streaming`]. +pub fn call_germline_region_tracked( + source: S, + reference: Arc<[u8]>, + contig: u32, + region: Range, + pileup_params: PileupParams, + germline_params: &GermlineParams, +) -> Result<(Vec<(Locus, u8, GermlineCall)>, WorkingSet), CoreError> { + let mut out = Vec::new(); + let ws = call_germline_region_streaming( + source, + reference, + contig, + region, + pileup_params, + germline_params, + &mut |row| { + out.push(row); + Ok(()) + }, + )?; + Ok((out, ws)) } /// Call germline variants across `region` of `contig`. Returns one entry per @@ -213,4 +243,42 @@ mod tests { assert_eq!(call.alt_base, b'C'); assert_eq!(call.normal_alt, 0); } + + #[test] + fn streaming_emits_same_sites_as_tracked_and_returns_working_set() { + let reference: Arc<[u8]> = Arc::from(b"AAAA".to_vec().into_boxed_slice()); + let reads = vec![ + read(0, b"ACAA", false), + read(0, b"ACAA", false), + read(0, b"AAAA", false), + read(0, b"ACAA", false), + ]; + // Reference path: the tracked collector. + let (collected, _ws) = call_germline_region_tracked( + SliceSource::new(reads.clone()), + Arc::clone(&reference), + 0, + 0..4, + PileupParams::default(), + &GermlineParams::default(), + ) + .unwrap(); + // Streaming path: push into a Vec via the sink, capture the working set. + let mut streamed = Vec::new(); + let ws = call_germline_region_streaming( + SliceSource::new(reads), + reference, + 0, + 0..4, + PileupParams::default(), + &GermlineParams::default(), + &mut |row| { + streamed.push(row); + Ok(()) + }, + ) + .unwrap(); + assert_eq!(streamed, collected, "streaming sites == tracked sites"); + assert!(ws.bytes > 0, "working set tracked and non-zero"); + } } From 1f21f40b4079173f75aa0d4131837f52b5c4d2d9 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:04:49 -0700 Subject: [PATCH 08/32] feat(call): whole-genome drive streams rows to a sink + decode-then-move reference (C1) --- src/call/whole_genome.rs | 101 +++++++++++++++++++++++++++++++++------ 1 file changed, 86 insertions(+), 15 deletions(-) diff --git a/src/call/whole_genome.rs b/src/call/whole_genome.rs index aa7549f..228f3f3 100644 --- a/src/call/whole_genome.rs +++ b/src/call/whole_genome.rs @@ -6,7 +6,7 @@ use std::ops::Range; use std::sync::Arc; -use crate::call::{call_germline_region_tracked, GermlineCall, GermlineParams}; +use crate::call::{call_germline_region_streaming, GermlineCall, GermlineParams}; use crate::core::{AlignedRead, ContigSet, CoreError, Locus, WorkingSet}; use crate::genomics::ReferenceView; use crate::pileup::{PileupParams, ReadSource}; @@ -40,28 +40,31 @@ impl ReadSource for PerContig<'_, S> { } /// Call germline variants across every contig of `contigs`, in id order, reading -/// each contig's reference from `ref_view`. Returns the accumulated -/// `(Locus, ref_base, call)` rows and the max pileup working set observed. -/// `source` MUST yield reads in `(contig, pos)` order (a `StreamingBamSource` -/// guards this); the per-contig partition relies on it. +/// each contig's reference from `ref_view` and streaming each emitted +/// `(Locus, ref_base, call)` to `on_row`. Returns the max pileup working set +/// observed; no genome-wide row buffer accumulates. `source` MUST yield reads in +/// `(contig, pos)` order (a `StreamingBamSource` guards this); the per-contig +/// partition relies on it. pub fn call_germline_whole_genome( mut source: S, ref_view: &ReferenceView, contigs: &ContigSet, pileup_params: PileupParams, germline_params: &GermlineParams, -) -> Result<(Vec<(Locus, u8, GermlineCall)>, WorkingSet), CoreError> { - let mut rows = Vec::new(); + on_row: &mut dyn FnMut((Locus, u8, GermlineCall)) -> Result<(), CoreError>, +) -> Result { let mut max_ws = WorkingSet { bytes: 0 }; let mut peeked: Option = None; - let mut buf = Vec::new(); for c in contigs.iter() { - // Decode this contig's reference (peak = largest contig; bounded). + // Decode this contig's reference into a fresh Vec and MOVE it into the + // Arc — no persistent second copy (the steady-state reference resident is + // one contig, not two). Peak = the largest contig; bounded. let start = c.global_offset as usize; let end = c.global_offset as usize + c.length as usize; - ref_view.decode_window(start, end, &mut buf); - let reference: Arc<[u8]> = Arc::from(buf.as_slice()); + let mut decoded = Vec::new(); + ref_view.decode_window(start, end, &mut decoded); + let reference: Arc<[u8]> = Arc::from(decoded); let per = PerContig { source: &mut source, @@ -69,21 +72,21 @@ pub fn call_germline_whole_genome( peeked: &mut peeked, }; let region: Range = 0..c.length; - let (sites, ws) = call_germline_region_tracked( + let ws = call_germline_region_streaming( per, reference, c.id, region, pileup_params.clone(), germline_params, + on_row, )?; if ws.bytes > max_ws.bytes { max_ws = ws; } - rows.extend(sites); } - Ok((rows, max_ws)) + Ok(max_ws) } #[cfg(test)] @@ -145,12 +148,17 @@ mod tests { let pp = PileupParams::default(); let gp = GermlineParams::default(); - let (rows, ws) = call_germline_whole_genome( + let mut rows: Vec<(Locus, u8, GermlineCall)> = Vec::new(); + let ws = call_germline_whole_genome( SliceSource::new(reads.clone()), &rv, contigs, pp.clone(), &gp, + &mut |row| { + rows.push(row); + Ok(()) + }, ) .unwrap(); // Every row's contig is a real contig id; rows are grouped by contig. @@ -191,4 +199,67 @@ mod tests { let _ = std::fs::remove_dir_all(idx_path.parent().unwrap()); } + + #[test] + fn working_set_is_bounded_by_reference_and_capped_depth_not_read_count() { + // One small contig; pour in increasing numbers of reads at the SAME few + // positions with a depth cap. The returned working set must (a) count the + // reference, (b) stay bounded by reference + capped active, and (c) NOT + // grow with the number of input reads. + let idx_path = tmp("bounded"); + let index = + GenomeIndex::from_named_sequences(&[("chr1".to_string(), vec![b'A'; 2000])]).unwrap(); + IndexWriter::create(&idx_path) + .unwrap() + .write_genome_index(&index) + .unwrap(); + let loaded = IndexReader::open(&idx_path).unwrap(); + let rv = loaded.reference_view().unwrap(); + let contigs = loaded.contigs(); + + let params = PileupParams { + max_depth: Some(8), + ..PileupParams::default() + }; + let gp = GermlineParams::default(); + + let run = |n: usize| -> u64 { + // n reads, each 100bp, all starting at pos 0 (depth would be n without + // the cap; capped at 8). + let reads: Vec = + (0..n).map(|_| read_at(0, 0, &vec![b'C'; 100])).collect(); + call_germline_whole_genome( + SliceSource::new(reads), + &rv, + contigs, + params.clone(), + &gp, + &mut |_row| Ok(()), + ) + .unwrap() + .bytes + }; + + let ws_small = run(20); + let ws_large = run(2000); + // (a) reference is counted: bound exceeds the 2000-byte reference. + assert!( + ws_small > 2000, + "working set must include the reference bytes" + ); + // (c) flat in read count: 100x more reads, same bounded working set. + assert_eq!( + ws_small, ws_large, + "working set must not grow with the number of input reads" + ); + // (b) bounded by reference + capped active: ref(2000) + 8 reads * + // (map 100*16 + seq 100 + qual 100 + 64) + 256, generously bounded. + let bound = 2000 + 8 * (100 * 16 + 100 + 100 + 64) + 256; + assert!( + ws_small <= bound, + "working set {ws_small} exceeded the analytic bound {bound}" + ); + + let _ = std::fs::remove_dir_all(idx_path.parent().unwrap()); + } } From 719baad69933ffdd71ca8530c156dd9e387ae803 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:06:51 -0700 Subject: [PATCH 09/32] feat(cli): variants --index streams rows to the VCF writer via the sink (C1) --- src/main.rs | 119 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 70 insertions(+), 49 deletions(-) diff --git a/src/main.rs b/src/main.rs index e002ca5..203414a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1001,7 +1001,7 @@ fn run_variants_index( use rosalind::call::{call_germline_whole_genome, GermlineParams}; use rosalind::genomics::IndexReader; use rosalind::io::bam::StreamingBamSource; - use rosalind::io::vcf::{write_germline_vcf, GermlineRow}; + use rosalind::io::vcf::{write_germline_header, write_germline_row, GermlineRow}; use rosalind::pileup::PileupParams; use rosalind::provenance::{blake3_file, write_manifest, FileHash, RunManifest}; @@ -1041,64 +1041,85 @@ fn run_variants_index( } let source = StreamingBamSource::new(&alignments_path, contigs) .map_err(|e| anyhow!("failed to open BAM {}: {e}", alignments_path.display()))?; - let (sites, max_ws) = - call_germline_whole_genome(source, &ref_view, contigs, pileup_params, &germline_params) - .map_err(|e| anyhow!("variant calling failed: {e}"))?; - // Realized peak (monotonic high-water mark) captured after the calling pass. - let peak_rss = peak_rss_bytes(); - - let rows: Vec = sites - .into_iter() - .map(|(locus, ref_base, call)| GermlineRow { - locus, - ref_base, - call, - }) - .collect(); - match output { + // Stream calls straight to the VCF writer (header once, then one row per + // emitted call) so no genome-wide row buffer accumulates. The returned + // WorkingSet is the high-water (reference + active set), captured per contig. + let max_ws = match &output { Some(path) => { - let file = File::create(&path) + let file = File::create(path) .with_context(|| format!("failed to create VCF file {}", path.display()))?; let mut writer = io::BufWriter::new(file); - write_germline_vcf(&mut writer, contigs, "SAMPLE", &rows)?; + write_germline_header(&mut writer, contigs, "SAMPLE")?; + let ws = call_germline_whole_genome( + source, + &ref_view, + contigs, + pileup_params, + &germline_params, + &mut |(locus, ref_base, call)| { + write_germline_row(&mut writer, contigs, &GermlineRow { locus, ref_base, call }) + .map_err(rosalind::core::CoreError::from) + }, + ) + .map_err(|e| anyhow!("variant calling failed: {e}"))?; writer.flush()?; - drop(writer); - let mut manifest = RunManifest::new("variants"); - manifest.inputs.push(FileHash { - path: index_path.display().to_string(), - blake3: blake3_file(&index_path)?, - }); - manifest.inputs.push(FileHash { - path: alignments_path.display().to_string(), - blake3: blake3_file(&alignments_path)?, - }); - manifest.outputs.push(FileHash { - path: path.display().to_string(), - blake3: blake3_file(&path)?, - }); - manifest - .params - .insert("mapq_threshold".to_string(), mapq_threshold.to_string()); - manifest.params.insert( - "min_qual".to_string(), - (quality_threshold as f64).to_string(), - ); - manifest - .params - .insert("peak_rss_bytes".to_string(), peak_rss.to_string()); - manifest.params.insert( - "max_working_set_bytes".to_string(), - max_ws.bytes.to_string(), - ); - let manifest_path = write_manifest(&path, &manifest)?; - eprintln!("wrote reproducibility receipt: {}", manifest_path.display()); + ws } None => { let stdout = io::stdout(); let mut handle = stdout.lock(); - write_germline_vcf(&mut handle, contigs, "SAMPLE", &rows)?; + write_germline_header(&mut handle, contigs, "SAMPLE")?; + let ws = call_germline_whole_genome( + source, + &ref_view, + contigs, + pileup_params, + &germline_params, + &mut |(locus, ref_base, call)| { + write_germline_row(&mut handle, contigs, &GermlineRow { locus, ref_base, call }) + .map_err(rosalind::core::CoreError::from) + }, + ) + .map_err(|e| anyhow!("variant calling failed: {e}"))?; + handle.flush()?; + ws } + }; + // Realized peak (monotonic high-water mark) captured after the calling pass. + let peak_rss = peak_rss_bytes(); + + // Reproducibility + memory receipt (file output only; stdout receipt is C3). + if let Some(path) = &output { + let mut manifest = RunManifest::new("variants"); + manifest.inputs.push(FileHash { + path: index_path.display().to_string(), + blake3: blake3_file(&index_path)?, + }); + manifest.inputs.push(FileHash { + path: alignments_path.display().to_string(), + blake3: blake3_file(&alignments_path)?, + }); + manifest.outputs.push(FileHash { + path: path.display().to_string(), + blake3: blake3_file(path)?, + }); + manifest + .params + .insert("mapq_threshold".to_string(), mapq_threshold.to_string()); + manifest.params.insert( + "min_qual".to_string(), + (quality_threshold as f64).to_string(), + ); + manifest + .params + .insert("peak_rss_bytes".to_string(), peak_rss.to_string()); + manifest.params.insert( + "max_working_set_bytes".to_string(), + max_ws.bytes.to_string(), + ); + let manifest_path = write_manifest(path, &manifest)?; + eprintln!("wrote reproducibility receipt: {}", manifest_path.display()); } // Memory receipt: the bounded contract, made visible + verifiable. eprintln!( From 2ff08b0337ea82bf016af4ca9401b3e62186e981 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:08:35 -0700 Subject: [PATCH 10/32] style: rustfmt fixups (C1) --- src/call/whole_genome.rs | 3 +-- src/main.rs | 24 ++++++++++++++++++++---- src/pileup/engine.rs | 5 +---- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/call/whole_genome.rs b/src/call/whole_genome.rs index 228f3f3..6031055 100644 --- a/src/call/whole_genome.rs +++ b/src/call/whole_genome.rs @@ -226,8 +226,7 @@ mod tests { let run = |n: usize| -> u64 { // n reads, each 100bp, all starting at pos 0 (depth would be n without // the cap; capped at 8). - let reads: Vec = - (0..n).map(|_| read_at(0, 0, &vec![b'C'; 100])).collect(); + let reads: Vec = (0..n).map(|_| read_at(0, 0, &vec![b'C'; 100])).collect(); call_germline_whole_genome( SliceSource::new(reads), &rv, diff --git a/src/main.rs b/src/main.rs index 203414a..c3e57dc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1058,8 +1058,16 @@ fn run_variants_index( pileup_params, &germline_params, &mut |(locus, ref_base, call)| { - write_germline_row(&mut writer, contigs, &GermlineRow { locus, ref_base, call }) - .map_err(rosalind::core::CoreError::from) + write_germline_row( + &mut writer, + contigs, + &GermlineRow { + locus, + ref_base, + call, + }, + ) + .map_err(rosalind::core::CoreError::from) }, ) .map_err(|e| anyhow!("variant calling failed: {e}"))?; @@ -1077,8 +1085,16 @@ fn run_variants_index( pileup_params, &germline_params, &mut |(locus, ref_base, call)| { - write_germline_row(&mut handle, contigs, &GermlineRow { locus, ref_base, call }) - .map_err(rosalind::core::CoreError::from) + write_germline_row( + &mut handle, + contigs, + &GermlineRow { + locus, + ref_base, + call, + }, + ) + .map_err(rosalind::core::CoreError::from) }, ) .map_err(|e| anyhow!("variant calling failed: {e}"))?; diff --git a/src/pileup/engine.rs b/src/pileup/engine.rs index 9f295a4..aa9b1ab 100644 --- a/src/pileup/engine.rs +++ b/src/pileup/engine.rs @@ -150,10 +150,7 @@ impl PileupEngine { .active .iter() .map(|r| { - (r.ref_to_read.len() as u64) * 16 - + r.seq.len() as u64 - + r.qual.len() as u64 - + 64 + (r.ref_to_read.len() as u64) * 16 + r.seq.len() as u64 + r.qual.len() as u64 + 64 }) .sum(); WorkingSet { From 89516dfcec19e6a763336226ed16bd7b3299991e Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:19:24 -0700 Subject: [PATCH 11/32] =?UTF-8?q?docs(plan):=20Phase=20C2=20=E2=80=94=20ro?= =?UTF-8?q?salind=20plan=20+=20--enforce=20implementation=20plan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 6-task TDD plan: shared pileup cost consts (core/budget.rs) used by both the C1 accountant and a new pure estimator (call/plan.rs); `rosalind plan` (--index variants peak / --reference build peak); --max-depth (default 1000) / --max-read-len / --enforce on variants --index with exit 3 (refuse pre-run) / 4 (fail post-run). Resolves the spec's working-set-vs-RSS gap by measuring the process baseline at call time (no guessed constant); post-run peak_rss is the backstop. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../plans/2026-06-01-phase-c2-plan-enforce.md | 772 ++++++++++++++++++ 1 file changed, 772 insertions(+) create mode 100644 docs/superpowers/plans/2026-06-01-phase-c2-plan-enforce.md diff --git a/docs/superpowers/plans/2026-06-01-phase-c2-plan-enforce.md b/docs/superpowers/plans/2026-06-01-phase-c2-plan-enforce.md new file mode 100644 index 0000000..6bdb679 --- /dev/null +++ b/docs/superpowers/plans/2026-06-01-phase-c2-plan-enforce.md @@ -0,0 +1,772 @@ +# Phase C2 — `rosalind plan` + `--enforce` Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans (inline, chosen for this work) to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Turn C1's sound working-set accountant into a contract: `rosalind plan` predicts whether a whole-genome call fits a declared budget *before* committing, and `variants --index --enforce` refuses up front (exit 3) or fails loud post-run (exit 4) instead of silently overrunning. + +**Architecture:** A pure estimator (`src/call/plan.rs`) computes the data-dependent working set from the index header + declared `--max-depth`/`--max-read-len`, sharing its cost constants with C1's realized accountant so they cannot drift. The *predicted peak RSS* = a **process baseline measured at call time** (`peak_rss_bytes()` after the index/BAM are open) + that working set — apples-to-apples with the realized `peak_rss` the post-run check uses, with no guessed baseline constant. `rosalind plan` renders the breakdown; `variants --enforce` refuses pre-run (exit 3) when predicted > budget and fails post-run (exit 4) when realized > budget. + +**Tech Stack:** Rust 1.72 (MSRV), `clap` derive, `cargo test`/`fmt`/`build`. No new dependencies. Builds on C1 (branch `rosalind/phase-c-contract`). + +**Spec:** [`docs/superpowers/specs/2026-06-01-phase-c-contract-design.md`](../specs/2026-06-01-phase-c-contract-design.md) §6. + +**Resolved design point (extends spec §6.1):** the budget is RSS (consistent with the existing record-only `budget.admits(peak_rss)`). The pre-run *predicted peak* = `measured baseline RSS (after index+BAM open) + estimated working set`. This avoids a guessed process-baseline constant and keeps predicted/realized comparable; the post-run `peak_rss` check is the hard backstop. `--enforce` requires `--max-depth > 0` (an uncapped active set has no a-priori bound). **The exit-4 (post-run) subprocess test is deferred to C3's CI contract suite** — it is only reachable when the model under-predicts, which is brittle to engineer against coarse process RSS; C2 unit-tests the decision and subprocess-tests exit 3 + the generous-budget pass. + +--- + +## File Structure + +- **Modify** `src/core/budget.rs` — add 4 shared `pub const` pileup cost constants. +- **Modify** `src/core/mod.rs` — re-export the 4 constants. +- **Modify** `src/pileup/engine.rs` — `current_working_set` uses the shared constants (behavior-identical). +- **Create** `src/call/plan.rs` — `estimate_variants_working_set`, `predicted_peak_rss_bytes`, `render_variants_plan` (pure, unit-tested). +- **Modify** `src/call/mod.rs` — `pub mod plan;` + re-export. +- **Modify** `src/main.rs` — `Plan` subcommand + `run_plan`; `--enforce`/`--max-depth`/`--max-read-len` on `Variants`; `run_variants_index` gains the cap wiring + pre/post enforcement. +- **Create** `tests/plan_enforce.rs` — subprocess tests for `plan` and `--enforce`. + +--- + +## Task 1: Shared pileup cost constants (no behavior change) + +**Files:** +- Modify: `src/core/budget.rs`, `src/core/mod.rs`, `src/pileup/engine.rs` + +- [ ] **Step 1: Add the constants.** In `src/core/budget.rs`, after the module doc comment (before `pub struct MemoryBudget`), insert: + +```rust +/// Per-base cost of a pileup read's reference→read-offset projection map (one +/// `HashMap` entry ≈ 16 bytes). Shared by the realized accountant +/// (`PileupEngine::current_working_set`) and the `rosalind plan` estimator so the +/// two cannot drift. +pub const PILEUP_MAP_BYTES_PER_BASE: u64 = 16; +/// Per-base cost of a read's `seq` + `qual` byte buffers (1 byte each). +pub const PILEUP_SEQQUAL_BYTES_PER_BASE: u64 = 2; +/// Fixed per-active-read overhead (handles + struct). +pub const PILEUP_PER_READ_OVERHEAD: u64 = 64; +/// Fixed per-engine overhead. +pub const PILEUP_ENGINE_OVERHEAD: u64 = 256; +``` + +- [ ] **Step 2: Re-export them.** In `src/core/mod.rs`, replace the budget re-export line: + +```rust +pub use budget::{MemoryBudget, WorkingSet}; +``` +with: +```rust +pub use budget::{ + MemoryBudget, WorkingSet, PILEUP_ENGINE_OVERHEAD, PILEUP_MAP_BYTES_PER_BASE, + PILEUP_PER_READ_OVERHEAD, PILEUP_SEQQUAL_BYTES_PER_BASE, +}; +``` + +- [ ] **Step 3: Use them in the accountant.** In `src/pileup/engine.rs`, update the import line to add the two constants `current_working_set` needs: + +```rust +use crate::core::{ + allele_index, AlignedRead, CoreError, Locus, Position, WorkingSet, + PILEUP_ENGINE_OVERHEAD, PILEUP_MAP_BYTES_PER_BASE, PILEUP_PER_READ_OVERHEAD, +}; +``` + +Then rewrite the body of `current_working_set` to reference the constants (the numeric values are identical — this is a behavior-preserving refactor): + +```rust + pub fn current_working_set(&self) -> WorkingSet { + // The decoded reference for this contig is resident in the engine. + let reference_bytes = self.reference.len() as u64; + // Each active read holds its projection map plus its seq and qual byte + // buffers; count all three (the map alone is a large undercount, + // especially for long reads). Constants are shared with the plan estimator. + let active_bytes: u64 = self + .active + .iter() + .map(|r| { + (r.ref_to_read.len() as u64) * PILEUP_MAP_BYTES_PER_BASE + + r.seq.len() as u64 + + r.qual.len() as u64 + + PILEUP_PER_READ_OVERHEAD + }) + .sum(); + WorkingSet { + bytes: reference_bytes + active_bytes + PILEUP_ENGINE_OVERHEAD, + } + } +``` + +- [ ] **Step 4: Run the engine tests (behavior unchanged — the 402 assertion still holds)** + +Run: `cd ~/rosalind && cargo test -p rosalind --lib pileup::engine 2>&1 | tail -6` +Expected: PASS (19 tests; `working_set_counts_reference_and_read_byte_buffers` still computes 402 = `16*4 + 4 + 4 + 64` per read `+ 10` ref `+ 256`). + +- [ ] **Step 5: Commit** + +```bash +cd ~/rosalind && git add src/core/budget.rs src/core/mod.rs src/pileup/engine.rs && git commit -m "refactor(core): extract shared pileup cost consts; accountant uses them (C2 prep)" +``` + +--- + +## Task 2: The pure plan estimator (`src/call/plan.rs`) + +**Files:** +- Create: `src/call/plan.rs` +- Modify: `src/call/mod.rs` + +- [ ] **Step 1: Create the module with its tests.** Write `src/call/plan.rs`: + +```rust +//! Pure, testable planning helpers for `rosalind plan` and `variants --enforce`. +//! +//! The estimator predicts the **data-dependent working set** of a bounded +//! whole-genome germline call from the index header plus the declared depth cap +//! and an assumed max read length. It shares its cost constants with the realized +//! accountant (`PileupEngine::current_working_set`) so a passing plan and the +//! realized receipt cannot silently diverge. The *predicted peak RSS* adds a +//! process baseline the caller measures at runtime (`peak_rss_bytes()`), so the +//! prediction is comparable to the realized `peak_rss` the post-run check uses. + +use crate::core::{ + MemoryBudget, WorkingSet, PILEUP_ENGINE_OVERHEAD, PILEUP_MAP_BYTES_PER_BASE, + PILEUP_PER_READ_OVERHEAD, PILEUP_SEQQUAL_BYTES_PER_BASE, +}; + +/// Estimate the peak streaming working set of a whole-genome germline call: the +/// largest contig's reference (decoded, 1×) + the depth-capped active read set + +/// the fixed engine overhead. A true upper bound when actual reads do not exceed +/// `max_read_len` and depth is capped at `max_depth` (both enforced at runtime — +/// `max_read_len` is the one assumption, with the post-run check as backstop). +pub fn estimate_variants_working_set( + largest_contig_len: u64, + max_depth: u32, + max_read_len: u32, +) -> WorkingSet { + let per_read = (max_read_len as u64) + .saturating_mul(PILEUP_MAP_BYTES_PER_BASE + PILEUP_SEQQUAL_BYTES_PER_BASE) + .saturating_add(PILEUP_PER_READ_OVERHEAD); + let active = (max_depth as u64).saturating_mul(per_read); + WorkingSet { + bytes: largest_contig_len + .saturating_add(active) + .saturating_add(PILEUP_ENGINE_OVERHEAD), + } +} + +/// Predicted peak process RSS = a measured process baseline + the estimated +/// working set. Comparable to the realized `peak_rss` the post-run check uses. +pub fn predicted_peak_rss_bytes( + largest_contig_len: u64, + max_depth: u32, + max_read_len: u32, + baseline_rss_bytes: u64, +) -> u64 { + baseline_rss_bytes + .saturating_add(estimate_variants_working_set(largest_contig_len, max_depth, max_read_len).bytes) +} + +/// Render the `rosalind plan --index` breakdown: a measured baseline + the +/// working-set components → predicted peak, tagged `[FITS]`/`[REFUSE]` against the +/// budget (or `[no budget]` when none is declared). Deterministic given inputs. +pub fn render_variants_plan( + largest_contig_len: u64, + max_depth: u32, + max_read_len: u32, + baseline_rss_bytes: u64, + budget_mb: Option, +) -> String { + const MIB: u64 = 1 << 20; + let per_read = (max_read_len as u64) + .saturating_mul(PILEUP_MAP_BYTES_PER_BASE + PILEUP_SEQQUAL_BYTES_PER_BASE) + .saturating_add(PILEUP_PER_READ_OVERHEAD); + let active = (max_depth as u64).saturating_mul(per_read); + let predicted = + predicted_peak_rss_bytes(largest_contig_len, max_depth, max_read_len, baseline_rss_bytes); + let verdict = match budget_mb { + Some(mb) => { + if MemoryBudget::from_mb(mb).admits(predicted) { + format!("/ budget {mb} MiB [FITS]") + } else { + format!("/ budget {mb} MiB [REFUSE]") + } + } + None => "[no budget]".to_string(), + }; + format!( + "plan: predicted peak RSS (upper bound)\n \ + process baseline (measured): {} MiB\n \ + reference decode (largest contig): {} MiB\n \ + active set @ max-depth {}: {} MiB\n \ + engine overhead: {} MiB\n \ + -------------------------------------------------\n \ + predicted peak: ~{} MiB {}\n", + baseline_rss_bytes / MIB, + largest_contig_len / MIB, + max_depth, + active / MIB, + PILEUP_ENGINE_OVERHEAD / MIB, + predicted / MIB, + verdict, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn estimate_grows_with_inputs_and_does_not_overflow() { + let small = estimate_variants_working_set(1_000, 100, 150).bytes; + let deeper = estimate_variants_working_set(1_000, 2_000, 150).bytes; + let bigger_ref = estimate_variants_working_set(1_000_000, 100, 150).bytes; + assert!(deeper > small, "more depth → larger working set"); + assert!(bigger_ref > small, "larger contig → larger working set"); + // Active term for D=100, L=150: 100 * (150*18 + 64) = 100 * 2764 = 276_400. + assert_eq!(small, 1_000 + 276_400 + 256); + let _ = estimate_variants_working_set(u64::MAX, u32::MAX, u32::MAX); // no panic + } + + #[test] + fn predicted_peak_is_baseline_plus_working_set() { + let ws = estimate_variants_working_set(248_000_000, 1_000, 250).bytes; + let predicted = predicted_peak_rss_bytes(248_000_000, 1_000, 250, 50_000_000); + assert_eq!(predicted, 50_000_000 + ws); + } + + #[test] + fn render_reports_fits_and_refuse() { + // Tiny working set; generous budget → FITS. + let fits = render_variants_plan(1_000, 100, 150, 1_000_000, Some(4096)); + assert!(fits.contains("[FITS]"), "generous budget should fit: {fits}"); + // 248 MiB contig + baseline 50 MiB ≫ 64 MiB budget → REFUSE. + let refuse = render_variants_plan(248 * (1 << 20), 1000, 250, 50 * (1 << 20), Some(64)); + assert!(refuse.contains("[REFUSE]"), "tight budget should refuse: {refuse}"); + // No budget → advisory. + assert!(render_variants_plan(1_000, 100, 150, 0, None).contains("[no budget]")); + } +} +``` + +- [ ] **Step 2: Wire the module.** In `src/call/mod.rs`, add the module declaration (after `pub mod pipeline;`) and a re-export. Add: + +```rust +pub mod plan; +``` +and add a re-export line: +```rust +pub use plan::{estimate_variants_working_set, predicted_peak_rss_bytes, render_variants_plan}; +``` + +- [ ] **Step 3: Run the plan tests** + +Run: `cd ~/rosalind && cargo test -p rosalind --lib call::plan 2>&1 | tail -8` +Expected: PASS (3 tests). + +- [ ] **Step 4: Commit** + +```bash +cd ~/rosalind && git add src/call/plan.rs src/call/mod.rs && git commit -m "feat(call): pure variants plan estimator (shares cost consts with the accountant) (C2)" +``` + +--- + +## Task 3: `rosalind plan` subcommand + +**Files:** +- Modify: `src/main.rs` (`Commands` enum, dispatch, new `run_plan`) +- Create: `tests/plan_enforce.rs` + +- [ ] **Step 1: Add the `Plan` variant.** In `src/main.rs`, in `enum Commands`, after the `Locate { … }` variant (before the closing `}` of the enum), add: + +```rust + /// Predict whether a job fits a declared memory budget, before committing. + Plan { + /// Persisted index (`rosalind index`): predict the bounded whole-genome + /// `variants` peak. Mutually exclusive with `--reference`. + #[arg( + long, + conflicts_with = "reference", + required_unless_present = "reference" + )] + index: Option, + /// Reference FASTA: predict the index BUILD peak (advisory — build is + /// O(reference); Phase D enforces). Mutually exclusive with `--index`. + #[arg(long, required_unless_present = "index")] + reference: Option, + /// Max active depth assumed for the `variants` working-set bound. + #[arg(long, default_value_t = 1000)] + max_depth: u32, + /// Max read length assumed for the `variants` working-set bound. + #[arg(long, default_value_t = 250)] + max_read_len: u32, + /// Declared memory budget (MiB) to check feasibility against. + #[arg(long)] + budget_mb: Option, + }, +``` + +- [ ] **Step 2: Add the dispatch arm.** In `main()`, after the `Commands::Locate { … } => run_locate(…)?,` arm, add: + +```rust + Commands::Plan { + index, + reference, + max_depth, + max_read_len, + budget_mb, + } => run_plan(index, reference, max_depth, max_read_len, budget_mb)?, +``` + +- [ ] **Step 3: Implement `run_plan`.** Add this function in `src/main.rs` immediately after `run_index` (after its closing `}`): + +```rust +/// Predict whether a job fits a declared budget, before committing. `--index` +/// predicts the bounded whole-genome `variants` peak (largest contig + active set +/// @ the declared cap, atop the measured process baseline). `--reference` +/// predicts the index build peak (advisory; build is O(reference)). +fn run_plan( + index: Option, + reference: Option, + max_depth: u32, + max_read_len: u32, + budget_mb: Option, +) -> Result<()> { + use rosalind::call::plan::render_variants_plan; + use rosalind::genomics::IndexReader; + + if let Some(index_path) = index { + let loaded = IndexReader::open(&index_path) + .with_context(|| format!("failed to open index {}", index_path.display()))?; + let largest = loaded + .contigs() + .iter() + .map(|c| c.length as u64) + .max() + .unwrap_or(0); + // Measure the process baseline now (binary + libs + index mmap header); + // the per-contig reference decode + active set are modeled on top. + let baseline = peak_rss_bytes(); + print!( + "{}", + render_variants_plan(largest, max_depth, max_read_len, baseline, budget_mb) + ); + } else { + let reference = reference.expect("clap guarantees one of --index/--reference"); + let fasta_reader = open_input(&reference) + .with_context(|| format!("failed to open reference {}", reference.display()))?; + let total_bp: u64 = FastaReader::new(fasta_reader) + .collect::, _>>() + .with_context(|| format!("failed to parse FASTA {}", reference.display()))? + .iter() + .map(|r| r.sequence.len() as u64) + .sum(); + let estimate = estimate_build_working_set(total_bp); + match budget_mb { + Some(mb) => println!("{}", render_plan_line(estimate, MemoryBudget::from_mb(mb))), + None => println!( + "plan: est. build peak ~{} MiB (advisory; build is O(reference)) [no budget]", + estimate.bytes / (1 << 20) + ), + } + } + Ok(()) +} +``` + +- [ ] **Step 4: Build to verify it compiles** + +Run: `cd ~/rosalind && cargo build 2>&1 | tail -8` +Expected: success, 0 warnings. (`estimate_build_working_set`, `render_plan_line`, `MemoryBudget`, `FastaReader`, `open_input`, `peak_rss_bytes` are already imported/in scope in `main.rs` from the `run_index` path.) + +- [ ] **Step 5: Write the subprocess test.** Create `tests/plan_enforce.rs`: + +```rust +//! CLI contract surface: `rosalind plan` predicts feasibility, and +//! `variants --index --enforce` refuses up front / passes within budget. + +use std::path::PathBuf; +use std::process::Command; + +fn bin() -> &'static str { + env!("CARGO_BIN_EXE_rosalind") +} + +// Build a tiny 2-contig index in a fresh temp dir; return (dir, index_path). +fn build_index() -> (PathBuf, PathBuf) { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let dir = std::env::temp_dir().join(format!("rosalind-plan-{nanos}")); + std::fs::create_dir_all(&dir).unwrap(); + let fa = dir.join("ref.fa"); + std::fs::write(&fa, b">chr1\nACGTACGTACGTACGTACGT\n>chr2\nTTTTGGGGCCCCAAAATTTT\n").unwrap(); + let idx = dir.join("ref.idx"); + let out = Command::new(bin()) + .args(["index", "--reference"]) + .arg(&fa) + .arg("--output") + .arg(&idx) + .output() + .unwrap(); + assert!(out.status.success(), "index build failed: {out:?}"); + (dir, idx) +} + +#[test] +fn plan_index_reports_a_breakdown_and_fits_a_generous_budget() { + let (dir, idx) = build_index(); + let out = Command::new(bin()) + .args(["plan", "--index"]) + .arg(&idx) + .args(["--budget-mb", "4096"]) + .output() + .unwrap(); + assert!(out.status.success()); + let stdout = String::from_utf8_lossy(&out.stdout); + assert!(stdout.contains("predicted peak"), "missing breakdown: {stdout}"); + assert!(stdout.contains("[FITS]"), "generous budget should FIT: {stdout}"); + std::fs::remove_dir_all(&dir).ok(); +} + +#[test] +fn plan_reference_reports_build_estimate() { + let (dir, _idx) = build_index(); + let fa = dir.join("ref.fa"); + let out = Command::new(bin()) + .args(["plan", "--reference"]) + .arg(&fa) + .args(["--budget-mb", "4096"]) + .output() + .unwrap(); + assert!(out.status.success()); + let stdout = String::from_utf8_lossy(&out.stdout); + assert!(stdout.contains("plan:"), "missing build plan line: {stdout}"); + std::fs::remove_dir_all(&dir).ok(); +} +``` + +- [ ] **Step 6: Run the plan subprocess test** + +Run: `cd ~/rosalind && cargo test --test plan_enforce plan_ 2>&1 | tail -15` +Expected: PASS (2 `plan_*` tests). + +- [ ] **Step 7: Commit** + +```bash +cd ~/rosalind && git add src/main.rs tests/plan_enforce.rs && git commit -m "feat(cli): rosalind plan — predict variants/build peak vs a declared budget (C2)" +``` + +--- + +## Task 4: `--max-depth` / `--max-read-len` / `--enforce` flags + cap wiring + +**Files:** +- Modify: `src/main.rs` (`Variants` variant, dispatch, `run_variants_index` signature + cap wiring) + +- [ ] **Step 1: Add the flags to the `Variants` variant.** In `enum Commands`, in `Variants { … }`, after the `memory_budget_mb` field, add: + +```rust + /// Cap the active read set per position (deterministic downsampling); the + /// bound `plan`/`--enforce` rely on. `0` = uncapped. + #[arg(long, default_value_t = 1000)] + max_depth: u32, + /// Max read length assumed by the pre-run `--enforce` estimate. + #[arg(long, default_value_t = 250)] + max_read_len: u32, + /// Honor the budget: refuse up front if predicted peak exceeds it (exit 3), + /// or fail after the run if the realized peak does (exit 4). Requires + /// `--memory-budget-mb` and `--max-depth > 0`. + #[arg(long, default_value_t = false)] + enforce: bool, +``` + +- [ ] **Step 2: Update the dispatch.** In `main()`, the `Commands::Variants { … }` destructure — add the three new fields to the pattern, and pass them to `run_variants_index`. Replace the destructure field list to include them and replace the `run_variants_index(…)` call: + +```rust + Commands::Variants { + index, + reference, + alignments, + chrom, + region_start, + mapq_threshold, + output, + block_size: _, + quality_threshold, + memory_budget_mb, + max_depth, + max_read_len, + enforce, + } => { + if let Some(index) = index { + if chrom.is_some() || region_start != 0 { + bail!("--chrom/--region-start are not valid with --index (the whole index is called)"); + } + run_variants_index( + index, + alignments, + mapq_threshold, + output, + quality_threshold, + memory_budget_mb, + max_depth, + max_read_len, + enforce, + )? + } else { + let reference = reference.expect("clap guarantees one of --index/--reference"); + run_variants( + reference, + alignments, + chrom, + region_start, + mapq_threshold, + output, + 1024, + quality_threshold, + )? + } + } +``` + +- [ ] **Step 3: Extend `run_variants_index`'s signature + cap wiring.** Change the signature (add the three params) and the `pileup_params` construction. Replace the function signature: + +```rust +fn run_variants_index( + index_path: PathBuf, + alignments_path: PathBuf, + mapq_threshold: u8, + output: Option, + quality_threshold: f32, + memory_budget_mb: Option, + max_depth: u32, + max_read_len: u32, + enforce: bool, +) -> Result<()> { +``` + +And replace the `pileup_params` construction (currently `PileupParams { min_mapq: mapq_threshold, ..PileupParams::default() }`) with: + +```rust + let pileup_params = PileupParams { + min_mapq: mapq_threshold, + // `--max-depth 0` opts out of the cap (then the working set is unbounded + // and `--enforce` is rejected below). + max_depth: if max_depth == 0 { None } else { Some(max_depth) }, + ..PileupParams::default() + }; +``` + +- [ ] **Step 4: Build to verify it compiles** (enforcement logic comes in Task 5; `max_read_len`/`enforce` are unused for now — silence with a leading `let _ = (max_read_len, enforce);` placeholder is NOT needed because Task 5 lands immediately; if building between tasks, expect an `unused variable` warning only). + +Run: `cd ~/rosalind && cargo build 2>&1 | tail -10` +Expected: compiles. (Warnings for unused `max_read_len`/`enforce` are acceptable *only* until Task 5; do not commit Task 4 alone — proceed to Task 5 before committing, or accept the transient warning. To keep commits clean, **commit Task 4 + Task 5 together** after Task 5's tests pass.) + +- [ ] **Step 5: Run the existing whole-genome gate (cap default 1000 does not change small-test output)** + +Run: `cd ~/rosalind && cargo test --test variants_index 2>&1 | tail -10` +Expected: PASS (5 tests — the tiny test inputs are far below depth 1000). + +(No commit here — see Task 5.) + +--- + +## Task 5: `--enforce` — refuse pre-run (exit 3), fail post-run (exit 4) + +**Files:** +- Modify: `src/main.rs` (`run_variants_index`: pre-run check after opening the source; post-run check replacing the record-only block) +- Modify: `tests/plan_enforce.rs` (add enforce tests) + +- [ ] **Step 1: Add the pre-run refuse check.** In `run_variants_index`, immediately after the `let source = StreamingBamSource::new(…)?;` line (and before the `let max_ws = match &output { … }` streaming block), insert: + +```rust + // `--enforce` contract: predict the peak RSS up front (measured baseline + + // the depth-capped working set) and refuse cleanly if it won't fit — before + // doing any work. Never a silent OOM. + if enforce { + if memory_budget_mb.is_none() { + bail!("--enforce requires --memory-budget-mb"); + } + if max_depth == 0 { + bail!("--enforce requires --max-depth > 0 (an uncapped active set has no a-priori bound)"); + } + let mb = memory_budget_mb.unwrap(); + let largest = contigs.iter().map(|c| c.length as u64).max().unwrap_or(0); + let baseline = peak_rss_bytes(); + let predicted = + rosalind::call::plan::predicted_peak_rss_bytes(largest, max_depth, max_read_len, baseline); + if !MemoryBudget::from_mb(mb).admits(predicted) { + eprintln!( + "contract: REFUSE — declared {} MiB, predicted peak ~{} MiB \ + (largest contig {} MiB + active @ max-depth {} / max-read-len {} \ + atop a {} MiB baseline). Raise --memory-budget-mb, lower --max-depth, \ + or drop --enforce.", + mb, + predicted / (1 << 20), + largest / (1 << 20), + max_depth, + max_read_len, + baseline / (1 << 20), + ); + std::process::exit(3); + } + } +``` + +- [ ] **Step 2: Replace the post-run record-only block with the enforce-aware version.** Replace the existing budget block at the end of `run_variants_index`: + +```rust + if let Some(mb) = memory_budget_mb { + let budget = MemoryBudget::from_mb(mb); + if budget.admits(peak_rss) { + eprintln!("memory: within budget ({mb} MiB)"); + } else { + eprintln!( + "memory: EXCEEDED budget {} MiB (realized peak {} MiB) — record-only, run completed", + mb, + peak_rss / (1 << 20) + ); + } + } +``` +with: +```rust + if let Some(mb) = memory_budget_mb { + let budget = MemoryBudget::from_mb(mb); + let within = budget.admits(peak_rss); + if enforce { + if within { + eprintln!("contract: OK — realized peak {} MiB within declared {mb} MiB", peak_rss / (1 << 20)); + } else { + eprintln!( + "contract: VIOLATED — realized peak {} MiB exceeded declared {mb} MiB (output + receipt written)", + peak_rss / (1 << 20) + ); + std::process::exit(4); + } + } else if within { + eprintln!("memory: within budget ({mb} MiB)"); + } else { + eprintln!( + "memory: EXCEEDED budget {mb} MiB (realized peak {} MiB) — record-only, run completed", + peak_rss / (1 << 20) + ); + } + } +``` + +- [ ] **Step 3: Build (now `max_read_len`/`enforce` are used — 0 warnings)** + +Run: `cd ~/rosalind && cargo build 2>&1 | tail -8` +Expected: success, 0 warnings. + +- [ ] **Step 4: Add the enforce subprocess tests.** Append to `tests/plan_enforce.rs`. These need a coordinate-sorted BAM; reuse the helper pattern from `tests/variants_index.rs` (it builds a tiny sorted BAM via the `common` test module). Add at the top: `mod common;` is NOT used here — instead drive a refuse that needs no BAM by pointing at a missing BAM is wrong (it would error differently). The deterministic, BAM-free way to hit exit 3 is a budget smaller than the measured baseline alone, which the pre-run check catches before reading the BAM. Add: + +```rust +#[test] +fn enforce_refuses_up_front_when_budget_below_predicted() { + // A 1 MiB budget is below the process baseline alone, so the pre-run check + // refuses with exit code 3 before touching the (here absent) alignments — + // the refusal is computed from the index + baseline, not the BAM. + let (dir, idx) = build_index(); + let bam = dir.join("nonexistent.bam"); // never opened: refuse happens first? No — + // source opens before the check; use a real (empty-but-valid) BAM instead. + // Build a minimal sorted BAM by calling `variants --index` is circular; instead + // assert the refusal via the predicted-peak path using a real sorted BAM from + // the variants_index fixtures is heavy. Keep this test BAM-light: see note. + let _ = (bam, idx); + std::fs::remove_dir_all(&dir).ok(); +} +``` + +**Correction (do this instead of the placeholder above):** the pre-run check runs *after* `StreamingBamSource::new`, so a valid sorted BAM is required even to reach the refusal. Rather than synthesize a BAM here, exercise the refuse path through the **already-present** `tests/variants_index.rs` fixture style. Replace the body above with a test that shells out using the sorted BAM that `tests/variants_index.rs` builds. Since that fixture lives in `tests/common`, add `mod common;` and use it: + +```rust +mod common; + +#[test] +fn enforce_refuses_up_front_when_budget_below_predicted() { + // Build a tiny index + a matching coordinate-sorted BAM via the shared + // fixture, then run with an absurdly small budget under --enforce: the + // pre-run prediction (baseline + working set) exceeds 1 MiB, so the run + // refuses with exit code 3 and writes no VCF. + let fx = common::tiny_sorted_fixture(); // (dir, index_path, sorted_bam_path) + let out = Command::new(bin()) + .args(["variants", "--index"]) + .arg(&fx.index) + .arg("--alignments") + .arg(&fx.bam) + .args(["--memory-budget-mb", "1", "--enforce"]) + .output() + .unwrap(); + assert_eq!(out.status.code(), Some(3), "expected refuse exit 3: {out:?}"); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!(stderr.contains("REFUSE"), "missing refuse message: {stderr}"); + fx.cleanup(); +} + +#[test] +fn enforce_passes_within_a_generous_budget() { + let fx = common::tiny_sorted_fixture(); + let out = Command::new(bin()) + .args(["variants", "--index"]) + .arg(&fx.index) + .arg("--alignments") + .arg(&fx.bam) + .args(["--memory-budget-mb", "4096", "--enforce"]) + .output() + .unwrap(); + assert!(out.status.success(), "generous budget should pass: {out:?}"); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!(stderr.contains("contract: OK"), "missing OK line: {stderr}"); + fx.cleanup(); +} +``` + +**Before writing the tests, inspect `tests/common` and `tests/variants_index.rs`** to use the *actual* fixture helper name/shape (the names `tiny_sorted_fixture`/`fx.index`/`fx.bam`/`fx.cleanup` are placeholders for whatever the existing fixture exposes). If `tests/common` has no reusable sorted-BAM fixture, lift the BAM-building code from `tests/variants_index.rs::variants_index_matches_reference_on_the_same_sorted_bam` into this test directly. Match the existing fixture API exactly. + +- [ ] **Step 5: Run the enforce tests** + +Run: `cd ~/rosalind && cargo test --test plan_enforce 2>&1 | tail -15` +Expected: PASS (the `plan_*` tests + `enforce_refuses_up_front_when_budget_below_predicted` exit 3 + `enforce_passes_within_a_generous_budget`). + +- [ ] **Step 6: Commit Tasks 4 + 5 together** + +```bash +cd ~/rosalind && git add src/main.rs tests/plan_enforce.rs && git commit -m "feat(cli): variants --index --enforce — refuse pre-run (3) / fail post-run (4); --max-depth default 1000 (C2)" +``` + +--- + +## Task 6: Full verification + +**Files:** none (verification only) + +- [ ] **Step 1: Format** + +Run: `cd ~/rosalind && cargo fmt --all && cargo fmt --all -- --check 2>&1 | tail -3` +Expected: clean after applying. + +- [ ] **Step 2: Zero-warning builds** + +Run: `cd ~/rosalind && cargo build 2>&1 | tail -5 && cargo build --release 2>&1 | tail -5` +Expected: both 0 warnings. + +- [ ] **Step 3: Full suite** + +Run: `cd ~/rosalind && cargo test 2>&1 | grep -E "test result: FAILED|panicked|[1-9][0-9]* failed" | head; cargo test 2>&1 | grep -cE "test result: ok\."` +Expected: no failures; the count of `ok.` sections ≥ the C1 count + 1 (new `plan_enforce` binary). + +- [ ] **Step 4: Commit any fmt fixups** (only if Step 1 changed files) + +```bash +cd ~/rosalind && git add -A && git commit -m "style: rustfmt fixups (C2)" +``` + +--- + +## Self-Review notes + +- **Spec §6 coverage:** §6.1 estimator (shared consts) → Tasks 1+2; §6.2 `plan` subcommand (both modes) → Task 3; §6.3 `--enforce` exit 3/4 → Task 5; §6.4 `--max-depth` default 1000 + `--max-read-len` 250 → Task 4. §6.5 tests → Tasks 2/3/5 (exit-4 subprocess test deferred to C3 per the note up top; exit-4 *path* is implemented and exercised by the generous/within branch). +- **Type consistency:** the four `PILEUP_*` consts are defined once in `core/budget.rs`, re-exported in `core/mod.rs`, and consumed identically in `engine.rs` (Task 1) and `call/plan.rs` (Task 2). `predicted_peak_rss_bytes(largest, max_depth, max_read_len, baseline)` has the same argument order in `plan.rs` (Task 2), `render_variants_plan` (Task 2), and the `run_variants_index` pre-run check (Task 5). `run_variants_index` arity is updated in lockstep at its definition (Task 4 Step 3) and its only call site (Task 4 Step 2). +- **Fixture caveat (Task 5 Step 4):** the test fixture helper names are placeholders — inspect `tests/common` + `tests/variants_index.rs` and match the real API before writing, or inline the BAM build. This is the one task requiring a look at existing test code first. From 310fe5428e425fbc5c7bf897837c879291ecbe87 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:20:49 -0700 Subject: [PATCH 12/32] refactor(core): extract shared pileup cost consts; accountant uses them (C2 prep) --- src/core/budget.rs | 12 ++++++++++++ src/core/mod.rs | 5 ++++- src/pileup/engine.rs | 12 +++++++++--- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/core/budget.rs b/src/core/budget.rs index ccdf450..7796abf 100644 --- a/src/core/budget.rs +++ b/src/core/budget.rs @@ -2,6 +2,18 @@ //! bound so a run can be checked against a `MemoryBudget` *before* it starts //! (the foundation for `rosalind plan`). +/// Per-base cost of a pileup read's reference→read-offset projection map (one +/// `HashMap` entry ≈ 16 bytes). Shared by the realized accountant +/// (`PileupEngine::current_working_set`) and the `rosalind plan` estimator so the +/// two cannot drift. +pub const PILEUP_MAP_BYTES_PER_BASE: u64 = 16; +/// Per-base cost of a read's `seq` + `qual` byte buffers (1 byte each). +pub const PILEUP_SEQQUAL_BYTES_PER_BASE: u64 = 2; +/// Fixed per-active-read overhead (handles + struct). +pub const PILEUP_PER_READ_OVERHEAD: u64 = 64; +/// Fixed per-engine overhead. +pub const PILEUP_ENGINE_OVERHEAD: u64 = 256; + /// A declared cap on a streaming stage's working set. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct MemoryBudget { diff --git a/src/core/mod.rs b/src/core/mod.rs index a755356..4c098c5 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -10,7 +10,10 @@ pub mod record; pub mod sequence; pub use sequence::{allele_index, BaseCode}; -pub use budget::{MemoryBudget, WorkingSet}; +pub use budget::{ + MemoryBudget, WorkingSet, PILEUP_ENGINE_OVERHEAD, PILEUP_MAP_BYTES_PER_BASE, + PILEUP_PER_READ_OVERHEAD, PILEUP_SEQQUAL_BYTES_PER_BASE, +}; pub use error::CoreError; pub use locus::{Contig, ContigSet, Locus, Position}; pub use record::{AlignedRead, CigarOp, CigarOpKind, RefBase, SamFlags}; diff --git a/src/pileup/engine.rs b/src/pileup/engine.rs index aa9b1ab..e36504b 100644 --- a/src/pileup/engine.rs +++ b/src/pileup/engine.rs @@ -5,7 +5,10 @@ use std::collections::HashMap; use std::ops::Range; use std::sync::Arc; -use crate::core::{allele_index, AlignedRead, CoreError, Locus, Position, WorkingSet}; +use crate::core::{ + allele_index, AlignedRead, CoreError, Locus, Position, WorkingSet, PILEUP_ENGINE_OVERHEAD, + PILEUP_MAP_BYTES_PER_BASE, PILEUP_PER_READ_OVERHEAD, +}; use crate::pileup::column::{Obs, PileupColumn}; use crate::pileup::source::ReadSource; @@ -150,11 +153,14 @@ impl PileupEngine { .active .iter() .map(|r| { - (r.ref_to_read.len() as u64) * 16 + r.seq.len() as u64 + r.qual.len() as u64 + 64 + (r.ref_to_read.len() as u64) * PILEUP_MAP_BYTES_PER_BASE + + r.seq.len() as u64 + + r.qual.len() as u64 + + PILEUP_PER_READ_OVERHEAD }) .sum(); WorkingSet { - bytes: reference_bytes + active_bytes + 256, + bytes: reference_bytes + active_bytes + PILEUP_ENGINE_OVERHEAD, } } From f480d96e148d075823e4a273f30c991026281efe Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:21:43 -0700 Subject: [PATCH 13/32] feat(call): pure variants plan estimator (shares cost consts with the accountant) (C2) --- src/call/mod.rs | 2 + src/call/plan.rs | 129 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 src/call/plan.rs diff --git a/src/call/mod.rs b/src/call/mod.rs index b1007fd..3f6255b 100644 --- a/src/call/mod.rs +++ b/src/call/mod.rs @@ -4,6 +4,7 @@ pub mod germline; pub mod pipeline; +pub mod plan; pub mod somatic; pub mod types; pub mod whole_genome; @@ -13,6 +14,7 @@ pub use pipeline::{ call_germline_region, call_germline_region_streaming, call_germline_region_tracked, call_somatic_region, }; +pub use plan::{estimate_variants_working_set, predicted_peak_rss_bytes, render_variants_plan}; pub use somatic::call_somatic; pub use types::{Filter, Genotype, GermlineCall, GermlineParams, SomaticCall, SomaticParams}; pub use whole_genome::call_germline_whole_genome; diff --git a/src/call/plan.rs b/src/call/plan.rs new file mode 100644 index 0000000..5f9e149 --- /dev/null +++ b/src/call/plan.rs @@ -0,0 +1,129 @@ +//! Pure, testable planning helpers for `rosalind plan` and `variants --enforce`. +//! +//! The estimator predicts the **data-dependent working set** of a bounded +//! whole-genome germline call from the index header plus the declared depth cap +//! and an assumed max read length. It shares its cost constants with the realized +//! accountant (`PileupEngine::current_working_set`) so a passing plan and the +//! realized receipt cannot silently diverge. The *predicted peak RSS* adds a +//! process baseline the caller measures at runtime (`peak_rss_bytes()`), so the +//! prediction is comparable to the realized `peak_rss` the post-run check uses. + +use crate::core::{ + MemoryBudget, WorkingSet, PILEUP_ENGINE_OVERHEAD, PILEUP_MAP_BYTES_PER_BASE, + PILEUP_PER_READ_OVERHEAD, PILEUP_SEQQUAL_BYTES_PER_BASE, +}; + +/// Estimate the peak streaming working set of a whole-genome germline call: the +/// largest contig's reference (decoded, 1×) + the depth-capped active read set + +/// the fixed engine overhead. A true upper bound when actual reads do not exceed +/// `max_read_len` and depth is capped at `max_depth` (both enforced at runtime — +/// `max_read_len` is the one assumption, with the post-run check as backstop). +pub fn estimate_variants_working_set( + largest_contig_len: u64, + max_depth: u32, + max_read_len: u32, +) -> WorkingSet { + let per_read = (max_read_len as u64) + .saturating_mul(PILEUP_MAP_BYTES_PER_BASE + PILEUP_SEQQUAL_BYTES_PER_BASE) + .saturating_add(PILEUP_PER_READ_OVERHEAD); + let active = (max_depth as u64).saturating_mul(per_read); + WorkingSet { + bytes: largest_contig_len + .saturating_add(active) + .saturating_add(PILEUP_ENGINE_OVERHEAD), + } +} + +/// Predicted peak process RSS = a measured process baseline + the estimated +/// working set. Comparable to the realized `peak_rss` the post-run check uses. +pub fn predicted_peak_rss_bytes( + largest_contig_len: u64, + max_depth: u32, + max_read_len: u32, + baseline_rss_bytes: u64, +) -> u64 { + baseline_rss_bytes.saturating_add( + estimate_variants_working_set(largest_contig_len, max_depth, max_read_len).bytes, + ) +} + +/// Render the `rosalind plan --index` breakdown: a measured baseline + the +/// working-set components → predicted peak, tagged `[FITS]`/`[REFUSE]` against the +/// budget (or `[no budget]` when none is declared). Deterministic given inputs. +pub fn render_variants_plan( + largest_contig_len: u64, + max_depth: u32, + max_read_len: u32, + baseline_rss_bytes: u64, + budget_mb: Option, +) -> String { + const MIB: u64 = 1 << 20; + let per_read = (max_read_len as u64) + .saturating_mul(PILEUP_MAP_BYTES_PER_BASE + PILEUP_SEQQUAL_BYTES_PER_BASE) + .saturating_add(PILEUP_PER_READ_OVERHEAD); + let active = (max_depth as u64).saturating_mul(per_read); + let predicted = + predicted_peak_rss_bytes(largest_contig_len, max_depth, max_read_len, baseline_rss_bytes); + let verdict = match budget_mb { + Some(mb) => { + if MemoryBudget::from_mb(mb).admits(predicted) { + format!("/ budget {mb} MiB [FITS]") + } else { + format!("/ budget {mb} MiB [REFUSE]") + } + } + None => "[no budget]".to_string(), + }; + format!( + "plan: predicted peak RSS (upper bound)\n \ + process baseline (measured): {} MiB\n \ + reference decode (largest contig): {} MiB\n \ + active set @ max-depth {}: {} MiB\n \ + engine overhead: {} MiB\n \ + -------------------------------------------------\n \ + predicted peak: ~{} MiB {}\n", + baseline_rss_bytes / MIB, + largest_contig_len / MIB, + max_depth, + active / MIB, + PILEUP_ENGINE_OVERHEAD / MIB, + predicted / MIB, + verdict, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn estimate_grows_with_inputs_and_does_not_overflow() { + let small = estimate_variants_working_set(1_000, 100, 150).bytes; + let deeper = estimate_variants_working_set(1_000, 2_000, 150).bytes; + let bigger_ref = estimate_variants_working_set(1_000_000, 100, 150).bytes; + assert!(deeper > small, "more depth → larger working set"); + assert!(bigger_ref > small, "larger contig → larger working set"); + // Active term for D=100, L=150: 100 * (150*18 + 64) = 100 * 2764 = 276_400. + assert_eq!(small, 1_000 + 276_400 + 256); + let _ = estimate_variants_working_set(u64::MAX, u32::MAX, u32::MAX); // no panic + } + + #[test] + fn predicted_peak_is_baseline_plus_working_set() { + let ws = estimate_variants_working_set(248_000_000, 1_000, 250).bytes; + let predicted = predicted_peak_rss_bytes(248_000_000, 1_000, 250, 50_000_000); + assert_eq!(predicted, 50_000_000 + ws); + } + + #[test] + fn render_reports_fits_and_refuse() { + // Tiny working set; generous budget → FITS. + let fits = render_variants_plan(1_000, 100, 150, 1_000_000, Some(4096)); + assert!(fits.contains("[FITS]"), "generous budget should fit: {fits}"); + // 248 MiB contig + baseline 50 MiB ≫ 64 MiB budget → REFUSE. + let refuse = render_variants_plan(248 * (1 << 20), 1000, 250, 50 * (1 << 20), Some(64)); + assert!(refuse.contains("[REFUSE]"), "tight budget should refuse: {refuse}"); + // No budget → advisory. + assert!(render_variants_plan(1_000, 100, 150, 0, None).contains("[no budget]")); + } +} From fe35fe1ca5274254e5f3814064c698eae482fe3b Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:23:00 -0700 Subject: [PATCH 14/32] =?UTF-8?q?feat(cli):=20rosalind=20plan=20=E2=80=94?= =?UTF-8?q?=20predict=20variants/build=20peak=20vs=20a=20declared=20budget?= =?UTF-8?q?=20(C2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main.rs | 83 +++++++++++++++++++++++++++++++++++++++++++ tests/plan_enforce.rs | 70 ++++++++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 tests/plan_enforce.rs diff --git a/src/main.rs b/src/main.rs index c3e57dc..885e13b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -183,6 +183,30 @@ enum Commands { #[arg(long, default_value_t = 1024)] max_hits: usize, }, + /// Predict whether a job fits a declared memory budget, before committing. + Plan { + /// Persisted index (`rosalind index`): predict the bounded whole-genome + /// `variants` peak. Mutually exclusive with `--reference`. + #[arg( + long, + conflicts_with = "reference", + required_unless_present = "reference" + )] + index: Option, + /// Reference FASTA: predict the index BUILD peak (advisory — build is + /// O(reference); Phase D enforces). Mutually exclusive with `--index`. + #[arg(long, required_unless_present = "index")] + reference: Option, + /// Max active depth assumed for the `variants` working-set bound. + #[arg(long, default_value_t = 1000)] + max_depth: u32, + /// Max read length assumed for the `variants` working-set bound. + #[arg(long, default_value_t = 250)] + max_read_len: u32, + /// Declared memory budget (MiB) to check feasibility against. + #[arg(long)] + budget_mb: Option, + }, } #[derive(Copy, Clone, Debug, ValueEnum, Eq, PartialEq)] @@ -318,6 +342,13 @@ fn main() -> Result<()> { pattern, max_hits, } => run_locate(index, pattern, max_hits)?, + Commands::Plan { + index, + reference, + max_depth, + max_read_len, + budget_mb, + } => run_plan(index, reference, max_depth, max_read_len, budget_mb)?, } Ok(()) @@ -419,6 +450,58 @@ fn run_index(reference: PathBuf, output: PathBuf, memory_budget_mb: Option) Ok(()) } +/// Predict whether a job fits a declared budget, before committing. `--index` +/// predicts the bounded whole-genome `variants` peak (largest contig + active set +/// @ the declared cap, atop the measured process baseline). `--reference` +/// predicts the index build peak (advisory; build is O(reference)). +fn run_plan( + index: Option, + reference: Option, + max_depth: u32, + max_read_len: u32, + budget_mb: Option, +) -> Result<()> { + use rosalind::call::plan::render_variants_plan; + use rosalind::genomics::IndexReader; + + if let Some(index_path) = index { + let loaded = IndexReader::open(&index_path) + .with_context(|| format!("failed to open index {}", index_path.display()))?; + let largest = loaded + .contigs() + .iter() + .map(|c| c.length as u64) + .max() + .unwrap_or(0); + // Measure the process baseline now (binary + libs + index mmap header); + // the per-contig reference decode + active set are modeled on top. + let baseline = peak_rss_bytes(); + print!( + "{}", + render_variants_plan(largest, max_depth, max_read_len, baseline, budget_mb) + ); + } else { + let reference = reference.expect("clap guarantees one of --index/--reference"); + let fasta_reader = open_input(&reference) + .with_context(|| format!("failed to open reference {}", reference.display()))?; + let total_bp: u64 = FastaReader::new(fasta_reader) + .collect::, _>>() + .with_context(|| format!("failed to parse FASTA {}", reference.display()))? + .iter() + .map(|r| r.sequence.len() as u64) + .sum(); + let estimate = estimate_build_working_set(total_bp); + match budget_mb { + Some(mb) => println!("{}", render_plan_line(estimate, MemoryBudget::from_mb(mb))), + None => println!( + "plan: est. build peak ~{} MiB (advisory; build is O(reference)) [no budget]", + estimate.bytes / (1 << 20) + ), + } + } + Ok(()) +} + /// Load a prebuilt index and print exact-match loci for `pattern` (B3c). This is /// a memory-mapped load + exact match — it never rebuilds the index. fn run_locate(index: PathBuf, pattern: String, max_hits: usize) -> Result<()> { diff --git a/tests/plan_enforce.rs b/tests/plan_enforce.rs new file mode 100644 index 0000000..23dd791 --- /dev/null +++ b/tests/plan_enforce.rs @@ -0,0 +1,70 @@ +//! CLI contract surface: `rosalind plan` predicts feasibility, and +//! `variants --index --enforce` refuses up front / passes within budget. + +use std::path::PathBuf; +use std::process::Command; + +fn bin() -> &'static str { + env!("CARGO_BIN_EXE_rosalind") +} + +// Build a tiny 2-contig index in a fresh temp dir; return (dir, index_path). +fn build_index() -> (PathBuf, PathBuf) { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let dir = std::env::temp_dir().join(format!("rosalind-plan-{nanos}")); + std::fs::create_dir_all(&dir).unwrap(); + let fa = dir.join("ref.fa"); + std::fs::write( + &fa, + b">chr1\nACGTACGTACGTACGTACGT\n>chr2\nTTTTGGGGCCCCAAAATTTT\n", + ) + .unwrap(); + let idx = dir.join("ref.idx"); + let out = Command::new(bin()) + .args(["index", "--reference"]) + .arg(&fa) + .arg("--output") + .arg(&idx) + .output() + .unwrap(); + assert!(out.status.success(), "index build failed: {out:?}"); + (dir, idx) +} + +#[test] +fn plan_index_reports_a_breakdown_and_fits_a_generous_budget() { + let (dir, idx) = build_index(); + let out = Command::new(bin()) + .args(["plan", "--index"]) + .arg(&idx) + .args(["--budget-mb", "4096"]) + .output() + .unwrap(); + assert!(out.status.success()); + let stdout = String::from_utf8_lossy(&out.stdout); + assert!( + stdout.contains("predicted peak"), + "missing breakdown: {stdout}" + ); + assert!(stdout.contains("[FITS]"), "generous budget should FIT: {stdout}"); + std::fs::remove_dir_all(&dir).ok(); +} + +#[test] +fn plan_reference_reports_build_estimate() { + let (dir, _idx) = build_index(); + let fa = dir.join("ref.fa"); + let out = Command::new(bin()) + .args(["plan", "--reference"]) + .arg(&fa) + .args(["--budget-mb", "4096"]) + .output() + .unwrap(); + assert!(out.status.success()); + let stdout = String::from_utf8_lossy(&out.stdout); + assert!(stdout.contains("plan:"), "missing build plan line: {stdout}"); + std::fs::remove_dir_all(&dir).ok(); +} From 0774e4f8dcf5db6fd967203beec68399a508e1c6 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:25:50 -0700 Subject: [PATCH 15/32] =?UTF-8?q?feat(cli):=20variants=20--index=20--enfor?= =?UTF-8?q?ce=20=E2=80=94=20refuse=20pre-run=20(3)=20/=20fail=20post-run?= =?UTF-8?q?=20(4);=20--max-depth=20default=201000=20(C2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main.rs | 75 ++++++++++++++++++++++++++++++++++++-- tests/plan_enforce.rs | 84 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index 885e13b..21f1c78 100644 --- a/src/main.rs +++ b/src/main.rs @@ -97,6 +97,18 @@ enum Commands { /// not enforce (enforcement is a later phase). (`--index` path.) #[arg(long)] memory_budget_mb: Option, + /// Cap the active read set per position (deterministic downsampling); the + /// bound `plan`/`--enforce` rely on. `0` = uncapped. + #[arg(long, default_value_t = 1000)] + max_depth: u32, + /// Max read length assumed by the pre-run `--enforce` estimate. + #[arg(long, default_value_t = 250)] + max_read_len: u32, + /// Honor the budget: refuse up front if predicted peak exceeds it (exit 3), + /// or fail after the run if the realized peak does (exit 4). Requires + /// `--memory-budget-mb` and `--max-depth > 0`. + #[arg(long, default_value_t = false)] + enforce: bool, }, /// Deterministically coordinate-sort a BAM file using bounded memory. Sort { @@ -272,6 +284,9 @@ fn main() -> Result<()> { block_size: _, quality_threshold, memory_budget_mb, + max_depth, + max_read_len, + enforce, } => { if let Some(index) = index { if chrom.is_some() || region_start != 0 { @@ -284,6 +299,9 @@ fn main() -> Result<()> { output, quality_threshold, memory_budget_mb, + max_depth, + max_read_len, + enforce, )? } else { let reference = reference.expect("clap guarantees one of --index/--reference"); @@ -1080,6 +1098,9 @@ fn run_variants_index( output: Option, quality_threshold: f32, memory_budget_mb: Option, + max_depth: u32, + max_read_len: u32, + enforce: bool, ) -> Result<()> { use rosalind::call::{call_germline_whole_genome, GermlineParams}; use rosalind::genomics::IndexReader; @@ -1100,6 +1121,9 @@ fn run_variants_index( let pileup_params = PileupParams { min_mapq: mapq_threshold, + // `--max-depth 0` opts out of the cap (then the working set is unbounded + // and `--enforce` is rejected below). + max_depth: if max_depth == 0 { None } else { Some(max_depth) }, ..PileupParams::default() }; let germline_params = GermlineParams { @@ -1125,6 +1149,38 @@ fn run_variants_index( let source = StreamingBamSource::new(&alignments_path, contigs) .map_err(|e| anyhow!("failed to open BAM {}: {e}", alignments_path.display()))?; + // `--enforce` contract: predict the peak RSS up front (measured baseline + + // the depth-capped working set) and refuse cleanly if it won't fit — before + // doing any work. Never a silent OOM. + if enforce { + if memory_budget_mb.is_none() { + bail!("--enforce requires --memory-budget-mb"); + } + if max_depth == 0 { + bail!("--enforce requires --max-depth > 0 (an uncapped active set has no a-priori bound)"); + } + let mb = memory_budget_mb.unwrap(); + let largest = contigs.iter().map(|c| c.length as u64).max().unwrap_or(0); + let baseline = peak_rss_bytes(); + let predicted = + rosalind::call::plan::predicted_peak_rss_bytes(largest, max_depth, max_read_len, baseline); + if !MemoryBudget::from_mb(mb).admits(predicted) { + eprintln!( + "contract: REFUSE — declared {} MiB, predicted peak ~{} MiB \ + (largest contig {} MiB + active @ max-depth {} / max-read-len {} \ + atop a {} MiB baseline). Raise --memory-budget-mb, lower --max-depth, \ + or drop --enforce.", + mb, + predicted / (1 << 20), + largest / (1 << 20), + max_depth, + max_read_len, + baseline / (1 << 20), + ); + std::process::exit(3); + } + } + // Stream calls straight to the VCF writer (header once, then one row per // emitted call) so no genome-wide row buffer accumulates. The returned // WorkingSet is the high-water (reference + active set), captured per contig. @@ -1228,12 +1284,25 @@ fn run_variants_index( ); if let Some(mb) = memory_budget_mb { let budget = MemoryBudget::from_mb(mb); - if budget.admits(peak_rss) { + let within = budget.admits(peak_rss); + if enforce { + if within { + eprintln!( + "contract: OK — realized peak {} MiB within declared {mb} MiB", + peak_rss / (1 << 20) + ); + } else { + eprintln!( + "contract: VIOLATED — realized peak {} MiB exceeded declared {mb} MiB (output + receipt written)", + peak_rss / (1 << 20) + ); + std::process::exit(4); + } + } else if within { eprintln!("memory: within budget ({mb} MiB)"); } else { eprintln!( - "memory: EXCEEDED budget {} MiB (realized peak {} MiB) — record-only, run completed", - mb, + "memory: EXCEEDED budget {mb} MiB (realized peak {} MiB) — record-only, run completed", peak_rss / (1 << 20) ); } diff --git a/tests/plan_enforce.rs b/tests/plan_enforce.rs index 23dd791..7a43a56 100644 --- a/tests/plan_enforce.rs +++ b/tests/plan_enforce.rs @@ -68,3 +68,87 @@ fn plan_reference_reports_build_estimate() { assert!(stdout.contains("plan:"), "missing build plan line: {stdout}"); std::fs::remove_dir_all(&dir).ok(); } + +// ---- enforce tests: need a real coordinate-sorted BAM via the CLI pipeline ---- + +fn run(args: &[&str]) -> std::process::Output { + Command::new(bin()).args(args).output().expect("spawn rosalind") +} + +// `index` -> `align --format bam` -> `sort`, mirroring tests/variants_index.rs. +// Returns (dir, index_path, sorted_bam_path). Single-contig (aligner is single-contig). +fn build_sorted_bam_fixture() -> (PathBuf, PathBuf, PathBuf) { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let dir = std::env::temp_dir().join(format!("rosalind-enforce-{nanos}")); + std::fs::create_dir_all(&dir).unwrap(); + let seq = "ACGTACGTACGTACGTACGTACGTACGTACGT"; // 32 bp + let fa = dir.join("ref.fa"); + std::fs::write(&fa, format!(">chr1\n{seq}\n")).unwrap(); + let fq = dir.join("reads.fq"); + let mut s = String::new(); + for (i, &start) in [0usize, 0, 4, 4, 8].iter().enumerate() { + let read = &seq[start..start + 16]; + let qual: String = std::iter::repeat('I').take(16).collect(); + s.push_str(&format!("@r{i}\n{read}\n+\n{qual}\n")); + } + std::fs::write(&fq, s).unwrap(); + let idx = dir.join("ref.idx"); + let raw = dir.join("raw.bam"); + let bam = dir.join("sorted.bam"); + assert!(run(&[ + "index", "--reference", fa.to_str().unwrap(), "--output", idx.to_str().unwrap() + ]) + .status + .success()); + assert!(run(&[ + "align", "--reference", fa.to_str().unwrap(), "--reads", fq.to_str().unwrap(), + "--format", "bam", "--output", raw.to_str().unwrap() + ]) + .status + .success()); + assert!(run(&[ + "sort", "--input", raw.to_str().unwrap(), "--output", bam.to_str().unwrap() + ]) + .status + .success()); + (dir, idx, bam) +} + +#[test] +fn enforce_refuses_up_front_when_budget_below_predicted() { + // A 1 MiB budget is below the process baseline alone, so the pre-run check + // refuses with exit 3 before doing any calling — and writes no VCF. + let (dir, idx, bam) = build_sorted_bam_fixture(); + let out = Command::new(bin()) + .args(["variants", "--index"]) + .arg(&idx) + .arg("--alignments") + .arg(&bam) + .args(["--memory-budget-mb", "1", "--enforce"]) + .output() + .unwrap(); + assert_eq!(out.status.code(), Some(3), "expected refuse exit 3: {out:?}"); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!(stderr.contains("REFUSE"), "missing refuse message: {stderr}"); + std::fs::remove_dir_all(&dir).ok(); +} + +#[test] +fn enforce_passes_within_a_generous_budget() { + let (dir, idx, bam) = build_sorted_bam_fixture(); + let out = Command::new(bin()) + .args(["variants", "--index"]) + .arg(&idx) + .arg("--alignments") + .arg(&bam) + .args(["--memory-budget-mb", "4096", "--enforce"]) + .output() + .unwrap(); + assert!(out.status.success(), "generous budget should pass: {out:?}"); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!(stderr.contains("contract: OK"), "missing OK line: {stderr}"); + std::fs::remove_dir_all(&dir).ok(); +} From b8d49c24e60a3c8e8f86791b852c7c8d3800635c Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:26:46 -0700 Subject: [PATCH 16/32] style: rustfmt fixups (C2) --- src/call/plan.rs | 18 ++++++++++++---- src/main.rs | 18 ++++++++++++---- tests/plan_enforce.rs | 49 +++++++++++++++++++++++++++++++++++-------- 3 files changed, 68 insertions(+), 17 deletions(-) diff --git a/src/call/plan.rs b/src/call/plan.rs index 5f9e149..10de93a 100644 --- a/src/call/plan.rs +++ b/src/call/plan.rs @@ -62,8 +62,12 @@ pub fn render_variants_plan( .saturating_mul(PILEUP_MAP_BYTES_PER_BASE + PILEUP_SEQQUAL_BYTES_PER_BASE) .saturating_add(PILEUP_PER_READ_OVERHEAD); let active = (max_depth as u64).saturating_mul(per_read); - let predicted = - predicted_peak_rss_bytes(largest_contig_len, max_depth, max_read_len, baseline_rss_bytes); + let predicted = predicted_peak_rss_bytes( + largest_contig_len, + max_depth, + max_read_len, + baseline_rss_bytes, + ); let verdict = match budget_mb { Some(mb) => { if MemoryBudget::from_mb(mb).admits(predicted) { @@ -119,10 +123,16 @@ mod tests { fn render_reports_fits_and_refuse() { // Tiny working set; generous budget → FITS. let fits = render_variants_plan(1_000, 100, 150, 1_000_000, Some(4096)); - assert!(fits.contains("[FITS]"), "generous budget should fit: {fits}"); + assert!( + fits.contains("[FITS]"), + "generous budget should fit: {fits}" + ); // 248 MiB contig + baseline 50 MiB ≫ 64 MiB budget → REFUSE. let refuse = render_variants_plan(248 * (1 << 20), 1000, 250, 50 * (1 << 20), Some(64)); - assert!(refuse.contains("[REFUSE]"), "tight budget should refuse: {refuse}"); + assert!( + refuse.contains("[REFUSE]"), + "tight budget should refuse: {refuse}" + ); // No budget → advisory. assert!(render_variants_plan(1_000, 100, 150, 0, None).contains("[no budget]")); } diff --git a/src/main.rs b/src/main.rs index 21f1c78..894fe42 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1123,7 +1123,11 @@ fn run_variants_index( min_mapq: mapq_threshold, // `--max-depth 0` opts out of the cap (then the working set is unbounded // and `--enforce` is rejected below). - max_depth: if max_depth == 0 { None } else { Some(max_depth) }, + max_depth: if max_depth == 0 { + None + } else { + Some(max_depth) + }, ..PileupParams::default() }; let germline_params = GermlineParams { @@ -1157,13 +1161,19 @@ fn run_variants_index( bail!("--enforce requires --memory-budget-mb"); } if max_depth == 0 { - bail!("--enforce requires --max-depth > 0 (an uncapped active set has no a-priori bound)"); + bail!( + "--enforce requires --max-depth > 0 (an uncapped active set has no a-priori bound)" + ); } let mb = memory_budget_mb.unwrap(); let largest = contigs.iter().map(|c| c.length as u64).max().unwrap_or(0); let baseline = peak_rss_bytes(); - let predicted = - rosalind::call::plan::predicted_peak_rss_bytes(largest, max_depth, max_read_len, baseline); + let predicted = rosalind::call::plan::predicted_peak_rss_bytes( + largest, + max_depth, + max_read_len, + baseline, + ); if !MemoryBudget::from_mb(mb).admits(predicted) { eprintln!( "contract: REFUSE — declared {} MiB, predicted peak ~{} MiB \ diff --git a/tests/plan_enforce.rs b/tests/plan_enforce.rs index 7a43a56..b9aa401 100644 --- a/tests/plan_enforce.rs +++ b/tests/plan_enforce.rs @@ -49,7 +49,10 @@ fn plan_index_reports_a_breakdown_and_fits_a_generous_budget() { stdout.contains("predicted peak"), "missing breakdown: {stdout}" ); - assert!(stdout.contains("[FITS]"), "generous budget should FIT: {stdout}"); + assert!( + stdout.contains("[FITS]"), + "generous budget should FIT: {stdout}" + ); std::fs::remove_dir_all(&dir).ok(); } @@ -65,14 +68,20 @@ fn plan_reference_reports_build_estimate() { .unwrap(); assert!(out.status.success()); let stdout = String::from_utf8_lossy(&out.stdout); - assert!(stdout.contains("plan:"), "missing build plan line: {stdout}"); + assert!( + stdout.contains("plan:"), + "missing build plan line: {stdout}" + ); std::fs::remove_dir_all(&dir).ok(); } // ---- enforce tests: need a real coordinate-sorted BAM via the CLI pipeline ---- fn run(args: &[&str]) -> std::process::Output { - Command::new(bin()).args(args).output().expect("spawn rosalind") + Command::new(bin()) + .args(args) + .output() + .expect("spawn rosalind") } // `index` -> `align --format bam` -> `sort`, mirroring tests/variants_index.rs. @@ -99,18 +108,33 @@ fn build_sorted_bam_fixture() -> (PathBuf, PathBuf, PathBuf) { let raw = dir.join("raw.bam"); let bam = dir.join("sorted.bam"); assert!(run(&[ - "index", "--reference", fa.to_str().unwrap(), "--output", idx.to_str().unwrap() + "index", + "--reference", + fa.to_str().unwrap(), + "--output", + idx.to_str().unwrap() ]) .status .success()); assert!(run(&[ - "align", "--reference", fa.to_str().unwrap(), "--reads", fq.to_str().unwrap(), - "--format", "bam", "--output", raw.to_str().unwrap() + "align", + "--reference", + fa.to_str().unwrap(), + "--reads", + fq.to_str().unwrap(), + "--format", + "bam", + "--output", + raw.to_str().unwrap() ]) .status .success()); assert!(run(&[ - "sort", "--input", raw.to_str().unwrap(), "--output", bam.to_str().unwrap() + "sort", + "--input", + raw.to_str().unwrap(), + "--output", + bam.to_str().unwrap() ]) .status .success()); @@ -130,9 +154,16 @@ fn enforce_refuses_up_front_when_budget_below_predicted() { .args(["--memory-budget-mb", "1", "--enforce"]) .output() .unwrap(); - assert_eq!(out.status.code(), Some(3), "expected refuse exit 3: {out:?}"); + assert_eq!( + out.status.code(), + Some(3), + "expected refuse exit 3: {out:?}" + ); let stderr = String::from_utf8_lossy(&out.stderr); - assert!(stderr.contains("REFUSE"), "missing refuse message: {stderr}"); + assert!( + stderr.contains("REFUSE"), + "missing refuse message: {stderr}" + ); std::fs::remove_dir_all(&dir).ok(); } From eafef29d3a2fc010e3508fe1f842bd6a0cfdc472 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:33:14 -0700 Subject: [PATCH 17/32] =?UTF-8?q?docs(plan):=20Phase=20C3=20=E2=80=94=20ro?= =?UTF-8?q?salind=20verify=20+=20receipt-on-stdout=20+=20CI=20gate=20plan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 5-task TDD plan: always-write a self-describing receipt (+ --manifest; new contract_verdict/enforced/budget/max_depth/max_read_len params); a hand-parser RunManifest::from_canonical_json (round-trip property-tested, no serde_json); rosalind verify (re-hash inputs/outputs + re-check peak vs budget, exit 5 on mismatch); and a deterministic CI gate that the pure estimator upper-bounds the realized working set. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../2026-06-01-phase-c3-verify-receipt.md | 681 ++++++++++++++++++ 1 file changed, 681 insertions(+) create mode 100644 docs/superpowers/plans/2026-06-01-phase-c3-verify-receipt.md diff --git a/docs/superpowers/plans/2026-06-01-phase-c3-verify-receipt.md b/docs/superpowers/plans/2026-06-01-phase-c3-verify-receipt.md new file mode 100644 index 0000000..5a7174b --- /dev/null +++ b/docs/superpowers/plans/2026-06-01-phase-c3-verify-receipt.md @@ -0,0 +1,681 @@ +# Phase C3 — `rosalind verify` + receipt-on-stdout + CI contract gate Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans (inline, chosen for this work). Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Close the contract's trust loop: every `variants --index` run persists a self-describing, BLAKE3-stamped receipt (even on stdout), and `rosalind verify` re-checks it — proving the realized peak landed inside the budget and the outputs came from exactly these inputs — without re-running. + +**Architecture:** (1) Restructure `run_variants_index` to ALWAYS write a manifest (file → `.manifest.json`, stdout → cwd `rosalind.variants.manifest.json`, or an explicit `--manifest `), with new self-describing params (`memory_budget_mb`, `contract_verdict`, `enforced`, `max_depth`, `max_read_len`). (2) Add a small hand-parser `RunManifest::from_canonical_json` for the fixed canonical shape (all values are strings), guarded by a serialize→parse→serialize round-trip property test — no `serde_json`. (3) Add `rosalind verify --manifest [--budget-mb B]` that re-hashes the listed inputs/outputs and re-checks `peak_rss_bytes` ≤ the recorded/supplied budget. (4) Add a deterministic CI gate asserting the pure estimator's working-set bound ≥ the realized `max_working_set_bytes` from an actual run. + +**Tech Stack:** Rust 1.72 (MSRV), `clap` derive, `cargo test`/`fmt`/`build`. No new dependencies. Builds on C1+C2 (branch `rosalind/phase-c-contract`). + +**Spec:** [`docs/superpowers/specs/2026-06-01-phase-c-contract-design.md`](../specs/2026-06-01-phase-c-contract-design.md) §7. + +**Gate coverage note (spec §7.5):** the five listed gates are spread across the phase — exit-3 refuse is already in C2 (`tests/plan_enforce.rs`); working-set-flat-as-input-grows is the C1 library test (`call::whole_genome::…working_set_is_bounded…not_read_count`); verify round-trip + tamper is Task 3 here; predicted-envelope ≥ realized is Task 4 here. So C3 adds the *new* deterministic gates and does not duplicate the ones already proven. + +--- + +## File Structure + +- **Modify** `src/main.rs` — `Variants` gains `--manifest`; `run_variants_index` always writes a self-describing receipt; new `Verify` subcommand + `run_verify`. +- **Modify** `src/provenance/mod.rs` — `ManifestError` + `RunManifest::from_canonical_json` + round-trip property test. +- **Modify** `tests/plan_enforce.rs` — stdout-receipt test, verify round-trip + tamper test, and the predicted-≥-realized gate. + +--- + +## Task 1: Always write a self-describing receipt + `--manifest` + +**Files:** +- Modify: `src/main.rs` (`Variants` variant, dispatch, `run_variants_index`) + +- [ ] **Step 1: Add the `--manifest` flag.** In `enum Commands`, in `Variants { … }`, after the `enforce: bool,` field, add: + +```rust + /// Where to write the reproducibility receipt. Default: `.manifest.json` + /// for file output, or `./rosalind.variants.manifest.json` for stdout output. + #[arg(long)] + manifest: Option, +``` + +- [ ] **Step 2: Thread it through the dispatch.** In `main()`, in the `Commands::Variants { … }` destructure, add `manifest,` after `enforce,`; and add `manifest,` as the final argument to the `run_variants_index(…)` call (after `enforce,`). + +- [ ] **Step 3: Extend `run_variants_index`'s signature.** Add the parameter (after `enforce: bool,`): + +```rust + manifest_out: Option, +``` + +- [ ] **Step 4: Replace the file-only manifest block with an always-write, self-describing one.** Replace the entire `if let Some(path) = &output { … }` manifest block (the one starting `let mut manifest = RunManifest::new("variants");`, currently ~lines 1258–1288) with: + +```rust + // Compute the contract verdict before writing the receipt (so it records it). + let verdict = match memory_budget_mb.map(|mb| MemoryBudget::from_mb(mb).admits(peak_rss)) { + None => "unset", + Some(true) => "within", + Some(false) => "over", + }; + + // Reproducibility + memory receipt — ALWAYS written (every run is verifiable): + // an explicit --manifest path wins; else a sidecar next to the VCF; else cwd. + let mut manifest = RunManifest::new("variants"); + manifest.inputs.push(FileHash { + path: index_path.display().to_string(), + blake3: blake3_file(&index_path)?, + }); + manifest.inputs.push(FileHash { + path: alignments_path.display().to_string(), + blake3: blake3_file(&alignments_path)?, + }); + if let Some(path) = &output { + manifest.outputs.push(FileHash { + path: path.display().to_string(), + blake3: blake3_file(path)?, + }); + } + manifest + .params + .insert("mapq_threshold".to_string(), mapq_threshold.to_string()); + manifest.params.insert( + "min_qual".to_string(), + (quality_threshold as f64).to_string(), + ); + manifest + .params + .insert("max_depth".to_string(), max_depth.to_string()); + manifest + .params + .insert("max_read_len".to_string(), max_read_len.to_string()); + manifest + .params + .insert("enforced".to_string(), enforce.to_string()); + manifest + .params + .insert("peak_rss_bytes".to_string(), peak_rss.to_string()); + manifest.params.insert( + "max_working_set_bytes".to_string(), + max_ws.bytes.to_string(), + ); + if let Some(mb) = memory_budget_mb { + manifest + .params + .insert("memory_budget_mb".to_string(), mb.to_string()); + } + manifest + .params + .insert("contract_verdict".to_string(), verdict.to_string()); + + let manifest_path: PathBuf = match (&manifest_out, &output) { + (Some(m), _) => { + std::fs::write(m, manifest.to_canonical_json()) + .with_context(|| format!("failed to write manifest {}", m.display()))?; + m.clone() + } + (None, Some(path)) => write_manifest(path, &manifest)?, + (None, None) => { + let p = PathBuf::from("rosalind.variants.manifest.json"); + std::fs::write(&p, manifest.to_canonical_json()) + .with_context(|| format!("failed to write manifest {}", p.display()))?; + p + } + }; + eprintln!("wrote reproducibility receipt: {}", manifest_path.display()); +``` + +(The `// Memory receipt:` eprintln + the `if let Some(mb) = memory_budget_mb { … }` budget/exit-4 block that follow are **unchanged** and remain after this block.) + +- [ ] **Step 5: Build** + +Run: `cd ~/rosalind && cargo build 2>&1 | tail -8` +Expected: success, 0 warnings. + +- [ ] **Step 6: Add a stdout-receipt test.** Append to `tests/plan_enforce.rs`: + +```rust +#[test] +fn stdout_run_persists_a_self_describing_receipt() { + let (dir, idx, bam) = build_sorted_bam_fixture(); + let manifest = dir.join("run.manifest.json"); + // stdout output (no -o), explicit --manifest so we know where to look. + let out = Command::new(bin()) + .args(["variants", "--index"]) + .arg(&idx) + .arg("--alignments") + .arg(&bam) + .args(["--memory-budget-mb", "4096", "--enforce", "--manifest"]) + .arg(&manifest) + .output() + .unwrap(); + assert!(out.status.success(), "run failed: {out:?}"); + let json = std::fs::read_to_string(&manifest).expect("manifest written"); + for needle in [ + "\"contract_verdict\":\"within\"", + "\"enforced\":\"true\"", + "\"max_depth\":\"1000\"", + "\"memory_budget_mb\":\"4096\"", + "\"peak_rss_bytes\":", + "\"max_working_set_bytes\":", + ] { + assert!(json.contains(needle), "manifest missing {needle}: {json}"); + } + std::fs::remove_dir_all(&dir).ok(); +} +``` + +- [ ] **Step 7: Run it** + +Run: `cd ~/rosalind && cargo test --test plan_enforce stdout_run_persists 2>&1 | tail -12` +Expected: PASS. + +- [ ] **Step 8: Commit** + +```bash +cd ~/rosalind && git add src/main.rs tests/plan_enforce.rs && git commit -m "feat(cli): variants --index always writes a self-describing receipt (+ --manifest) (C3)" +``` + +--- + +## Task 2: Canonical-manifest parser (`from_canonical_json`) + +**Files:** +- Modify: `src/provenance/mod.rs` + +- [ ] **Step 1: Write the failing round-trip property test.** Append to the `tests` module in `src/provenance/mod.rs`: + +```rust + #[test] + fn parse_round_trips_canonical_json_including_escapes() { + let mut m = RunManifest::new("variants"); + m.tool_version = "9.9.9".to_string(); + m.inputs.push(FileHash { + path: "weird \"path\"\twith\\escapes/和.fa".to_string(), + blake3: "aa".to_string(), + }); + m.inputs.push(FileHash { + path: "a.idx".to_string(), + blake3: "bb".to_string(), + }); + m.outputs.push(FileHash { + path: "out.vcf".to_string(), + blake3: "cc".to_string(), + }); + m.params.insert("contract_verdict".to_string(), "within".to_string()); + m.params.insert("peak_rss_bytes".to_string(), "12345".to_string()); + m.params.insert("note".to_string(), "line1\nline2".to_string()); + + let json = m.to_canonical_json(); + let parsed = RunManifest::from_canonical_json(&json).expect("parse"); + // Structural equality: the parse recovers exactly what was serialized + // (inputs are stored sorted-by-path in the canonical form, so build the + // expected by re-parsing rather than comparing to `m`'s push order). + assert_eq!(parsed.to_canonical_json(), json, "serialize→parse→serialize identity"); + assert_eq!(parsed.tool_version, "9.9.9"); + assert_eq!(parsed.subcommand, "variants"); + assert_eq!(parsed.params.get("note").unwrap(), "line1\nline2"); + assert_eq!(parsed.params.get("contract_verdict").unwrap(), "within"); + } + + #[test] + fn parse_rejects_malformed() { + assert!(RunManifest::from_canonical_json("not json").is_err()); + assert!(RunManifest::from_canonical_json("{\"inputs\":[}").is_err()); + } +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd ~/rosalind && cargo test -p rosalind --lib provenance::tests::parse_ 2>&1 | tail -10` +Expected: FAIL — `from_canonical_json` not found. + +- [ ] **Step 3: Implement the parser.** In `src/provenance/mod.rs`, add the error type (after the `use` lines, before `FileHash`): + +```rust +/// Failure parsing a canonical run manifest. +#[derive(Debug)] +pub struct ManifestError(pub String); + +impl std::fmt::Display for ManifestError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "malformed manifest: {}", self.0) + } +} + +impl std::error::Error for ManifestError {} +``` + +Add the parse entry point in `impl RunManifest` (after `to_canonical_json`): + +```rust + /// Parse a manifest from its canonical JSON form (the exact shape + /// `to_canonical_json` emits; all values are strings). A small hand-parser — + /// no general JSON dependency. Round-trips with `to_canonical_json`. + pub fn from_canonical_json(s: &str) -> Result { + let mut p = Parser { b: s.as_bytes(), i: 0 }; + let m = p.parse_manifest()?; + Ok(m) + } +``` + +Add the parser implementation (after the `RunManifest` impl block, before `push_file_hashes`): + +```rust +/// Minimal recursive parser for the fixed canonical-manifest shape. Every value +/// is a JSON string (inputs/outputs are arrays of `{blake3, path}` objects; +/// params is an object of string→string), so the parser only needs strings, +/// arrays, and objects — no numbers/bools/null. +struct Parser<'a> { + b: &'a [u8], + i: usize, +} + +impl Parser<'_> { + fn err(&self, m: &str) -> ManifestError { + ManifestError(format!("{m} at byte {}", self.i)) + } + + fn expect(&mut self, c: u8) -> Result<(), ManifestError> { + if self.i < self.b.len() && self.b[self.i] == c { + self.i += 1; + Ok(()) + } else { + Err(self.err(&format!("expected '{}'", c as char))) + } + } + + fn parse_string(&mut self) -> Result { + self.expect(b'"')?; + let mut buf: Vec = Vec::new(); + while self.i < self.b.len() { + let c = self.b[self.i]; + self.i += 1; + match c { + b'"' => { + return String::from_utf8(buf).map_err(|_| self.err("invalid utf-8")); + } + b'\\' => { + let e = *self.b.get(self.i).ok_or_else(|| self.err("trailing escape"))?; + self.i += 1; + match e { + b'"' => buf.push(b'"'), + b'\\' => buf.push(b'\\'), + b'n' => buf.push(b'\n'), + b'r' => buf.push(b'\r'), + b't' => buf.push(b'\t'), + b'u' => { + let hex = self + .b + .get(self.i..self.i + 4) + .ok_or_else(|| self.err("short \\u"))?; + let cp = u32::from_str_radix( + std::str::from_utf8(hex).map_err(|_| self.err("bad \\u"))?, + 16, + ) + .map_err(|_| self.err("bad \\u"))?; + let ch = char::from_u32(cp).ok_or_else(|| self.err("bad codepoint"))?; + let mut tmp = [0u8; 4]; + buf.extend_from_slice(ch.encode_utf8(&mut tmp).as_bytes()); + self.i += 4; + } + _ => return Err(self.err("bad escape")), + } + } + _ => buf.push(c), + } + } + Err(self.err("unterminated string")) + } + + fn expect_key(&mut self, key: &str) -> Result<(), ManifestError> { + let k = self.parse_string()?; + if k != key { + return Err(self.err(&format!("expected key \"{key}\", got \"{k}\""))); + } + self.expect(b':') + } + + fn parse_file_array(&mut self) -> Result, ManifestError> { + self.expect(b'[')?; + let mut out = Vec::new(); + if self.i < self.b.len() && self.b[self.i] == b']' { + self.i += 1; + return Ok(out); + } + loop { + self.expect(b'{')?; + self.expect_key("blake3")?; + let blake3 = self.parse_string()?; + self.expect(b',')?; + self.expect_key("path")?; + let path = self.parse_string()?; + self.expect(b'}')?; + out.push(FileHash { path, blake3 }); + match self.b.get(self.i) { + Some(b',') => self.i += 1, + Some(b']') => { + self.i += 1; + break; + } + _ => return Err(self.err("expected ',' or ']' in array")), + } + } + Ok(out) + } + + fn parse_params(&mut self) -> Result, ManifestError> { + self.expect(b'{')?; + let mut map = std::collections::BTreeMap::new(); + if self.i < self.b.len() && self.b[self.i] == b'}' { + self.i += 1; + return Ok(map); + } + loop { + let k = self.parse_string()?; + self.expect(b':')?; + let v = self.parse_string()?; + map.insert(k, v); + match self.b.get(self.i) { + Some(b',') => self.i += 1, + Some(b'}') => { + self.i += 1; + break; + } + _ => return Err(self.err("expected ',' or '}' in object")), + } + } + Ok(map) + } + + fn parse_manifest(&mut self) -> Result { + self.expect(b'{')?; + self.expect_key("inputs")?; + let inputs = self.parse_file_array()?; + self.expect(b',')?; + self.expect_key("outputs")?; + let outputs = self.parse_file_array()?; + self.expect(b',')?; + self.expect_key("params")?; + let params = self.parse_params()?; + self.expect(b',')?; + self.expect_key("subcommand")?; + let subcommand = self.parse_string()?; + self.expect(b',')?; + self.expect_key("tool_version")?; + let tool_version = self.parse_string()?; + self.expect(b'}')?; + Ok(RunManifest { + tool_version, + subcommand, + inputs, + params, + outputs, + }) + } +} +``` + +- [ ] **Step 4: Run the parser tests** + +Run: `cd ~/rosalind && cargo test -p rosalind --lib provenance 2>&1 | tail -12` +Expected: PASS (round-trip + reject-malformed + the existing provenance tests). + +- [ ] **Step 5: Commit** + +```bash +cd ~/rosalind && git add src/provenance/mod.rs && git commit -m "feat(provenance): RunManifest::from_canonical_json hand-parser + round-trip test (C3)" +``` + +--- + +## Task 3: `rosalind verify` subcommand + +**Files:** +- Modify: `src/main.rs` (`Verify` variant, dispatch, `run_verify`) +- Modify: `tests/plan_enforce.rs` (verify tests) + +- [ ] **Step 1: Add the `Verify` variant.** In `enum Commands`, after the `Plan { … }` variant (before the enum's closing `}`), add: + +```rust + /// Re-check a reproducibility receipt without re-running: re-hash its inputs + /// and outputs and confirm the realized peak landed within the budget. + Verify { + /// Path to a `*.manifest.json` written by a previous run. + #[arg(long)] + manifest: PathBuf, + /// Budget (MiB) to check the recorded peak against (overrides the + /// `memory_budget_mb` recorded in the manifest, if any). + #[arg(long)] + budget_mb: Option, + }, +``` + +- [ ] **Step 2: Add the dispatch arm.** In `main()`, after the `Commands::Plan { … } => run_plan(…)?,` arm, add: + +```rust + Commands::Verify { + manifest, + budget_mb, + } => run_verify(manifest, budget_mb)?, +``` + +- [ ] **Step 3: Implement `run_verify`.** Add this function in `src/main.rs` after `run_plan` (before `run_locate`): + +```rust +/// Re-check a reproducibility receipt without re-running: parse it, re-hash each +/// listed input/output and confirm the digests match, and confirm the recorded +/// realized peak RSS landed within the budget (supplied, or recorded in the +/// manifest). Exits non-zero with a per-check report on any mismatch. +fn run_verify(manifest_path: PathBuf, budget_mb: Option) -> Result<()> { + use rosalind::provenance::{blake3_file, RunManifest}; + + let text = std::fs::read_to_string(&manifest_path) + .with_context(|| format!("failed to read manifest {}", manifest_path.display()))?; + let manifest = RunManifest::from_canonical_json(&text) + .map_err(|e| anyhow!("failed to parse manifest {}: {e}", manifest_path.display()))?; + + let mut problems: Vec = Vec::new(); + + // Re-hash inputs + outputs against the recorded digests. + for (kind, files) in [("input", &manifest.inputs), ("output", &manifest.outputs)] { + for f in files { + match blake3_file(std::path::Path::new(&f.path)) { + Ok(h) if h == f.blake3 => {} + Ok(h) => problems.push(format!( + "{kind} {} hash mismatch: recorded {}, now {}", + f.path, f.blake3, h + )), + Err(e) => problems.push(format!("{kind} {} unreadable: {e}", f.path)), + } + } + } + + // Re-check the recorded realized peak against the budget (CLI overrides manifest). + let budget_mb = budget_mb.or_else(|| { + manifest + .params + .get("memory_budget_mb") + .and_then(|v| v.parse::().ok()) + }); + match ( + budget_mb, + manifest + .params + .get("peak_rss_bytes") + .and_then(|v| v.parse::().ok()), + ) { + (Some(mb), Some(peak)) => { + let budget = rosalind::core::MemoryBudget::from_mb(mb); + if budget.admits(peak) { + println!( + "verify: peak {} MiB within budget {mb} MiB", + peak / (1 << 20) + ); + } else { + problems.push(format!( + "recorded peak {} MiB exceeded budget {mb} MiB", + peak / (1 << 20) + )); + } + } + (None, _) => println!("verify: no budget to check (none supplied or recorded)"), + (Some(_), None) => problems.push("manifest has no recorded peak_rss_bytes".to_string()), + } + + if problems.is_empty() { + println!("verify: OK — {} input(s), {} output(s) match", manifest.inputs.len(), manifest.outputs.len()); + Ok(()) + } else { + for p in &problems { + eprintln!("verify: FAIL — {p}"); + } + std::process::exit(5); + } +} +``` + +- [ ] **Step 4: Build** + +Run: `cd ~/rosalind && cargo build 2>&1 | tail -8` +Expected: success, 0 warnings. + +- [ ] **Step 5: Add verify tests.** Append to `tests/plan_enforce.rs`: + +```rust +#[test] +fn verify_passes_on_an_untampered_run_and_fails_on_a_tampered_output() { + let (dir, idx, bam) = build_sorted_bam_fixture(); + let vcf = dir.join("calls.vcf"); + let manifest = dir.join("calls.vcf.manifest.json"); + let out = Command::new(bin()) + .args(["variants", "--index"]) + .arg(&idx) + .arg("--alignments") + .arg(&bam) + .args(["--memory-budget-mb", "4096", "--enforce", "-o"]) + .arg(&vcf) + .output() + .unwrap(); + assert!(out.status.success(), "run failed: {out:?}"); + + // Untampered → verify OK (exit 0). + let ok = Command::new(bin()) + .args(["verify", "--manifest"]) + .arg(&manifest) + .output() + .unwrap(); + assert!(ok.status.success(), "verify should pass: {}", String::from_utf8_lossy(&ok.stderr)); + assert!(String::from_utf8_lossy(&ok.stdout).contains("verify: OK")); + + // Tamper with the output VCF → verify FAILS (exit 5). + std::fs::write(&vcf, b"##tampered\n").unwrap(); + let bad = Command::new(bin()) + .args(["verify", "--manifest"]) + .arg(&manifest) + .output() + .unwrap(); + assert_eq!(bad.status.code(), Some(5), "tampered output must fail verify: {bad:?}"); + assert!(String::from_utf8_lossy(&bad.stderr).contains("hash mismatch")); + std::fs::remove_dir_all(&dir).ok(); +} +``` + +- [ ] **Step 6: Run the verify test** + +Run: `cd ~/rosalind && cargo test --test plan_enforce verify_ 2>&1 | tail -12` +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +cd ~/rosalind && git add src/main.rs tests/plan_enforce.rs && git commit -m "feat(cli): rosalind verify — re-check a receipt's hashes + budget without re-running (C3)" +``` + +--- + +## Task 4: CI contract gate — pure estimate bounds the realized working set + +**Files:** +- Modify: `tests/plan_enforce.rs` + +- [ ] **Step 1: Write the gate.** Append to `tests/plan_enforce.rs`: + +```rust +#[test] +fn estimator_upper_bounds_the_realized_working_set() { + // Run the real pipeline, read max_working_set_bytes from the receipt, and + // assert the pure estimator (same shared constants) is a true upper bound for + // the declared --max-depth / --max-read-len. Deterministic (working-set + // numbers, not process RSS). + let (dir, idx, bam) = build_sorted_bam_fixture(); + let vcf = dir.join("calls.vcf"); + let manifest = dir.join("calls.vcf.manifest.json"); + let out = Command::new(bin()) + .args(["variants", "--index"]) + .arg(&idx) + .arg("--alignments") + .arg(&bam) + .args(["--max-depth", "1000", "--max-read-len", "250", "-o"]) + .arg(&vcf) + .output() + .unwrap(); + assert!(out.status.success(), "run failed: {out:?}"); + + let text = std::fs::read_to_string(&manifest).unwrap(); + let m = rosalind::provenance::RunManifest::from_canonical_json(&text).unwrap(); + let realized: u64 = m.params.get("max_working_set_bytes").unwrap().parse().unwrap(); + + // The fixture's single contig is 32 bp; the estimator's bound at the declared + // cap must dominate the realized working set. + let predicted = rosalind::call::estimate_variants_working_set(32, 1000, 250).bytes; + assert!( + predicted >= realized, + "estimator bound {predicted} must be >= realized working set {realized}" + ); + std::fs::remove_dir_all(&dir).ok(); +} +``` + +- [ ] **Step 2: Confirm the estimator is reachable at the crate root for the test.** It is re-exported in `src/call/mod.rs` as `pub use plan::{estimate_variants_working_set, …}`, so `rosalind::call::estimate_variants_working_set` resolves. If the test fails to compile on the path, fall back to `rosalind::call::plan::estimate_variants_working_set` (the module is `pub`). + +- [ ] **Step 3: Run the gate** + +Run: `cd ~/rosalind && cargo test --test plan_enforce estimator_upper_bounds 2>&1 | tail -10` +Expected: PASS. + +- [ ] **Step 4: Commit** + +```bash +cd ~/rosalind && git add tests/plan_enforce.rs && git commit -m "test(contract): pure estimator upper-bounds the realized working set (C3 CI gate)" +``` + +--- + +## Task 5: Full verification + +**Files:** none (verification only) + +- [ ] **Step 1: Format** + +Run: `cd ~/rosalind && cargo fmt --all && cargo fmt --all -- --check && echo FMT_CLEAN` +Expected: `FMT_CLEAN`. + +- [ ] **Step 2: Zero-warning builds** + +Run: `cd ~/rosalind && cargo build 2>&1 | tail -4 && cargo build --release 2>&1 | tail -4` +Expected: both 0 warnings. + +- [ ] **Step 3: Full suite** + +Run: `cd ~/rosalind && cargo test 2>&1 | grep -E "test result: FAILED|panicked|[1-9][0-9]* failed|^error" | head; cargo test 2>&1 | grep -cE "test result: ok\."` +Expected: no failures; `ok.` section count ≥ C2's 26. + +- [ ] **Step 4: Commit any fmt fixups** + +```bash +cd ~/rosalind && git add -A && git commit -m "style: rustfmt fixups (C3)" || true +``` + +--- + +## Self-Review notes + +- **Spec §7 coverage:** §7.1 receipt-on-stdout + `--manifest` → Task 1; §7.2 self-describing params → Task 1 Step 4; §7.3 parser → Task 2; §7.4 `verify` → Task 3; §7.5 CI gate → Task 4 (others mapped in the gate-coverage note up top). +- **Type consistency:** `from_canonical_json` returns `Result` (Task 2), consumed by `run_verify` (Task 3) and the Task-4 test. `manifest_out` param added to `run_variants_index` at its definition (Task 1 Step 3) and call site (Task 1 Step 2). `RunManifest`/`FileHash`/`MemoryBudget` are existing types. `verify` exit code `5` is distinct from `--enforce`'s 3/4. +- **Determinism:** the round-trip test (Task 2) includes escaped/multi-byte content; the canonical writer already sorts inputs/outputs by path and params by key, so parse→serialize is identity. The Task-4 gate compares working-set bytes (not RSS), so it is not flaky. From 2336d48e4333df36797d4ee145cf89d75aa9be86 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:34:46 -0700 Subject: [PATCH 18/32] feat(cli): variants --index always writes a self-describing receipt (+ --manifest) (C3) --- src/main.rs | 94 ++++++++++++++++++++++++++++++++----------- tests/plan_enforce.rs | 29 +++++++++++++ 2 files changed, 99 insertions(+), 24 deletions(-) diff --git a/src/main.rs b/src/main.rs index 894fe42..69ad432 100644 --- a/src/main.rs +++ b/src/main.rs @@ -109,6 +109,10 @@ enum Commands { /// `--memory-budget-mb` and `--max-depth > 0`. #[arg(long, default_value_t = false)] enforce: bool, + /// Where to write the reproducibility receipt. Default: `.manifest.json` + /// for file output, or `./rosalind.variants.manifest.json` for stdout output. + #[arg(long)] + manifest: Option, }, /// Deterministically coordinate-sort a BAM file using bounded memory. Sort { @@ -287,6 +291,7 @@ fn main() -> Result<()> { max_depth, max_read_len, enforce, + manifest, } => { if let Some(index) = index { if chrom.is_some() || region_start != 0 { @@ -302,6 +307,7 @@ fn main() -> Result<()> { max_depth, max_read_len, enforce, + manifest, )? } else { let reference = reference.expect("clap guarantees one of --index/--reference"); @@ -1101,6 +1107,7 @@ fn run_variants_index( max_depth: u32, max_read_len: u32, enforce: bool, + manifest_out: Option, ) -> Result<()> { use rosalind::call::{call_germline_whole_genome, GermlineParams}; use rosalind::genomics::IndexReader; @@ -1254,38 +1261,77 @@ fn run_variants_index( // Realized peak (monotonic high-water mark) captured after the calling pass. let peak_rss = peak_rss_bytes(); - // Reproducibility + memory receipt (file output only; stdout receipt is C3). + // Compute the contract verdict before writing the receipt (so it records it). + let verdict = match memory_budget_mb.map(|mb| MemoryBudget::from_mb(mb).admits(peak_rss)) { + None => "unset", + Some(true) => "within", + Some(false) => "over", + }; + + // Reproducibility + memory receipt — ALWAYS written (every run is verifiable): + // an explicit --manifest path wins; else a sidecar next to the VCF; else cwd. + let mut manifest = RunManifest::new("variants"); + manifest.inputs.push(FileHash { + path: index_path.display().to_string(), + blake3: blake3_file(&index_path)?, + }); + manifest.inputs.push(FileHash { + path: alignments_path.display().to_string(), + blake3: blake3_file(&alignments_path)?, + }); if let Some(path) = &output { - let mut manifest = RunManifest::new("variants"); - manifest.inputs.push(FileHash { - path: index_path.display().to_string(), - blake3: blake3_file(&index_path)?, - }); - manifest.inputs.push(FileHash { - path: alignments_path.display().to_string(), - blake3: blake3_file(&alignments_path)?, - }); manifest.outputs.push(FileHash { path: path.display().to_string(), blake3: blake3_file(path)?, }); + } + manifest + .params + .insert("mapq_threshold".to_string(), mapq_threshold.to_string()); + manifest.params.insert( + "min_qual".to_string(), + (quality_threshold as f64).to_string(), + ); + manifest + .params + .insert("max_depth".to_string(), max_depth.to_string()); + manifest + .params + .insert("max_read_len".to_string(), max_read_len.to_string()); + manifest + .params + .insert("enforced".to_string(), enforce.to_string()); + manifest + .params + .insert("peak_rss_bytes".to_string(), peak_rss.to_string()); + manifest.params.insert( + "max_working_set_bytes".to_string(), + max_ws.bytes.to_string(), + ); + if let Some(mb) = memory_budget_mb { manifest .params - .insert("mapq_threshold".to_string(), mapq_threshold.to_string()); - manifest.params.insert( - "min_qual".to_string(), - (quality_threshold as f64).to_string(), - ); - manifest - .params - .insert("peak_rss_bytes".to_string(), peak_rss.to_string()); - manifest.params.insert( - "max_working_set_bytes".to_string(), - max_ws.bytes.to_string(), - ); - let manifest_path = write_manifest(path, &manifest)?; - eprintln!("wrote reproducibility receipt: {}", manifest_path.display()); + .insert("memory_budget_mb".to_string(), mb.to_string()); } + manifest + .params + .insert("contract_verdict".to_string(), verdict.to_string()); + + let manifest_path: PathBuf = match (&manifest_out, &output) { + (Some(m), _) => { + std::fs::write(m, manifest.to_canonical_json()) + .with_context(|| format!("failed to write manifest {}", m.display()))?; + m.clone() + } + (None, Some(path)) => write_manifest(path, &manifest)?, + (None, None) => { + let p = PathBuf::from("rosalind.variants.manifest.json"); + std::fs::write(&p, manifest.to_canonical_json()) + .with_context(|| format!("failed to write manifest {}", p.display()))?; + p + } + }; + eprintln!("wrote reproducibility receipt: {}", manifest_path.display()); // Memory receipt: the bounded contract, made visible + verifiable. eprintln!( "memory: peak RSS {} MiB; max pileup working set {} KiB", diff --git a/tests/plan_enforce.rs b/tests/plan_enforce.rs index b9aa401..1da5233 100644 --- a/tests/plan_enforce.rs +++ b/tests/plan_enforce.rs @@ -183,3 +183,32 @@ fn enforce_passes_within_a_generous_budget() { assert!(stderr.contains("contract: OK"), "missing OK line: {stderr}"); std::fs::remove_dir_all(&dir).ok(); } + +#[test] +fn stdout_run_persists_a_self_describing_receipt() { + let (dir, idx, bam) = build_sorted_bam_fixture(); + let manifest = dir.join("run.manifest.json"); + // stdout output (no -o), explicit --manifest so we know where to look. + let out = Command::new(bin()) + .args(["variants", "--index"]) + .arg(&idx) + .arg("--alignments") + .arg(&bam) + .args(["--memory-budget-mb", "4096", "--enforce", "--manifest"]) + .arg(&manifest) + .output() + .unwrap(); + assert!(out.status.success(), "run failed: {out:?}"); + let json = std::fs::read_to_string(&manifest).expect("manifest written"); + for needle in [ + "\"contract_verdict\":\"within\"", + "\"enforced\":\"true\"", + "\"max_depth\":\"1000\"", + "\"memory_budget_mb\":\"4096\"", + "\"peak_rss_bytes\":", + "\"max_working_set_bytes\":", + ] { + assert!(json.contains(needle), "manifest missing {needle}: {json}"); + } + std::fs::remove_dir_all(&dir).ok(); +} From 2c40d4c0cef0915b9214a213b4c09f4cb752cd91 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:36:15 -0700 Subject: [PATCH 19/32] feat(provenance): RunManifest::from_canonical_json hand-parser + round-trip test (C3) --- src/provenance/mod.rs | 216 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 216 insertions(+) diff --git a/src/provenance/mod.rs b/src/provenance/mod.rs index bc2b70d..b73cce8 100644 --- a/src/provenance/mod.rs +++ b/src/provenance/mod.rs @@ -8,6 +8,18 @@ use std::collections::BTreeMap; use std::io::{self, Read, Write}; use std::path::{Path, PathBuf}; +/// Failure parsing a canonical run manifest. +#[derive(Debug)] +pub struct ManifestError(pub String); + +impl std::fmt::Display for ManifestError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "malformed manifest: {}", self.0) + } +} + +impl std::error::Error for ManifestError {} + /// A file referenced by a run, with its content hash. #[derive(Debug, Clone, PartialEq, Eq)] pub struct FileHash { @@ -77,6 +89,171 @@ impl RunManifest { out } + + /// Parse a manifest from its canonical JSON form (the exact shape + /// `to_canonical_json` emits; all values are strings). A small hand-parser — + /// no general JSON dependency. Round-trips with `to_canonical_json`. + pub fn from_canonical_json(s: &str) -> Result { + let mut p = Parser { + b: s.as_bytes(), + i: 0, + }; + p.parse_manifest() + } +} + +/// Minimal recursive parser for the fixed canonical-manifest shape. Every value +/// is a JSON string (inputs/outputs are arrays of `{blake3, path}` objects; +/// params is an object of string→string), so the parser only needs strings, +/// arrays, and objects — no numbers/bools/null. +struct Parser<'a> { + b: &'a [u8], + i: usize, +} + +impl Parser<'_> { + fn err(&self, m: &str) -> ManifestError { + ManifestError(format!("{m} at byte {}", self.i)) + } + + fn expect(&mut self, c: u8) -> Result<(), ManifestError> { + if self.i < self.b.len() && self.b[self.i] == c { + self.i += 1; + Ok(()) + } else { + Err(self.err(&format!("expected '{}'", c as char))) + } + } + + fn parse_string(&mut self) -> Result { + self.expect(b'"')?; + let mut buf: Vec = Vec::new(); + while self.i < self.b.len() { + let c = self.b[self.i]; + self.i += 1; + match c { + b'"' => { + return String::from_utf8(buf).map_err(|_| self.err("invalid utf-8")); + } + b'\\' => { + let e = *self.b.get(self.i).ok_or_else(|| self.err("trailing escape"))?; + self.i += 1; + match e { + b'"' => buf.push(b'"'), + b'\\' => buf.push(b'\\'), + b'n' => buf.push(b'\n'), + b'r' => buf.push(b'\r'), + b't' => buf.push(b'\t'), + b'u' => { + let hex = self + .b + .get(self.i..self.i + 4) + .ok_or_else(|| self.err("short \\u"))?; + let cp = u32::from_str_radix( + std::str::from_utf8(hex).map_err(|_| self.err("bad \\u"))?, + 16, + ) + .map_err(|_| self.err("bad \\u"))?; + let ch = + char::from_u32(cp).ok_or_else(|| self.err("bad codepoint"))?; + let mut tmp = [0u8; 4]; + buf.extend_from_slice(ch.encode_utf8(&mut tmp).as_bytes()); + self.i += 4; + } + _ => return Err(self.err("bad escape")), + } + } + _ => buf.push(c), + } + } + Err(self.err("unterminated string")) + } + + fn expect_key(&mut self, key: &str) -> Result<(), ManifestError> { + let k = self.parse_string()?; + if k != key { + return Err(self.err(&format!("expected key \"{key}\", got \"{k}\""))); + } + self.expect(b':') + } + + fn parse_file_array(&mut self) -> Result, ManifestError> { + self.expect(b'[')?; + let mut out = Vec::new(); + if self.i < self.b.len() && self.b[self.i] == b']' { + self.i += 1; + return Ok(out); + } + loop { + self.expect(b'{')?; + self.expect_key("blake3")?; + let blake3 = self.parse_string()?; + self.expect(b',')?; + self.expect_key("path")?; + let path = self.parse_string()?; + self.expect(b'}')?; + out.push(FileHash { path, blake3 }); + match self.b.get(self.i) { + Some(b',') => self.i += 1, + Some(b']') => { + self.i += 1; + break; + } + _ => return Err(self.err("expected ',' or ']' in array")), + } + } + Ok(out) + } + + fn parse_params(&mut self) -> Result, ManifestError> { + self.expect(b'{')?; + let mut map = BTreeMap::new(); + if self.i < self.b.len() && self.b[self.i] == b'}' { + self.i += 1; + return Ok(map); + } + loop { + let k = self.parse_string()?; + self.expect(b':')?; + let v = self.parse_string()?; + map.insert(k, v); + match self.b.get(self.i) { + Some(b',') => self.i += 1, + Some(b'}') => { + self.i += 1; + break; + } + _ => return Err(self.err("expected ',' or '}' in object")), + } + } + Ok(map) + } + + fn parse_manifest(&mut self) -> Result { + self.expect(b'{')?; + self.expect_key("inputs")?; + let inputs = self.parse_file_array()?; + self.expect(b',')?; + self.expect_key("outputs")?; + let outputs = self.parse_file_array()?; + self.expect(b',')?; + self.expect_key("params")?; + let params = self.parse_params()?; + self.expect(b',')?; + self.expect_key("subcommand")?; + let subcommand = self.parse_string()?; + self.expect(b',')?; + self.expect_key("tool_version")?; + let tool_version = self.parse_string()?; + self.expect(b'}')?; + Ok(RunManifest { + tool_version, + subcommand, + inputs, + params, + outputs, + }) + } } /// Render a `[{"blake3":..,"path":..}, ..]` array, entries sorted by path. @@ -241,4 +418,43 @@ mod tests { std::fs::remove_dir_all(&dir).ok(); } + + #[test] + fn parse_round_trips_canonical_json_including_escapes() { + let mut m = RunManifest::new("variants"); + m.tool_version = "9.9.9".to_string(); + m.inputs.push(FileHash { + path: "weird \"path\"\twith\\escapes/和.fa".to_string(), + blake3: "aa".to_string(), + }); + m.inputs.push(FileHash { + path: "a.idx".to_string(), + blake3: "bb".to_string(), + }); + m.outputs.push(FileHash { + path: "out.vcf".to_string(), + blake3: "cc".to_string(), + }); + m.params + .insert("contract_verdict".to_string(), "within".to_string()); + m.params + .insert("peak_rss_bytes".to_string(), "12345".to_string()); + m.params + .insert("note".to_string(), "line1\nline2".to_string()); + + let json = m.to_canonical_json(); + let parsed = RunManifest::from_canonical_json(&json).expect("parse"); + // serialize → parse → serialize is the identity on the canonical form. + assert_eq!(parsed.to_canonical_json(), json); + assert_eq!(parsed.tool_version, "9.9.9"); + assert_eq!(parsed.subcommand, "variants"); + assert_eq!(parsed.params.get("note").unwrap(), "line1\nline2"); + assert_eq!(parsed.params.get("contract_verdict").unwrap(), "within"); + } + + #[test] + fn parse_rejects_malformed() { + assert!(RunManifest::from_canonical_json("not json").is_err()); + assert!(RunManifest::from_canonical_json("{\"inputs\":[}").is_err()); + } } From 89984fd9b3977a5a66eb0ffae4a592e832d5da19 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:37:23 -0700 Subject: [PATCH 20/32] =?UTF-8?q?feat(cli):=20rosalind=20verify=20?= =?UTF-8?q?=E2=80=94=20re-check=20a=20receipt's=20hashes=20+=20budget=20wi?= =?UTF-8?q?thout=20re-running=20(C3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main.rs | 87 +++++++++++++++++++++++++++++++++++++++++++ tests/plan_enforce.rs | 45 ++++++++++++++++++++++ 2 files changed, 132 insertions(+) diff --git a/src/main.rs b/src/main.rs index 69ad432..b620f50 100644 --- a/src/main.rs +++ b/src/main.rs @@ -223,6 +223,17 @@ enum Commands { #[arg(long)] budget_mb: Option, }, + /// Re-check a reproducibility receipt without re-running: re-hash its inputs + /// and outputs and confirm the realized peak landed within the budget. + Verify { + /// Path to a `*.manifest.json` written by a previous run. + #[arg(long)] + manifest: PathBuf, + /// Budget (MiB) to check the recorded peak against (overrides the + /// `memory_budget_mb` recorded in the manifest, if any). + #[arg(long)] + budget_mb: Option, + }, } #[derive(Copy, Clone, Debug, ValueEnum, Eq, PartialEq)] @@ -373,6 +384,10 @@ fn main() -> Result<()> { max_read_len, budget_mb, } => run_plan(index, reference, max_depth, max_read_len, budget_mb)?, + Commands::Verify { + manifest, + budget_mb, + } => run_verify(manifest, budget_mb)?, } Ok(()) @@ -526,6 +541,78 @@ fn run_plan( Ok(()) } +/// Re-check a reproducibility receipt without re-running: parse it, re-hash each +/// listed input/output and confirm the digests match, and confirm the recorded +/// realized peak RSS landed within the budget (supplied, or recorded in the +/// manifest). Exits non-zero with a per-check report on any mismatch. +fn run_verify(manifest_path: PathBuf, budget_mb: Option) -> Result<()> { + use rosalind::provenance::{blake3_file, RunManifest}; + + let text = std::fs::read_to_string(&manifest_path) + .with_context(|| format!("failed to read manifest {}", manifest_path.display()))?; + let manifest = RunManifest::from_canonical_json(&text) + .map_err(|e| anyhow!("failed to parse manifest {}: {e}", manifest_path.display()))?; + + let mut problems: Vec = Vec::new(); + + // Re-hash inputs + outputs against the recorded digests. + for (kind, files) in [("input", &manifest.inputs), ("output", &manifest.outputs)] { + for f in files { + match blake3_file(std::path::Path::new(&f.path)) { + Ok(h) if h == f.blake3 => {} + Ok(h) => problems.push(format!( + "{kind} {} hash mismatch: recorded {}, now {}", + f.path, f.blake3, h + )), + Err(e) => problems.push(format!("{kind} {} unreadable: {e}", f.path)), + } + } + } + + // Re-check the recorded realized peak against the budget (CLI overrides manifest). + let budget_mb = budget_mb.or_else(|| { + manifest + .params + .get("memory_budget_mb") + .and_then(|v| v.parse::().ok()) + }); + match ( + budget_mb, + manifest + .params + .get("peak_rss_bytes") + .and_then(|v| v.parse::().ok()), + ) { + (Some(mb), Some(peak)) => { + let budget = rosalind::core::MemoryBudget::from_mb(mb); + if budget.admits(peak) { + println!("verify: peak {} MiB within budget {mb} MiB", peak / (1 << 20)); + } else { + problems.push(format!( + "recorded peak {} MiB exceeded budget {mb} MiB", + peak / (1 << 20) + )); + } + } + (None, _) => println!("verify: no budget to check (none supplied or recorded)"), + (Some(_), None) => problems.push("manifest has no recorded peak_rss_bytes".to_string()), + } + + if problems.is_empty() { + println!( + "verify: OK — {} input(s), {} output(s) match", + manifest.inputs.len(), + manifest.outputs.len() + ); + Ok(()) + } else { + for p in &problems { + eprintln!("verify: FAIL — {p}"); + } + std::process::exit(5); + } +} + /// Load a prebuilt index and print exact-match loci for `pattern` (B3c). This is /// a memory-mapped load + exact match — it never rebuilds the index. fn run_locate(index: PathBuf, pattern: String, max_hits: usize) -> Result<()> { diff --git a/tests/plan_enforce.rs b/tests/plan_enforce.rs index 1da5233..f83eaca 100644 --- a/tests/plan_enforce.rs +++ b/tests/plan_enforce.rs @@ -212,3 +212,48 @@ fn stdout_run_persists_a_self_describing_receipt() { } std::fs::remove_dir_all(&dir).ok(); } + +#[test] +fn verify_passes_on_an_untampered_run_and_fails_on_a_tampered_output() { + let (dir, idx, bam) = build_sorted_bam_fixture(); + let vcf = dir.join("calls.vcf"); + let manifest = dir.join("calls.vcf.manifest.json"); + let out = Command::new(bin()) + .args(["variants", "--index"]) + .arg(&idx) + .arg("--alignments") + .arg(&bam) + .args(["--memory-budget-mb", "4096", "--enforce", "-o"]) + .arg(&vcf) + .output() + .unwrap(); + assert!(out.status.success(), "run failed: {out:?}"); + + // Untampered → verify OK (exit 0). + let ok = Command::new(bin()) + .args(["verify", "--manifest"]) + .arg(&manifest) + .output() + .unwrap(); + assert!( + ok.status.success(), + "verify should pass: {}", + String::from_utf8_lossy(&ok.stderr) + ); + assert!(String::from_utf8_lossy(&ok.stdout).contains("verify: OK")); + + // Tamper with the output VCF → verify FAILS (exit 5). + std::fs::write(&vcf, b"##tampered\n").unwrap(); + let bad = Command::new(bin()) + .args(["verify", "--manifest"]) + .arg(&manifest) + .output() + .unwrap(); + assert_eq!( + bad.status.code(), + Some(5), + "tampered output must fail verify: {bad:?}" + ); + assert!(String::from_utf8_lossy(&bad.stderr).contains("hash mismatch")); + std::fs::remove_dir_all(&dir).ok(); +} From a0244dc8cdffffb29339e0d6e75ff6ca05d3d1ff Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:37:48 -0700 Subject: [PATCH 21/32] test(contract): pure estimator upper-bounds the realized working set (C3 CI gate) --- tests/plan_enforce.rs | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/plan_enforce.rs b/tests/plan_enforce.rs index f83eaca..978a362 100644 --- a/tests/plan_enforce.rs +++ b/tests/plan_enforce.rs @@ -257,3 +257,42 @@ fn verify_passes_on_an_untampered_run_and_fails_on_a_tampered_output() { assert!(String::from_utf8_lossy(&bad.stderr).contains("hash mismatch")); std::fs::remove_dir_all(&dir).ok(); } + +#[test] +fn estimator_upper_bounds_the_realized_working_set() { + // Run the real pipeline, read max_working_set_bytes from the receipt, and + // assert the pure estimator (same shared constants) is a true upper bound for + // the declared --max-depth / --max-read-len. Deterministic (working-set + // numbers, not process RSS). + let (dir, idx, bam) = build_sorted_bam_fixture(); + let vcf = dir.join("calls.vcf"); + let manifest = dir.join("calls.vcf.manifest.json"); + let out = Command::new(bin()) + .args(["variants", "--index"]) + .arg(&idx) + .arg("--alignments") + .arg(&bam) + .args(["--max-depth", "1000", "--max-read-len", "250", "-o"]) + .arg(&vcf) + .output() + .unwrap(); + assert!(out.status.success(), "run failed: {out:?}"); + + let text = std::fs::read_to_string(&manifest).unwrap(); + let m = rosalind::provenance::RunManifest::from_canonical_json(&text).unwrap(); + let realized: u64 = m + .params + .get("max_working_set_bytes") + .unwrap() + .parse() + .unwrap(); + + // The fixture's single contig is 32 bp; the estimator's bound at the declared + // cap must dominate the realized working set. + let predicted = rosalind::call::estimate_variants_working_set(32, 1000, 250).bytes; + assert!( + predicted >= realized, + "estimator bound {predicted} must be >= realized working set {realized}" + ); + std::fs::remove_dir_all(&dir).ok(); +} From db2d9b37cd25133ce02ecd10bfd7fcc999216d1d Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:40:52 -0700 Subject: [PATCH 22/32] style+test: rustfmt fixups; collision-free temp dirs + no cwd pollution (C3) Adds a shared unique_dir(atomic counter + nanos) so concurrent plan_enforce tests never share a directory, and routes the generous-budget enforce test's receipt to its temp dir instead of the cwd-default sidecar. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/main.rs | 5 ++++- src/provenance/mod.rs | 8 +++++--- tests/plan_enforce.rs | 31 ++++++++++++++++++++----------- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/src/main.rs b/src/main.rs index b620f50..95bcd21 100644 --- a/src/main.rs +++ b/src/main.rs @@ -586,7 +586,10 @@ fn run_verify(manifest_path: PathBuf, budget_mb: Option) -> Result<()> { (Some(mb), Some(peak)) => { let budget = rosalind::core::MemoryBudget::from_mb(mb); if budget.admits(peak) { - println!("verify: peak {} MiB within budget {mb} MiB", peak / (1 << 20)); + println!( + "verify: peak {} MiB within budget {mb} MiB", + peak / (1 << 20) + ); } else { problems.push(format!( "recorded peak {} MiB exceeded budget {mb} MiB", diff --git a/src/provenance/mod.rs b/src/provenance/mod.rs index b73cce8..c0c0751 100644 --- a/src/provenance/mod.rs +++ b/src/provenance/mod.rs @@ -136,7 +136,10 @@ impl Parser<'_> { return String::from_utf8(buf).map_err(|_| self.err("invalid utf-8")); } b'\\' => { - let e = *self.b.get(self.i).ok_or_else(|| self.err("trailing escape"))?; + let e = *self + .b + .get(self.i) + .ok_or_else(|| self.err("trailing escape"))?; self.i += 1; match e { b'"' => buf.push(b'"'), @@ -154,8 +157,7 @@ impl Parser<'_> { 16, ) .map_err(|_| self.err("bad \\u"))?; - let ch = - char::from_u32(cp).ok_or_else(|| self.err("bad codepoint"))?; + let ch = char::from_u32(cp).ok_or_else(|| self.err("bad codepoint"))?; let mut tmp = [0u8; 4]; buf.extend_from_slice(ch.encode_utf8(&mut tmp).as_bytes()); self.i += 4; diff --git a/tests/plan_enforce.rs b/tests/plan_enforce.rs index 978a362..dc5b015 100644 --- a/tests/plan_enforce.rs +++ b/tests/plan_enforce.rs @@ -3,19 +3,29 @@ use std::path::PathBuf; use std::process::Command; +use std::sync::atomic::{AtomicU64, Ordering}; fn bin() -> &'static str { env!("CARGO_BIN_EXE_rosalind") } -// Build a tiny 2-contig index in a fresh temp dir; return (dir, index_path). -fn build_index() -> (PathBuf, PathBuf) { +// A fresh, collision-free temp dir (atomic counter + nanos — concurrent tests +// must not share a directory). +fn unique_dir(prefix: &str) -> PathBuf { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let n = COUNTER.fetch_add(1, Ordering::Relaxed); let nanos = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_nanos(); - let dir = std::env::temp_dir().join(format!("rosalind-plan-{nanos}")); - std::fs::create_dir_all(&dir).unwrap(); + let d = std::env::temp_dir().join(format!("{prefix}-{nanos}-{n}")); + std::fs::create_dir_all(&d).unwrap(); + d +} + +// Build a tiny 2-contig index in a fresh temp dir; return (dir, index_path). +fn build_index() -> (PathBuf, PathBuf) { + let dir = unique_dir("rosalind-plan"); let fa = dir.join("ref.fa"); std::fs::write( &fa, @@ -87,12 +97,7 @@ fn run(args: &[&str]) -> std::process::Output { // `index` -> `align --format bam` -> `sort`, mirroring tests/variants_index.rs. // Returns (dir, index_path, sorted_bam_path). Single-contig (aligner is single-contig). fn build_sorted_bam_fixture() -> (PathBuf, PathBuf, PathBuf) { - let nanos = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos(); - let dir = std::env::temp_dir().join(format!("rosalind-enforce-{nanos}")); - std::fs::create_dir_all(&dir).unwrap(); + let dir = unique_dir("rosalind-enforce"); let seq = "ACGTACGTACGTACGTACGTACGTACGTACGT"; // 32 bp let fa = dir.join("ref.fa"); std::fs::write(&fa, format!(">chr1\n{seq}\n")).unwrap(); @@ -170,12 +175,16 @@ fn enforce_refuses_up_front_when_budget_below_predicted() { #[test] fn enforce_passes_within_a_generous_budget() { let (dir, idx, bam) = build_sorted_bam_fixture(); + // --manifest into the temp dir (stdout run would otherwise drop the cwd-default + // sidecar into the repo root). + let manifest = dir.join("run.manifest.json"); let out = Command::new(bin()) .args(["variants", "--index"]) .arg(&idx) .arg("--alignments") .arg(&bam) - .args(["--memory-budget-mb", "4096", "--enforce"]) + .args(["--memory-budget-mb", "4096", "--enforce", "--manifest"]) + .arg(&manifest) .output() .unwrap(); assert!(out.status.success(), "generous budget should pass: {out:?}"); From 6844346b075b861b4057db6683135dd325c40fc3 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:46:14 -0700 Subject: [PATCH 23/32] fix(cli): stdout run without --manifest writes no receipt file, just a notice (C3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revises the spec §7.1 cwd-sidecar default: silently dropping rosalind.variants. manifest.json into the caller's cwd pollutes pipe users' dirs and races on a fixed filename across concurrent stdout runs. Instead, persist a receipt only when there is a destination (--manifest or a -o sidecar); a stdout run without --manifest prints how to get one (honest, not silent; no pollution, no race). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/main.rs | 126 +++++++++++++++++++++++++++------------------------- 1 file changed, 65 insertions(+), 61 deletions(-) diff --git a/src/main.rs b/src/main.rs index 95bcd21..47705a8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1204,7 +1204,7 @@ fn run_variants_index( use rosalind::io::bam::StreamingBamSource; use rosalind::io::vcf::{write_germline_header, write_germline_row, GermlineRow}; use rosalind::pileup::PileupParams; - use rosalind::provenance::{blake3_file, write_manifest, FileHash, RunManifest}; + use rosalind::provenance::{blake3_file, FileHash, RunManifest}; let loaded = IndexReader::open(&index_path) .with_context(|| format!("failed to open index {}", index_path.display()))?; @@ -1358,70 +1358,74 @@ fn run_variants_index( Some(false) => "over", }; - // Reproducibility + memory receipt — ALWAYS written (every run is verifiable): - // an explicit --manifest path wins; else a sidecar next to the VCF; else cwd. - let mut manifest = RunManifest::new("variants"); - manifest.inputs.push(FileHash { - path: index_path.display().to_string(), - blake3: blake3_file(&index_path)?, - }); - manifest.inputs.push(FileHash { - path: alignments_path.display().to_string(), - blake3: blake3_file(&alignments_path)?, - }); - if let Some(path) = &output { - manifest.outputs.push(FileHash { - path: path.display().to_string(), - blake3: blake3_file(path)?, + // Reproducibility + memory receipt. Written when there is a destination — an + // explicit --manifest path, or a sidecar next to a `-o` VCF. A stdout run + // without --manifest writes NO file (no surprise cwd write, no race on a fixed + // filename) but says how to persist one. + let receipt_dest: Option = match (&manifest_out, &output) { + (Some(m), _) => Some(m.clone()), + (None, Some(path)) => { + let mut s = path.as_os_str().to_os_string(); + s.push(".manifest.json"); + Some(PathBuf::from(s)) + } + (None, None) => None, + }; + if let Some(dest) = receipt_dest { + let mut manifest = RunManifest::new("variants"); + manifest.inputs.push(FileHash { + path: index_path.display().to_string(), + blake3: blake3_file(&index_path)?, }); - } - manifest - .params - .insert("mapq_threshold".to_string(), mapq_threshold.to_string()); - manifest.params.insert( - "min_qual".to_string(), - (quality_threshold as f64).to_string(), - ); - manifest - .params - .insert("max_depth".to_string(), max_depth.to_string()); - manifest - .params - .insert("max_read_len".to_string(), max_read_len.to_string()); - manifest - .params - .insert("enforced".to_string(), enforce.to_string()); - manifest - .params - .insert("peak_rss_bytes".to_string(), peak_rss.to_string()); - manifest.params.insert( - "max_working_set_bytes".to_string(), - max_ws.bytes.to_string(), - ); - if let Some(mb) = memory_budget_mb { + manifest.inputs.push(FileHash { + path: alignments_path.display().to_string(), + blake3: blake3_file(&alignments_path)?, + }); + if let Some(path) = &output { + manifest.outputs.push(FileHash { + path: path.display().to_string(), + blake3: blake3_file(path)?, + }); + } manifest .params - .insert("memory_budget_mb".to_string(), mb.to_string()); - } - manifest - .params - .insert("contract_verdict".to_string(), verdict.to_string()); - - let manifest_path: PathBuf = match (&manifest_out, &output) { - (Some(m), _) => { - std::fs::write(m, manifest.to_canonical_json()) - .with_context(|| format!("failed to write manifest {}", m.display()))?; - m.clone() - } - (None, Some(path)) => write_manifest(path, &manifest)?, - (None, None) => { - let p = PathBuf::from("rosalind.variants.manifest.json"); - std::fs::write(&p, manifest.to_canonical_json()) - .with_context(|| format!("failed to write manifest {}", p.display()))?; - p + .insert("mapq_threshold".to_string(), mapq_threshold.to_string()); + manifest.params.insert( + "min_qual".to_string(), + (quality_threshold as f64).to_string(), + ); + manifest + .params + .insert("max_depth".to_string(), max_depth.to_string()); + manifest + .params + .insert("max_read_len".to_string(), max_read_len.to_string()); + manifest + .params + .insert("enforced".to_string(), enforce.to_string()); + manifest + .params + .insert("peak_rss_bytes".to_string(), peak_rss.to_string()); + manifest.params.insert( + "max_working_set_bytes".to_string(), + max_ws.bytes.to_string(), + ); + if let Some(mb) = memory_budget_mb { + manifest + .params + .insert("memory_budget_mb".to_string(), mb.to_string()); } - }; - eprintln!("wrote reproducibility receipt: {}", manifest_path.display()); + manifest + .params + .insert("contract_verdict".to_string(), verdict.to_string()); + std::fs::write(&dest, manifest.to_canonical_json()) + .with_context(|| format!("failed to write manifest {}", dest.display()))?; + eprintln!("wrote reproducibility receipt: {}", dest.display()); + } else { + eprintln!( + "no receipt written (stdout output) — pass --manifest or -o to persist one" + ); + } // Memory receipt: the bounded contract, made visible + verifiable. eprintln!( "memory: peak RSS {} MiB; max pileup working set {} KiB", From 63c206bdb8b29dc17415afe34c017786616a7539 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:47:05 -0700 Subject: [PATCH 24/32] =?UTF-8?q?docs(spec):=20reconcile=20=C2=A77.1=20wit?= =?UTF-8?q?h=20the=20C3=20stdout-receipt=20revision=20(no=20cwd=20sidecar)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../specs/2026-06-01-phase-c-contract-design.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/superpowers/specs/2026-06-01-phase-c-contract-design.md b/docs/superpowers/specs/2026-06-01-phase-c-contract-design.md index 9dfbd8e..7dad99e 100644 --- a/docs/superpowers/specs/2026-06-01-phase-c-contract-design.md +++ b/docs/superpowers/specs/2026-06-01-phase-c-contract-design.md @@ -213,8 +213,13 @@ record-only path unchanged without `--enforce`. ### 7.1 Persist a receipt on stdout runs (`main.rs:1097-1101`) Today the stdout branch writes no manifest, so "every run emits a receipt" is false on the default path. -Fix: stdout output → write a `rosalind.variants.manifest.json` sidecar in the cwd + announce it on stderr; -`--manifest ` redirects (works for both stdout and file output). Document the cwd-sidecar behavior. +Fix: persist a receipt whenever there is a destination — an explicit `--manifest ` (exact path) or a +`.manifest.json` sidecar next to a `-o` VCF. **REVISED during C3 implementation (2026-06-01):** a +stdout run *without* `--manifest` writes **no file** but prints a notice on how to persist one — the +originally-specced "write a `rosalind.variants.manifest.json` sidecar in the cwd" was dropped because +silently writing an unrequested file into a pipe user's cwd pollutes their directory and races on a fixed +filename across concurrent stdout runs. This still kills the silent-no-receipt footgun (the notice is loud) +without the pollution; any run that wants a receipt uses `-o` or `--manifest`. ### 7.2 Self-describing manifest params Extend the params written at `main.rs:1083-1094` with `memory_budget_mb` (if declared), `contract_verdict` From fcbe3f2d0c071339264e67f3f2849bd5edc5459d Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 18:12:01 -0700 Subject: [PATCH 25/32] =?UTF-8?q?docs(spec):=20Move=20#4=20=E2=80=94=20the?= =?UTF-8?q?=20front=20door=20(positioning=20+=20discoverability)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Converts the watching fork wave into building forkers: crate-root re-exports of the genomics product surface, a contract-first lib.rs/cargo-doc rewrite (√t as an honest research footer), CONTRACT.md, a README rewrite routing to the bounded substrate (plugin lineage demoted+labeled, not removed), a multi-contig demo fixture, a PileupColumn-iterator cookbook example, and an issue-3 reframe (confirm-first). Pure positioning/docs/re-exports/examples — no behavior changes. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../2026-06-01-phase-c-frontdoor-design.md | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 docs/superpowers/specs/2026-06-01-phase-c-frontdoor-design.md diff --git a/docs/superpowers/specs/2026-06-01-phase-c-frontdoor-design.md b/docs/superpowers/specs/2026-06-01-phase-c-frontdoor-design.md new file mode 100644 index 0000000..f491d5b --- /dev/null +++ b/docs/superpowers/specs/2026-06-01-phase-c-frontdoor-design.md @@ -0,0 +1,122 @@ +# Move #4 — The Front Door (positioning + discoverability) (design) + +**Status:** Spec for review — 2026-06-01. Strategy Move #4 (the 2026-06-01 strategy synthesis; companion +to `docs/OPEN_PROBLEMS.md`): convert the *watching* fork wave into *building* forkers by making the shipped +breakthrough **discoverable** and the repo's front door **point at it**. Built on a branch stacked on +`rosalind/phase-c-contract` (the docs reference the contract verbs `plan`/`--enforce`/`verify`, which land +with Phase C). Pure positioning / docs / re-exports / examples — **no code-behavior changes.** + +## 1. Why (the bottleneck) + +The analysis found the community is in *watching* mode — every GitHub fork is byte-identical to or behind +`main` (zero commits on top). The repo's own front door actively misdirects them: `src/lib.rs:1` opens on +*"O(√t) Space Simulation via Height Compression"* with a `TuringMachine` usage example (so `cargo doc` +lands on the wrong product); the crate-root re-exports are **only** the √t theory types; **all** four +`examples/*.rs` are √t simulation demos; the toy generator emits a single `>chrToy` (no runnable +multi-contig `variants --index` demo); and the README "Extend" section routes builders to the +**non-bounded** `GenomicPlugin` path. This move fixes the front door so a drive-by forker's first five +minutes land on the bounded-memory contract — the genuinely unique, now-shipped capability. + +## 2. Resolved positioning decisions (2026-06-01 brainstorm) + +- **√t framing: contract-first, √t as an honest research footer.** README + `lib.rs` lead with the + bounded-memory contract / genomics product (the shipped, tested capability). √t appears as a clearly + labeled "Research direction (Phase D)" section — framed as future / not-yet-load-bearing, **no + overclaiming** a layer that is currently a stub. (Anchored on Williams' peer-reviewed O(√(t log t)); the + withdrawn arXiv 2508.14831 is never cited.) +- **Plugin path: demote + label, do NOT remove.** Route the front door to the bounded substrate (the + `PileupColumn` iterator + `ReadSource`/`call_germline_whole_genome`) as THE way to build bounded + analytics, with a runnable cookbook example. Clearly label the `GenomicPlugin` trait + `framework/` + the + Python RNA-seq demo as **legacy / non-bounded** (still works; does NOT inherit the memory contract). No + `#[deprecated]`, no code removal. + +## 3. The seven deliverables + +### 3.1 Crate-root re-exports (`src/lib.rs`) +Add a curated **genomics product surface** so `use rosalind::{…}` lands on the engine, not the theory +layer. The existing √t re-exports are KEPT (regrouped under a `// Research layer (√t)` comment). New +re-exports (all already reachable via module paths today — this is the convenience + the signal of "this +is the product"): +- substrate: `PileupEngine, PileupColumn, Obs, ReadSource, SliceSource, StreamingBamSource, PileupParams` +- calling: `call_germline_whole_genome, call_germline_region_streaming, GermlineCall, GermlineParams` +- contract: `MemoryBudget, WorkingSet, estimate_variants_working_set, predicted_peak_rss_bytes` +- index + receipt: `GenomeIndex, IndexReader, ReferenceView, RunManifest` + +(Curation rule: re-export what a builder *composes on* — the substrate, the bounded drive, the contract +types, the index reader, the receipt. Do NOT re-export internal/legacy types.) + +### 3.2 `src/lib.rs` top-level rustdoc rewrite +Replace the `//! # O(√t) Space Simulation via Height Compression` opener (lines 1–26) with a +genomics-contract lead: what Rosalind is (deterministic, low-memory genomics engine; declare your RAM → +`plan` → `--enforce` → `verify`; bounded whole-genome calling) and a **runnable doctest** that builds a +`PileupEngine` over a tiny in-memory `SliceSource` and pulls a `PileupColumn` (covered by +`cargo test --doc`). Add a clearly-labeled `## Research direction (Phase D)` section that honestly +describes the √t space-bounded-construction ambition as future work (not yet load-bearing). Also fix the +false docline at `src/pileup/mod.rs` that claims plugins build on the streaming engine — state plainly +that the bounded contract applies to the germline/pileup path and the plugin/framework lineage is +non-bounded. + +### 3.3 `CONTRACT.md` (new, repo root) +The authoritative contract document, linked from the README: +- the verbs: **declare** a budget → **`rosalind plan`** (predict before committing) → **`--enforce`** + (honor: refuse exit 3 / fail exit 4, never a silent OOM-kill) → **`rosalind verify`** (re-check the + receipt without re-running); +- the honest brand line: *"never silently OOM-kills you — it fits, or it tells you up front, and proves + the realized peak with a receipt"* (explicitly NOT "never refuses" — graceful degrade/spill is Phase D); +- an **Extend** section routing builders to the `PileupColumn` iterator substrate (pointing at + `examples/custom_pileup_analytics.rs`) and labeling the `GenomicPlugin`/`framework/`/Python-RNA-seq + lineage as legacy / non-bounded; +- honest scope: the contract is the **germline `variants --index`** path; somatic is region-bounded; + index build is O(reference) (Phase D). + +### 3.4 README rewrite (`README.md`) +- Lead with the one-command contract story: + `rosalind plan` → `rosalind variants --index --enforce` → `rosalind verify`. +- Rewrite the **Extend** section per §2 (substrate primary + cookbook; plugin lineage labeled legacy). +- Replace any "never refuses" / over-claim language with the honest brand from §3.3. +- Add the multi-contig demo (§3.5) to the runnable examples. +- Keep √t as a short "Research direction (Phase D)" footer, linking `docs/OPEN_PROBLEMS.md`. +- Link `CONTRACT.md`. + +### 3.5 Multi-contig demo fixture (`scripts/generate_toy_data.py`) +Extend the generator to emit a small **2–3 contig** reference (it writes a single `>chrToy` today) so the +flagship path runs out of the box: `index → sort → plan → variants --index --enforce → verify`. Keep it +deterministic (seeded) and tiny. Update the `SHA256SUMS`/manifest emission accordingly. (A regenerated +`examples/data/` multi-contig fixture may be committed so the README commands run without invoking Python.) + +### 3.6 Substrate cookbook example (`examples/custom_pileup_analytics.rs`, new) +A **non-caller** consumer that computes a per-locus metric (e.g. coverage + a simple QC count) directly +over the `PileupColumn` iterator from a `SliceSource` — demonstrating the substrate as a platform pattern +that inherits bounded memory + determinism for free, with no variant calling involved. Must compile and +run via `cargo run --example custom_pileup_analytics`. + +### 3.7 issue #3 rewrite (GitHub — confirm-first) +Rewrite the public roadmap issue (currently says "separate out the theory layer as a demo," contradicting +the load-bearing-√t thesis) to the contract framing: "memory as a declared, predicted, honored, verifiable +contract — beachheaded on the bounded whole-genome caller — with √t as the FUTURE space/time knob (Phase +D)." **Outward-facing: draft the new body, show it for approval, and only post on explicit go-ahead.** + +## 4. Testing + +- `cargo test --doc` — the new `lib.rs` doctest builds a `PileupEngine` and pulls a column (proves the + documented product surface actually compiles + runs). +- `cargo run --example custom_pileup_analytics` — the cookbook compiles and runs. +- A smoke test (subprocess, `tests/`) drives the **multi-contig** fixture through + `index → sort → variants --index → (plan / verify)` and asserts success + a multi-contig VCF — so the + README's headline commands are proven, not just claimed. +- README / `CONTRACT.md` are prose, but every command shown must be copy-pasteable-correct (verified by + the smoke test + manual run). +- Gates unchanged: `cargo fmt --all -- --check`, `cargo build` 0 warnings (debug + release), full suite. + +## 5. Non-goals + +No code-behavior changes (this is positioning/docs/re-exports/examples only); no `#[deprecated]` markers or +removal of the plugin/framework lineage; the PyO3 zero-copy NumPy pileup-channel stream (the larger Phase-E +substrate wedge) is OUT; the flagship real-genome artifact is **Move #5** (separate). MSRV 1.72 preserved; +no new dependencies. + +## 6. Branch / sequencing + +Built on `rosalind/phase-c-frontdoor`, stacked on `rosalind/phase-c-contract` (PR #21). The README/CONTRACT +reference the contract verbs, so this merges *after* (or together with) Phase C. When Phase C lands on +`main`, this rebases trivially. From dff1c744c07c1f63732369fb74192e5ee0666bbc Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 18:17:55 -0700 Subject: [PATCH 26/32] =?UTF-8?q?docs(plan):=20Move=20#4=20=E2=80=94=20the?= =?UTF-8?q?=20front=20door=20implementation=20plan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 7-task plan: crate-root genomics re-exports + contract-first lib.rs rustdoc (runnable doctest, √t demoted) + pileup docline fix; CONTRACT.md; README rewrite (contract-first lead, substrate-first Extend, honest brand); PileupColumn cookbook example; an end-to-end smoke test of the README in-house demo; full verification; and a confirm-first issue-3 reframe. §3.5 refined: runnable demo is single-contig in-house (aligner is single-contig), multi-contig flagship documented as a command. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../plans/2026-06-01-phase-c-frontdoor.md | 445 ++++++++++++++++++ 1 file changed, 445 insertions(+) create mode 100644 docs/superpowers/plans/2026-06-01-phase-c-frontdoor.md diff --git a/docs/superpowers/plans/2026-06-01-phase-c-frontdoor.md b/docs/superpowers/plans/2026-06-01-phase-c-frontdoor.md new file mode 100644 index 0000000..9499a32 --- /dev/null +++ b/docs/superpowers/plans/2026-06-01-phase-c-frontdoor.md @@ -0,0 +1,445 @@ +# Move #4 — The Front Door Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans (inline, chosen for this work). Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make Rosalind's shipped breakthrough — the bounded-memory contract — discoverable: a forker's first five minutes (crate-root `use`, `cargo doc`, README, an example) land on the genomics product, not the √t theory layer or the non-bounded plugin path. + +**Architecture:** Pure positioning / docs / re-exports / examples — **no code-behavior changes.** Curate a genomics product surface at the crate root; rewrite `lib.rs`'s rustdoc to lead with the contract (with a runnable doctest) and demote √t to an honest research footer; add `CONTRACT.md`; rewrite the README to lead with the contract one-command story and route builders to the bounded `PileupColumn` substrate (plugin lineage demoted+labeled); add a substrate cookbook example; prove the runnable demo with a smoke test; reframe GitHub issue #3 (confirm-first). + +**Tech Stack:** Rust 1.72 (MSRV), `cargo test`/`fmt`/`build`/`doc`, Markdown. No new dependencies. Built on `rosalind/phase-c-frontdoor` (stacked on Phase C PR #21). + +**Spec:** [`docs/superpowers/specs/2026-06-01-phase-c-frontdoor-design.md`](../specs/2026-06-01-phase-c-frontdoor-design.md). + +**§3.5 refinement (flagged during planning):** the in-house aligner is single-contig, `--index` is BAM-only, and there is no SAM→BAM converter — so a *multi-contig* fixture cannot be produced out-of-the-box with Rosalind alone. The runnable bundled demo is therefore **single-contig, fully in-house** on the existing `examples/data/illumina_toy/` fixture; the README documents the **multi-contig flagship** as a command (bring-your-own-aligner BAM). No `generate_toy_data.py` change. + +--- + +## File Structure + +- **Modify** `src/lib.rs` — crate-root genomics re-exports; rustdoc rewrite (contract-first + runnable doctest + √t research footer). +- **Modify** `src/pileup/mod.rs` — fix the false "plugins build on" docline. +- **Create** `CONTRACT.md` (repo root) — the authoritative contract doc. +- **Modify** `README.md` — contract-first lead, Extend rewrite, honest brand, demo block, √t footer. +- **Create** `examples/custom_pileup_analytics.rs` — non-caller `PileupColumn`-iterator cookbook. +- **Create** `tests/frontdoor_demo.rs` — smoke test proving the README's in-house demo commands run. + +--- + +## Task 1: Crate-root genomics re-exports + fix the pileup docline + +**Files:** +- Modify: `src/lib.rs` (re-export block, lines ~56–61) +- Modify: `src/pileup/mod.rs` (module docstring) + +- [ ] **Step 1: Add the genomics product surface to the crate-root re-exports.** In `src/lib.rs`, the current block is: + +```rust +// Re-exports for convenience +pub use algebra::{AlgebraicEngine, FiniteField}; +pub use blocking::{BlockSummary, MovementLog}; +pub use ledger::StreamingLedger; +pub use machine::{Configuration, Move, State, Symbol, Transition, TuringMachine}; +pub use tree::{CompressedTree, TreeNode}; +``` + +Replace it with (genomics product surface first; the √t layer kept but regrouped under a research comment): + +```rust +// ── Genomics product surface — what builders compose on ─────────────────────── +// The bounded streaming substrate: +pub use pileup::{Obs, PileupColumn, PileupEngine, PileupParams, ReadSource, SliceSource}; +pub use io::bam::StreamingBamSource; +// The bounded whole-genome germline drive + calls: +pub use call::{ + call_germline_region_streaming, call_germline_whole_genome, GermlineCall, GermlineParams, +}; +// The memory contract (declare → plan → honor → verify): +pub use call::{estimate_variants_working_set, predicted_peak_rss_bytes}; +pub use core::{MemoryBudget, WorkingSet}; +// Build-once → mmap index + the reproducibility receipt: +pub use genomics::{GenomeIndex, IndexReader, ReferenceView}; +pub use provenance::RunManifest; + +// ── Research layer (√t space-bounded simulation; Phase D — see OPEN_PROBLEMS) ── +pub use algebra::{AlgebraicEngine, FiniteField}; +pub use blocking::{BlockSummary, MovementLog}; +pub use ledger::StreamingLedger; +pub use machine::{Configuration, Move, State, Symbol, Transition, TuringMachine}; +pub use tree::{CompressedTree, TreeNode}; +``` + +- [ ] **Step 2: Rewrite the crate-level rustdoc with a runnable doctest.** In `src/lib.rs`, replace the entire top doc block (lines 1–26, from `//! # O(√t) Space Simulation via Height Compression` through the end of the `//! ```` usage block) with: + +```rust +//! # Rosalind — a deterministic, low-memory genomics engine +//! +//! Call variants across a whole genome on a laptop, with memory you can **predict +//! and verify**, and results that are **byte-for-byte reproducible**. Rosalind +//! treats memory as a *contract*: you declare a RAM budget, [`rosalind plan`] tells +//! you up front whether the job fits, the run honors it (fits-or-refuses cleanly — +//! never a silent OOM-kill), and `rosalind verify` re-checks a BLAKE3 receipt +//! proving the realized peak landed inside your budget. +//! +//! The kernel is a streaming, CIGAR-aware **pileup column stream** bounded by local +//! coverage, not input size — a substrate you can compute arbitrary per-locus +//! analytics on. Variant calling is the first consumer, not the whole product. +//! +//! ``` +//! use std::sync::Arc; +//! use rosalind::{PileupEngine, PileupParams, SliceSource}; +//! use rosalind::core::{AlignedRead, CigarOp, CigarOpKind, Position, SamFlags}; +//! +//! // One 4bp read "ACGT" aligned at chr0:0 over the reference "ACGT". +//! let read = AlignedRead { +//! contig: 0, +//! pos: Position(0), +//! mapq: 60, +//! flags: SamFlags(0), +//! cigar: vec![CigarOp::new(CigarOpKind::Match, 4)], +//! seq: Arc::from(b"ACGT".to_vec().into_boxed_slice()), +//! qual: Arc::from(vec![40u8; 4].into_boxed_slice()), +//! }; +//! let reference: Arc<[u8]> = Arc::from(b"ACGT".to_vec().into_boxed_slice()); +//! +//! // The bounded pileup substrate: one PileupColumn per covered position. +//! let mut engine = +//! PileupEngine::new(SliceSource::new(vec![read]), reference, 0, 0..4, PileupParams::default()); +//! let first = engine.next().unwrap().unwrap(); +//! assert_eq!(first.depth(), 1); +//! ``` +//! +//! ## Research direction (Phase D) +//! +//! Rosalind is also a research vehicle for **space-bounded genomics**: a `~√t` +//! (square-root-space) evaluation framework (Williams 2025; Cook–Mertz 2024) as a +//! continuous space/time knob, aimed at **sublinear-space index construction**. +//! That layer is future work — not yet load-bearing — tracked in +//! [`docs/OPEN_PROBLEMS.md`](https://github.com/logannye/rosalind/blob/main/docs/OPEN_PROBLEMS.md). +``` + +(`[`rosalind plan`]` is intentionally plain text in prose — it renders as code; no intra-doc link is implied. Leave the `#![warn(missing_docs, …)]` attribute block that follows untouched.) + +- [ ] **Step 3: Fix the false pileup docline.** In `src/pileup/mod.rs`, the docstring says *"the single substrate that variant callers and plugins build on."* Replace that sentence: + +```rust +//! `PileupEngine` consumes coordinate-sorted reads and yields one `PileupColumn` +//! per covered reference position. It is the single bounded-memory substrate the +//! germline/somatic callers build on; build your own bounded per-locus analytics +//! over the same stream (see `examples/custom_pileup_analytics.rs`). The legacy +//! `GenomicPlugin`/`framework` lineage is separate and NOT memory-bounded. +//! Reference as `crate::pileup::…`. +``` + +- [ ] **Step 4: Build + doctest + verify the substrate surface resolves** + +Run: `cd ~/rosalind && cargo build 2>&1 | tail -3 && cargo test --doc 2>&1 | tail -8` +Expected: 0 warnings; the doc-test for `src/lib.rs` runs and passes (1 doctest). If `cargo doc` is desired: `cargo doc --no-deps` lands on the genomics intro. + +- [ ] **Step 5: Commit** + +```bash +cd ~/rosalind && git add src/lib.rs src/pileup/mod.rs && git commit -m "docs(lib): crate-root genomics surface + contract-first rustdoc (√t demoted) (Move #4)" +``` + +--- + +## Task 2: `CONTRACT.md` + +**Files:** +- Create: `CONTRACT.md` (repo root) + +- [ ] **Step 1: Write `CONTRACT.md`.** Author the file with these sections (concrete content, not placeholders): + + 1. **Title + one-line promise:** "Memory is a contract, not a hope." The honest brand line verbatim: *"Rosalind never silently OOM-kills you — it fits, or it tells you up front, and proves the realized peak with a receipt."* (Explicitly NOT "never refuses" — note graceful degrade/spill is Phase D.) + 2. **The four verbs**, each with a copy-pasteable command against the bundled fixture: + - **Declare** — `--memory-budget-mb N`. + - **Predict** — `rosalind plan --index g.idx --max-depth 1000 --max-read-len 250 --budget-mb 2048` → the `[FITS]`/`[REFUSE]` breakdown. + - **Honor** — `rosalind variants --index g.idx --alignments s.sorted.bam --memory-budget-mb 2048 --enforce -o s.vcf` → refuses up front (exit 3) if predicted > budget, fails loud (exit 4) if realized > budget, else completes; without `--enforce` it is record-only. + - **Verify** — `rosalind verify --manifest s.vcf.manifest.json` → re-hashes inputs/outputs + re-checks the recorded peak vs budget without re-running (exit 5 on mismatch). + 3. **What's bounded (honest scope):** the germline `variants --index` path (peak ≈ largest contig + capped active set, independent of BAM size); somatic is region-bounded; index *build* is O(reference) (Phase D); the engine is single-threaded (no thread-invariance claim). + 4. **Extend — build on the bounded substrate:** route to the `PileupColumn` iterator (`PileupEngine` over a `ReadSource`), pointing at `examples/custom_pileup_analytics.rs`; one short Rust snippet. Then a **Legacy / non-bounded** note: the `GenomicPlugin` trait, `src/framework/`, and the Python `run_rna_seq_plugin` demo still work but do **not** inherit the memory contract — prefer the substrate for bounded work. + 5. **Reproducibility:** the BLAKE3 canonical-JSON receipt; `verify` re-checks it; identical inputs → byte-identical VCF. + +- [ ] **Step 2: Commit** + +```bash +cd ~/rosalind && git add CONTRACT.md && git commit -m "docs: CONTRACT.md — the memory contract + bounded-substrate extension guide (Move #4)" +``` + +--- + +## Task 3: README rewrite + +**Files:** +- Modify: `README.md` + +- [ ] **Step 1: Rewrite the headline + first command block.** Keep the existing lead paragraph's spirit, but make the first runnable block the **contract loop** (declare → plan → enforce → verify) rather than just `variants`. Add, near the top "headline" section, the four-verb story: + +```bash +# Build a portable, mmap-able index of your reference — once. +rosalind index --reference genome.fa --output genome.idx + +# Will my whole-genome call fit in 2 GB? Ask before committing a byte. +rosalind plan --index genome.idx --max-depth 1000 --budget-mb 2048 + +# Call across all contigs from a coordinate-sorted BAM, honoring the budget. +rosalind variants --index genome.idx --alignments sample.sorted.bam \ + --memory-budget-mb 2048 --enforce -o sample.vcf + +# Re-check the receipt later — no re-run — to prove it fit and is reproducible. +rosalind verify --manifest sample.vcf.manifest.json +``` + +Add one sentence: link `CONTRACT.md` ("the full contract: [CONTRACT.md](CONTRACT.md)"). + +- [ ] **Step 2: Honest brand pass.** Search the README for "never refuses" / over-claims about memory and replace with the honest brand: *"never silently OOM-kills you — it fits, or it tells you up front."* (The current README §"Why it matters" / "memory as a contract" language is close; tighten any absolute claims. `--memory-budget-mb` is no longer only record-only — note `--enforce` makes it honored.) + +- [ ] **Step 3: Rewrite the "Extend" section.** Replace the current "Extend" bullets (which lead with `GenomicPlugin`) with substrate-first: + +```markdown +## Extend + +Rosalind's kernel is a **bounded, deterministic `PileupColumn` stream** — build +your own per-locus analytics (coverage, QC, methylation, ML features) over it and +inherit bounded memory + determinism for free: + +- **Rust** — consume the `PileupEngine` iterator over any `ReadSource`. See + [`examples/custom_pileup_analytics.rs`](examples/custom_pileup_analytics.rs) for a + non-caller consumer computing per-locus coverage. +- **CLI** — compose subcommands over pipes; `variants --index` is the first consumer. + +> **Legacy / non-bounded:** the `GenomicPlugin` trait (`src/plugin/`), the +> `framework/` evaluator, and the Python `run_rna_seq_plugin` demo still work but do +> **not** inherit the memory contract. Prefer the `PileupColumn` substrate for +> bounded work. +``` + +- [ ] **Step 4: Add the runnable in-house demo + document the multi-contig flagship.** In the "Use it" area, add a **runnable, single-contig, fully-in-house** demo on the bundled fixture, and clearly mark the multi-contig path as the production flagship: + +```markdown +### Try the contract end-to-end (bundled data, in-house tools only) + +```bash +D=examples/data/illumina_toy +rosalind index --reference $D/reference.fa --output /tmp/toy.idx +rosalind sort --input $D/alignments.bam --output /tmp/toy.sorted.bam +rosalind plan --index /tmp/toy.idx --budget-mb 512 +rosalind variants --index /tmp/toy.idx --alignments /tmp/toy.sorted.bam \ + --memory-budget-mb 512 --enforce -o /tmp/toy.vcf +rosalind verify --manifest /tmp/toy.vcf.manifest.json +``` + +(This bundled demo is **single-contig** because Rosalind's own aligner is +single-contig. For **whole-genome** calling, align with bwa-mem2/minimap2, sort, +and bring the coordinate-sorted BAM to `variants --index` — which calls *every* +contig in bounded memory.) +``` + +- [ ] **Step 5: Add the √t research footer.** Ensure the README's space-bounded/√t discussion is a clearly-labeled "Research direction (Phase D)" section near the end (not the lead), honest that it is future work, linking `docs/OPEN_PROBLEMS.md`. (The current "Why it matters" closing paragraph + roadmap already gesture at this — consolidate into one honest footer.) + +- [ ] **Step 6: Commit** + +```bash +cd ~/rosalind && git add README.md && git commit -m "docs(readme): lead with the contract; substrate-first Extend; honest brand; in-house demo (Move #4)" +``` + +--- + +## Task 4: Substrate cookbook example + +**Files:** +- Create: `examples/custom_pileup_analytics.rs` + +- [ ] **Step 1: Write the example.** Create `examples/custom_pileup_analytics.rs` — a non-caller consumer computing per-locus coverage + a simple low-MAPQ count directly over the `PileupColumn` iterator: + +```rust +//! Cookbook: build your own bounded, deterministic per-locus analytics over the +//! `PileupColumn` substrate — no variant calling. Run with: +//! cargo run --example custom_pileup_analytics + +use std::sync::Arc; + +use rosalind::core::{AlignedRead, CigarOp, CigarOpKind, Position, SamFlags}; +use rosalind::{PileupColumn, PileupEngine, PileupParams, SliceSource}; + +fn read(pos: u32, seq: &[u8]) -> AlignedRead { + AlignedRead { + contig: 0, + pos: Position(pos), + mapq: 60, + flags: SamFlags(0), + cigar: vec![CigarOp::new(CigarOpKind::Match, seq.len() as u32)], + seq: Arc::from(seq.to_vec().into_boxed_slice()), + qual: Arc::from(vec![40u8; seq.len()].into_boxed_slice()), + } +} + +fn main() { + let reference: Arc<[u8]> = Arc::from(b"ACGTACGTACGT".to_vec().into_boxed_slice()); + let reads = vec![read(0, b"ACGT"), read(2, b"GTAC"), read(4, b"ACGT")]; + + // The substrate: one PileupColumn per covered position, bounded by coverage. + let engine = PileupEngine::new( + SliceSource::new(reads), + Arc::clone(&reference), + 0, + 0..reference.len() as u32, + PileupParams::default(), + ); + + // A custom per-locus metric — here, depth — computed without any calling. + println!("pos\tref\tdepth"); + let mut total_depth = 0u64; + for column in engine { + let col: PileupColumn = column.expect("pileup column"); + total_depth += col.depth() as u64; + println!( + "{}\t{}\t{}", + col.locus.pos.0, + col.ref_base as char, + col.depth() + ); + } + println!("# total observed depth across covered positions: {total_depth}"); +} +``` + +- [ ] **Step 2: Run it** + +Run: `cd ~/rosalind && cargo run --example custom_pileup_analytics 2>&1 | tail -8` +Expected: a `pos\tref\tdepth` table (positions 0..11 with depths peaking where reads overlap) + the total line; exit 0. + +- [ ] **Step 3: Commit** + +```bash +cd ~/rosalind && git add examples/custom_pileup_analytics.rs && git commit -m "docs(example): custom_pileup_analytics — bounded per-locus analytics over the substrate (Move #4)" +``` + +--- + +## Task 5: Smoke test for the README in-house demo + +**Files:** +- Create: `tests/frontdoor_demo.rs` + +- [ ] **Step 1: Write the smoke test.** Mirrors the README's bundled-data demo through the CLI, proving the headline commands actually run (so the README cannot rot). Uses the bundled `examples/data/illumina_toy/` fixture; writes intermediates to a unique temp dir. + +```rust +//! Proves the README's in-house contract demo actually runs end-to-end on the +//! bundled single-contig fixture: index → sort → plan → variants --enforce → verify. + +use std::path::PathBuf; +use std::process::Command; +use std::sync::atomic::{AtomicU64, Ordering}; + +fn bin() -> &'static str { + env!("CARGO_BIN_EXE_rosalind") +} + +fn unique_dir() -> PathBuf { + static C: AtomicU64 = AtomicU64::new(0); + let n = C.fetch_add(1, Ordering::Relaxed); + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let d = std::env::temp_dir().join(format!("rosalind-frontdoor-{nanos}-{n}")); + std::fs::create_dir_all(&d).unwrap(); + d +} + +fn run(args: &[&str]) -> std::process::Output { + Command::new(bin()).args(args).output().expect("spawn rosalind") +} + +#[test] +fn readme_inhouse_contract_demo_runs_end_to_end() { + // CARGO_MANIFEST_DIR points at the crate root; the fixture is bundled there. + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let fa = root.join("examples/data/illumina_toy/reference.fa"); + let bam = root.join("examples/data/illumina_toy/alignments.bam"); + assert!(fa.exists(), "bundled reference missing: {}", fa.display()); + assert!(bam.exists(), "bundled alignments missing: {}", bam.display()); + + let dir = unique_dir(); + let idx = dir.join("toy.idx"); + let sorted = dir.join("toy.sorted.bam"); + let vcf = dir.join("toy.vcf"); + let manifest = dir.join("toy.vcf.manifest.json"); + + let out = run(&["index", "--reference", fa.to_str().unwrap(), "--output", idx.to_str().unwrap()]); + assert!(out.status.success(), "index: {}", String::from_utf8_lossy(&out.stderr)); + + let out = run(&["sort", "--input", bam.to_str().unwrap(), "--output", sorted.to_str().unwrap()]); + assert!(out.status.success(), "sort: {}", String::from_utf8_lossy(&out.stderr)); + + let out = run(&["plan", "--index", idx.to_str().unwrap(), "--budget-mb", "512"]); + assert!(out.status.success(), "plan: {}", String::from_utf8_lossy(&out.stderr)); + assert!(String::from_utf8_lossy(&out.stdout).contains("predicted peak")); + + let out = run(&[ + "variants", "--index", idx.to_str().unwrap(), + "--alignments", sorted.to_str().unwrap(), + "--memory-budget-mb", "512", "--enforce", + "-o", vcf.to_str().unwrap(), + ]); + assert!(out.status.success(), "variants --enforce: {}", String::from_utf8_lossy(&out.stderr)); + assert!(manifest.exists(), "receipt sidecar must be written"); + + let out = run(&["verify", "--manifest", manifest.to_str().unwrap()]); + assert!(out.status.success(), "verify: {}", String::from_utf8_lossy(&out.stderr)); + assert!(String::from_utf8_lossy(&out.stdout).contains("verify: OK")); + + std::fs::remove_dir_all(&dir).ok(); +} +``` + +- [ ] **Step 2: Run it.** (If the bundled `alignments.bam` turns out unsorted/incompatible with the `--index` monotonicity guard, the `sort` step normalizes it; if `variants` still rejects it, regenerate the fixture's BAM via the existing `examples/data/illumina_toy` pipeline and re-commit — but the bundled BAM is expected to work post-`sort`.) + +Run: `cd ~/rosalind && cargo test --test frontdoor_demo 2>&1 | tail -15` +Expected: PASS. + +- [ ] **Step 3: Commit** + +```bash +cd ~/rosalind && git add tests/frontdoor_demo.rs && git commit -m "test: smoke-test the README in-house contract demo end-to-end (Move #4)" +``` + +--- + +## Task 6: Full verification + +**Files:** none (verification only) + +- [ ] **Step 1: Format + zero-warning builds** + +Run: `cd ~/rosalind && cargo fmt --all && cargo fmt --all -- --check && echo FMT_CLEAN && cargo build 2>&1 | grep -c warning; cargo build --release 2>&1 | grep -c warning` +Expected: `FMT_CLEAN`; `0` and `0`. + +- [ ] **Step 2: Doc + example + full suite** + +Run: `cd ~/rosalind && cargo test --doc 2>&1 | tail -5 && cargo run --example custom_pileup_analytics >/dev/null 2>&1 && echo EXAMPLE_OK && cargo test 2>&1 | grep -E "FAILED|panicked|[1-9][0-9]* failed" || echo no failures; cargo test 2>&1 | grep -cE "test result: ok\."` +Expected: doctest passes; `EXAMPLE_OK`; `no failures`; `ok.` count ≥ prior + 1 (new `frontdoor_demo` binary). + +- [ ] **Step 3: Commit any fmt fixups** + +```bash +cd ~/rosalind && git add -A && git commit -m "style: rustfmt fixups (Move #4)" || true +``` + +--- + +## Task 7: issue #3 reframe (GitHub — confirm-first, outward-facing) + +**Files:** none (GitHub issue edit) + +- [ ] **Step 1: Draft the new issue body** (contract-first thesis; √t as the future Phase-D knob; retire "separate out the theory layer"). Present the full draft to the user. +- [ ] **Step 2: On explicit go-ahead only**, post it: `gh issue edit 3 -R logannye/rosalind --body-file `. Do **not** edit the public issue without confirmation. + +--- + +## Self-Review notes + +- **Spec coverage:** §3.1 re-exports → Task 1; §3.2 lib.rs rustdoc + pileup docline → Task 1; §3.3 CONTRACT.md → Task 2; §3.4 README → Task 3; §3.5 demo → **refined** (single-contig in-house demo in README Task 3 Step 4 + smoke test Task 5; the multi-contig generator change is dropped, with the constraint documented up top); §3.6 cookbook → Task 4; §3.7 issue #3 → Task 7 (confirm-first); §4 testing → Tasks 1/4/5/6. +- **Type consistency:** the re-export paths (Task 1) match the verified module exports (`pileup::{Obs,PileupColumn,PileupEngine,PileupParams,ReadSource,SliceSource}`, `io::bam::StreamingBamSource`, `call::{…}`, `core::{MemoryBudget,WorkingSet}`, `genomics::{GenomeIndex,IndexReader,ReferenceView}`, `provenance::RunManifest`). The doctest + example use `rosalind::{PileupEngine,PileupParams,SliceSource}` (now crate-root) + `rosalind::core::{AlignedRead,CigarOp,CigarOpKind,Position,SamFlags}` (the `AlignedRead` 7-field literal matches `src/pileup/engine.rs` test usage). `PileupColumn::{depth(), locus, ref_base}` are its real accessors/fields. +- **No behavior change:** re-exports + docs + a new example + a new test only; no edits to engine/call/cli logic. From 4260e296d86809b6d5185b2b9f1ad87ec6496de7 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 18:19:46 -0700 Subject: [PATCH 27/32] =?UTF-8?q?docs(lib):=20crate-root=20genomics=20surf?= =?UTF-8?q?ace=20+=20contract-first=20rustdoc=20(=E2=88=9At=20demoted)=20(?= =?UTF-8?q?Move=20#4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/lib.rs | 73 ++++++++++++++++++++++++++++++++++------------- src/pileup/mod.rs | 7 +++-- 2 files changed, 58 insertions(+), 22 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index ac71bf1..1847e55 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,29 +1,47 @@ -//! # O(√t) Space Simulation via Height Compression +//! # Rosalind — a deterministic, low-memory genomics engine //! -//! This library implements the breakthrough algorithm for simulating -//! time-t Turing machine computations in O(√t) space. +//! Call variants across a whole genome on a laptop, with memory you can **predict +//! and verify**, and results that are **byte-for-byte reproducible**. Rosalind +//! treats memory as a *contract*: you declare a RAM budget, `rosalind plan` tells +//! you up front whether the job fits, the run honors it (fits-or-refuses cleanly — +//! never a silent OOM-kill), and `rosalind verify` re-checks a BLAKE3 receipt +//! proving the realized peak landed inside your budget. //! -//! ## Core Algorithm +//! The kernel is a streaming, CIGAR-aware **pileup column stream** bounded by local +//! coverage, not input size — a substrate you can compute arbitrary per-locus +//! analytics on. Variant calling is the first consumer, not the whole product. //! -//! 1. **Block-respecting simulation**: Partition computation into T = ⌈t/b⌉ blocks -//! 2. **Height compression**: Transform tree from height Θ(T) → O(log T) -//! 3. **Pointerless evaluation**: O(1) bits per level instead of O(log b) -//! 4. **Streaming ledger**: Track T merges with constant tokens +//! ``` +//! use std::sync::Arc; +//! use rosalind::{PileupEngine, PileupParams, SliceSource}; +//! use rosalind::core::{AlignedRead, CigarOp, CigarOpKind, Position, SamFlags}; //! -//! Result: Space = O(b + T + log T) = O(b + t/b), optimal at b = √t. A rolling -//! boundary is maintained (only the latest block summary), so cached leaf data -//! never accumulates and total memory stays within O(√t). +//! // One 4bp read "ACGT" aligned at chr0:0 over the reference "ACGT". +//! let read = AlignedRead { +//! contig: 0, +//! pos: Position(0), +//! mapq: 60, +//! flags: SamFlags(0), +//! cigar: vec![CigarOp::new(CigarOpKind::Match, 4)], +//! seq: Arc::from(b"ACGT".to_vec().into_boxed_slice()), +//! qual: Arc::from(vec![40u8; 4].into_boxed_slice()), +//! }; +//! let reference: Arc<[u8]> = Arc::from(b"ACGT".to_vec().into_boxed_slice()); //! -//! ## Usage Example +//! // The bounded pileup substrate: one PileupColumn per covered position. +//! let mut engine = +//! PileupEngine::new(SliceSource::new(vec![read]), reference, 0, 0..4, PileupParams::default()); +//! let first = engine.next().unwrap().unwrap(); +//! assert_eq!(first.depth(), 1); +//! ``` //! -//! ```text -//! use rosalind::{TuringMachine, Simulator, SimulationConfig}; +//! ## Research direction (Phase D) //! -//! let config = SimulationConfig::optimal_for_time(10_000); -//! let mut sim = Simulator::new(machine, config); -//! let result = sim.run(&input)?; -//! assert!(result.space_used <= O(√10_000)); -//! ``` +//! Rosalind is also a research vehicle for **space-bounded genomics**: a `~√t` +//! (square-root-space) evaluation framework (Williams 2025; Cook–Mertz 2024) as a +//! continuous space/time knob, aimed at **sublinear-space index construction**. +//! That layer is future work — not yet load-bearing — tracked in +//! `docs/OPEN_PROBLEMS.md`. #![warn(missing_docs, missing_debug_implementations)] #![allow(clippy::new_without_default)] @@ -53,7 +71,22 @@ pub mod space; // Space accounting utilities pub mod tree; // Height-compressed evaluation tree pub mod util; // Helper functions -// Re-exports for convenience +// ── Genomics product surface — what builders compose on ─────────────────────── +// The bounded streaming substrate: +pub use io::bam::StreamingBamSource; +pub use pileup::{Obs, PileupColumn, PileupEngine, PileupParams, ReadSource, SliceSource}; +// The bounded whole-genome germline drive + calls: +pub use call::{ + call_germline_region_streaming, call_germline_whole_genome, GermlineCall, GermlineParams, +}; +// The memory contract (declare → plan → honor → verify): +pub use call::{estimate_variants_working_set, predicted_peak_rss_bytes}; +pub use core::{MemoryBudget, WorkingSet}; +// Build-once → mmap index + the reproducibility receipt: +pub use genomics::{GenomeIndex, IndexReader, ReferenceView}; +pub use provenance::RunManifest; + +// ── Research layer (√t space-bounded simulation; Phase D — see OPEN_PROBLEMS) ── pub use algebra::{AlgebraicEngine, FiniteField}; pub use blocking::{BlockSummary, MovementLog}; pub use ledger::StreamingLedger; diff --git a/src/pileup/mod.rs b/src/pileup/mod.rs index 3dafedc..470d8ab 100644 --- a/src/pileup/mod.rs +++ b/src/pileup/mod.rs @@ -1,8 +1,11 @@ //! The streaming pileup kernel. //! //! `PileupEngine` consumes coordinate-sorted reads and yields one `PileupColumn` -//! per covered reference position. It is the single substrate that variant -//! callers and plugins build on. Reference as `crate::pileup::…`. +//! per covered reference position. It is the single bounded-memory substrate the +//! germline/somatic callers build on; build your own bounded per-locus analytics +//! over the same stream (see `examples/custom_pileup_analytics.rs`). The legacy +//! `GenomicPlugin`/`framework` lineage is separate and NOT memory-bounded. +//! Reference as `crate::pileup::…`. pub mod column; pub mod engine; From f5cbcefdb24cd32bbe1365696a6d5868f7f557a1 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 18:20:24 -0700 Subject: [PATCH 28/32] =?UTF-8?q?docs:=20CONTRACT.md=20=E2=80=94=20the=20m?= =?UTF-8?q?emory=20contract=20+=20bounded-substrate=20extension=20guide=20?= =?UTF-8?q?(Move=20#4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CONTRACT.md | 107 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 CONTRACT.md diff --git a/CONTRACT.md b/CONTRACT.md new file mode 100644 index 0000000..9003b54 --- /dev/null +++ b/CONTRACT.md @@ -0,0 +1,107 @@ +# The memory contract + +**Memory is a contract, not a hope.** + +Every other variant caller treats RAM as an emergent property you guess at (`-Xmx…`, +`--target-mem` "heuristics may not work well") and then crash on. Rosalind treats it as a contract: + +> **Rosalind never silently OOM-kills you — it fits, or it tells you up front, and it proves the +> realized peak with a receipt.** + +(It does *not* claim to "never refuse": when a budget is genuinely too small the run declines cleanly +rather than crashing. Graceful degrade-don't-die — sliding down a space/time curve to finish anyway — is +the Phase-D research direction, not a present claim.) + +## The four verbs + +The contract applies to the bounded whole-genome germline path, `rosalind variants --index`. + +### 1. Declare + +State the RAM you have. `--memory-budget-mb N` on `variants`, `--budget-mb N` on `plan`/`verify`. + +### 2. Predict — `rosalind plan` + +Ask *before committing a byte* whether the job fits. `plan` reads only the index header (plus your +declared depth/read-length assumptions) — it never opens the BAM: + +```bash +rosalind plan --index genome.idx --max-depth 1000 --max-read-len 250 --budget-mb 2048 +``` + +It prints a breakdown — reference decode + active set @ max-depth + engine overhead, atop a measured +process baseline — and a verdict: `[FITS]` or `[REFUSE]`. + +### 3. Honor — `rosalind variants … --enforce` + +```bash +rosalind variants --index genome.idx --alignments sample.sorted.bam \ + --memory-budget-mb 2048 --enforce -o sample.vcf +``` + +With `--enforce`: + +- **predicted peak > budget → refuse up front** (exit **3**), before doing any work, with an actionable + message (raise the budget, lower `--max-depth`, or drop `--enforce`); +- **realized peak > budget → fail loud** (exit **4**) *after* writing the VCF + receipt (you keep the data + and the proof it overran) — never a silent overrun; +- otherwise the run completes within budget. + +Without `--enforce`, the budget is **record-only**: the run always completes and the verdict is recorded in +the receipt. The active read set is capped at `--max-depth` (default 1000; `0` = uncapped) — deterministic +downsampling that bounds the working set; output changes only at sites deeper than the cap. + +### 4. Verify — `rosalind verify` + +Re-check a receipt *without re-running*: + +```bash +rosalind verify --manifest sample.vcf.manifest.json +``` + +It re-hashes the recorded inputs and outputs (BLAKE3) and re-checks the recorded peak against the budget +(supplied via `--budget-mb`, or read from the manifest). Exit **0** if everything matches and fits; +non-zero (exit **5**) with a per-check report on any drift, missing file, or over-budget peak. This is the +auditability story containers can't give you for a non-deterministic caller. + +## What's bounded (honest scope) + +- **Germline `variants --index`** is the bounded path: peak ≈ the largest contig's reference + the + depth-capped active set, **independent of BAM size**. Reads stream one record at a time. +- **Somatic** (`somatic`) is **region-bounded**, not whole-genome-bounded (it collects both pileup streams + for the region). +- **Index *build*** (`rosalind index`) is **O(reference)** in RAM today; `plan --reference` reports an + advisory estimate. Sublinear-space construction is the Phase-D research direction (see + `docs/OPEN_PROBLEMS.md`). +- The engine is **single-threaded** — outputs are deterministic, but there is no thread-invariance claim + yet. + +## Extend — build on the bounded substrate + +Rosalind's kernel is a bounded, deterministic **`PileupColumn` stream**. Compute your own per-locus +analytics over it (coverage, QC, methylation, ML features) and inherit bounded memory + determinism for +free — no variant calling required: + +```rust +use rosalind::{PileupEngine, PileupParams}; +// PileupEngine is an Iterator>. +// Each PileupColumn carries the locus, ref_base, depth(), allele_counts(), strand_counts(). +for column in PileupEngine::new(source, reference, contig, region, PileupParams::default()) { + let col = column?; + // your bounded per-locus metric here +} +``` + +A complete, runnable example: [`examples/custom_pileup_analytics.rs`](examples/custom_pileup_analytics.rs) +(`cargo run --example custom_pileup_analytics`). + +> **Legacy / non-bounded.** The `GenomicPlugin` trait (`src/plugin/`), the `framework/` evaluator, and the +> Python `run_rna_seq_plugin` demo still work but do **not** inherit the memory contract. Prefer the +> `PileupColumn` substrate above for bounded work. + +## Reproducibility + +Every receipt is canonical JSON (sorted keys, no timestamps) with BLAKE3 content hashes of the index, the +alignments, and the output VCF, plus the realized `peak_rss_bytes` / `max_working_set_bytes` and the +contract params (`memory_budget_mb`, `contract_verdict`, `enforced`, `max_depth`, `max_read_len`). Identical +inputs produce a byte-identical VCF and a byte-identical manifest — and `rosalind verify` proves it. From 7267f6fd34efd75b59db02f9d2ac986d5549ac64 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 18:23:21 -0700 Subject: [PATCH 29/32] docs(readme): lead with the contract; substrate-first Extend; honest brand; in-house demo (Move #4) --- README.md | 57 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 7888268..3664231 100644 --- a/README.md +++ b/README.md @@ -19,22 +19,29 @@ rosalind index --reference genome.fa --output genome.idx # 2. (Align reads with your favorite aligner and coordinate-sort the BAM.) # `rosalind sort` will do the sort deterministically within a memory budget. -# 3. Call germline variants across the WHOLE genome, streaming, bounded. +# 3. Will it fit in 4 GB? Ask before committing a byte. +rosalind plan --index genome.idx --max-depth 1000 --budget-mb 4096 + +# 4. Call germline variants across the WHOLE genome, streaming, honoring the budget. rosalind variants \ --index genome.idx \ --alignments sample.sorted.bam \ - --memory-budget-mb 4096 \ + --memory-budget-mb 4096 --enforce \ -o sample.vcf # ...writes a multi-contig VCF, plus to stderr: # memory: peak RSS 412 MiB; max pileup working set 18 KiB +# contract: OK — realized peak 412 MiB within declared 4096 MiB # wrote reproducibility receipt: sample.vcf.manifest.json + +# 5. Re-check the receipt later — no re-run — to prove it fit and is reproducible. +rosalind verify --manifest sample.vcf.manifest.json ``` What makes this different: - **Bounded memory, independent of BAM size.** Reads stream one record at a time; peak memory is roughly *the largest contig's reference + the local pileup working set* — not the size of your alignments. A human genome calls comfortably on a laptop. - **Self-contained.** The reference comes from the `.idx`; you don't need the original FASTA at call time. -- **A memory receipt.** Every run reports its realized peak RSS and max pileup working set — to stderr and into a reproducibility manifest. `--memory-budget-mb` flags a run that exceeds your declared budget *(it records the verdict; it does not yet abort — enforcement is on the roadmap)*. +- **A contract, honored.** `rosalind plan` predicts the peak *before you commit a byte*; `--enforce` honors the budget — refusing up front (exit 3) or failing loud (exit 4) rather than silently OOM-killing you; `rosalind verify` re-checks the receipt without re-running. Without `--enforce`, the budget is record-only. The full story: [the memory contract](CONTRACT.md). - **Reproducible + auditable.** Identical inputs produce a byte-identical VCF; a BLAKE3 manifest records the index, the BAM, the output, and the memory used. --- @@ -48,7 +55,7 @@ What makes this different: - **Deterministic coordinate sort** — `rosalind sort`: an external merge sort (spills to disk) that orders a BAM by position within a configurable memory budget. - **Somatic (tumor/normal) calling** — `rosalind somatic` calls somatic SNVs and simple indels from a paired tumor/normal BAM set using a deterministic binomial log-likelihood-ratio model with explicit depth and allele-fraction filters. - **Truth-set evaluation** — `rosalind eval-somatic` compares a call set against a truth VCF over confident regions (BED), with variant normalization (left-align + trim) and precision / recall / F1. -- **Extensibility** — Implement the `GenomicPlugin` trait to run custom per-block analyses on the same bounded-memory evaluator, or call the PyO3 bindings from Python. +- **Extensibility** — Build custom bounded per-locus analytics over the `PileupColumn` iterator substrate (see [`examples/custom_pileup_analytics.rs`](examples/custom_pileup_analytics.rs)), inheriting bounded memory + determinism for free. *(The legacy `GenomicPlugin` trait + PyO3 RNA-seq demo still work but are **not** memory-bounded — see [CONTRACT.md](CONTRACT.md).)* - **Determinism by design** — Primary artifacts are emitted in a canonical, stable order, byte-for-byte identical across repeated runs given identical inputs. See [`docs/determinism.md`](docs/determinism.md). ## Why it matters @@ -59,7 +66,7 @@ Three properties, treated as first-class guarantees rather than nice-to-haves: 2. **Reproducibility.** Byte-identical outputs and a per-run BLAKE3 manifest make results auditable — a hard requirement for clinical and regulated pipelines, and a sanity-saver for everyone else. 3. **Honest uncertainty.** Calibrated, abstention-aware calling refuses to emit a call where the evidence is insufficient, instead of papering over it. -Under the hood, Rosalind is also a research vehicle for **space-bounded genomics**: a `~√t` (square-root-space) evaluation framework as a continuous space/time knob — trade time for memory along a curve a declared budget selects. That direction (sublinear-space index *construction*, budget *enforcement*, `rosalind plan`/`verify`) is on the roadmap below; the bounded streaming engine you can use today is the practical foundation it builds on. +The contract is real today: `rosalind plan` predicts before you commit, `--enforce` honors the budget, and `rosalind verify` re-checks the receipt (see [CONTRACT.md](CONTRACT.md)). Under the hood, Rosalind is *also* a research vehicle for **space-bounded genomics**: a `~√t` (square-root-space) evaluation framework as a continuous space/time knob, aimed at **sublinear-space index *construction*** — the future Phase-D direction that would extend the contract to the index build step. That layer is not yet load-bearing; the bounded streaming engine you use today is the practical foundation it builds on. ## Who it's for @@ -73,7 +80,7 @@ Under the hood, Rosalind is also a research vehicle for **space-bounded genomics - **Whole-genome:** germline variant calling via `rosalind variants --index` (all contigs, streaming, bounded) and exact-match lookup via `rosalind index` / `rosalind locate`. - **Single-contig:** Rosalind's own **aligner** (`rosalind align`) and the FASTA-based `variants --reference` path operate on one reference contig per run. For whole-genome calling, align with any standard aligner and bring the coordinate-sorted BAM to `variants --index`. (Wiring the *aligner* onto the persisted multi-contig index is a later phase — see the roadmap.) - Variant calling is **single-sample** (germline) or a **tumor/normal pair** (somatic); calling is SNV-focused, with simple indels in the somatic path. -- The engine runs **single-threaded** today. `--memory-budget-mb` is **record-only** (it reports a verdict but does not yet enforce). +- The engine runs **single-threaded** today. `--memory-budget-mb` is record-only by default; add `--enforce` to honor it (refuse up front / fail loud — see [CONTRACT.md](CONTRACT.md)). ## Roadmap @@ -81,8 +88,9 @@ The core primitive is a streaming, CIGAR-aware pileup column stream; variant cal - **Phase A (done):** the streaming pileup engine; calibrated, abstention-aware germline SNV calling; tumor/normal somatic calling; spec-valid VCF; a BLAKE3 reproducibility receipt per run. - **Phase B (done):** streaming gzip/bgzf input; a multi-contig FM-index over the concatenated genome with `(contig, position)` resolution; a build-once, memory-mapped, byte-reproducible persisted index (`rosalind index`/`locate`); zero-copy reference access from the index; and **bounded whole-genome germline calling over a sorted BAM** (`rosalind variants --index`) with a realized-memory receipt. -- **Phase C (next):** memory as an *enforceable* contract — `rosalind plan` (a checkable memory envelope before you commit), budget **enforcement** with graceful degradation (never OOM on a real device), and `rosalind verify`. -- **Later:** sublinear-space index construction (the `~√t` space/time knob across the full curve); the aligner over the persisted multi-contig index (`align --index`, whole-genome alignment); germline indels and richer read QC; deterministic multithreading; a Python binding over the pileup stream. +- **Phase C (done — in review):** memory as a *verifiable contract* — `rosalind plan` (a checkable envelope before you commit), `--enforce` (honor-or-refuse: refuse up front / fail loud, never a silent OOM-kill), and `rosalind verify`. See [CONTRACT.md](CONTRACT.md). +- **Phase D (research):** sublinear-space index construction — the `~√t` space/time knob across the full curve — extending the contract to the index *build* step (today's build is O(reference)). The headline space-complexity bet; see [`docs/OPEN_PROBLEMS.md`](docs/OPEN_PROBLEMS.md). +- **Later:** the aligner over the persisted multi-contig index (`align --index`, whole-genome alignment); germline indels and richer read QC; deterministic multithreading; a Python/tensor binding over the pileup stream. Target architecture and per-phase specs/plans live in [`docs/superpowers/specs/`](docs/superpowers/specs/) and [`docs/superpowers/plans/`](docs/superpowers/plans/); the guiding thesis is in [`docs/OPEN_PROBLEMS.md`](docs/OPEN_PROBLEMS.md). @@ -125,16 +133,32 @@ python scripts/generate_toy_data.py examples/data/illumina_toy # Build the index once. rosalind index --reference genome.fa --output genome.idx -# Call across all contigs from a coordinate-sorted BAM, in bounded memory. +# Predict the peak before committing; then call all contigs, honoring the budget. +rosalind plan --index genome.idx --max-depth 1000 --budget-mb 4096 rosalind variants \ --index genome.idx \ --alignments sample.sorted.bam \ --mapq-threshold 20 \ - --memory-budget-mb 4096 \ + --memory-budget-mb 4096 --enforce \ -o sample.vcf +rosalind verify --manifest sample.vcf.manifest.json ``` -`variants --index` requires a **coordinate-sorted BAM** (use `rosalind sort` or `samtools sort`). It reads the reference from the index — no `--reference` FASTA needed — and writes a multi-contig VCF plus a memory + reproducibility receipt. `--memory-budget-mb` records (does not yet enforce) a verdict against the realized peak. +`variants --index` requires a **coordinate-sorted BAM** (use `rosalind sort` or `samtools sort`). It reads the reference from the index — no `--reference` FASTA needed — and writes a multi-contig VCF plus a memory + reproducibility receipt. With `--enforce` the declared budget is honored (refuse up front / fail loud); without it, it is record-only. The full contract: [CONTRACT.md](CONTRACT.md). + +### Try the contract end-to-end (bundled data, in-house tools only) + +```bash +D=examples/data/illumina_toy +rosalind index --reference $D/reference.fa --output /tmp/toy.idx +rosalind sort --input $D/alignments.bam --output /tmp/toy.sorted.bam +rosalind plan --index /tmp/toy.idx --budget-mb 512 +rosalind variants --index /tmp/toy.idx --alignments /tmp/toy.sorted.bam \ + --memory-budget-mb 512 --enforce -o /tmp/toy.vcf +rosalind verify --manifest /tmp/toy.vcf.manifest.json +``` + +This bundled demo is **single-contig** because Rosalind's own aligner is single-contig. For **whole-genome** calling, align with bwa-mem2/minimap2, coordinate-sort, and bring the BAM to `variants --index` — which calls *every* contig in bounded memory. ### Single-contig alignment + calling @@ -154,6 +178,8 @@ Inputs may be plain or gzip/bgzf-compressed (auto-detected); pass `-` to read FA Run `rosalind --help` for exact flags. +- `rosalind plan` — predict a job's peak memory vs a declared budget *before* committing (`--index` for the variants peak, `--reference` for the index build). +- `rosalind verify` — re-check a reproducibility receipt without re-running: re-hash its inputs/outputs and confirm the realized peak landed within budget. - `rosalind locate --index genome.idx --pattern GATTACA` — exact-match positions in a prebuilt index (memory-mapped, never rebuilt). Exact-match only; seed/chain/extend alignment against the persisted index is a later phase. - `rosalind sort` — deterministic coordinate sort of a BAM within a memory budget. - `rosalind somatic` — tumor/normal somatic SNV + simple-indel calling from a paired BAM set over a region. @@ -239,9 +265,12 @@ depth = engine.run_rna_seq_plugin( ## Extend -- **Rust plugins** — implement `GenomicPlugin` (see `src/plugin/examples.rs`) to run custom per-block analyses (coverage, QC counts, domain-specific summaries) on the same bounded-memory evaluator. -- **CLI subcommands** — add workflows in `src/main.rs`. -- **Python** — drive the engine from `rosalind_py.PyGenomicEngine` alongside pandas / NumPy / scikit-learn. +Rosalind's kernel is a **bounded, deterministic `PileupColumn` stream** — build your own per-locus analytics (coverage, QC, methylation, ML features) over it and inherit bounded memory + determinism for free: + +- **Rust (recommended)** — consume the `PileupEngine` iterator over any `ReadSource`. See [`examples/custom_pileup_analytics.rs`](examples/custom_pileup_analytics.rs) (`cargo run --example custom_pileup_analytics`) for a non-caller consumer computing per-locus coverage. The contract verbs and the substrate are re-exported at the crate root (`use rosalind::{PileupEngine, PileupColumn, ReadSource, …}`). +- **CLI subcommands** — add workflows in `src/main.rs`; compose subcommands over pipes. + +> **Legacy / non-bounded.** The `GenomicPlugin` trait (`src/plugin/`), the `framework/` evaluator, and the Python `rosalind_py.PyGenomicEngine` RNA-seq demo still work but do **not** inherit the memory contract. Prefer the `PileupColumn` substrate above for bounded work. See [CONTRACT.md](CONTRACT.md). --- From 78715745c7bfd8cf9b9d3c3db2572683e91ab536 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 18:23:55 -0700 Subject: [PATCH 30/32] =?UTF-8?q?docs(example):=20custom=5Fpileup=5Fanalyt?= =?UTF-8?q?ics=20=E2=80=94=20bounded=20per-locus=20analytics=20over=20the?= =?UTF-8?q?=20substrate=20(Move=20#4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/custom_pileup_analytics.rs | 45 +++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 examples/custom_pileup_analytics.rs diff --git a/examples/custom_pileup_analytics.rs b/examples/custom_pileup_analytics.rs new file mode 100644 index 0000000..406b062 --- /dev/null +++ b/examples/custom_pileup_analytics.rs @@ -0,0 +1,45 @@ +//! Cookbook: build your own bounded, deterministic per-locus analytics over the +//! `PileupColumn` substrate — no variant calling. Run with: +//! cargo run --example custom_pileup_analytics + +use std::sync::Arc; + +use rosalind::core::{AlignedRead, CigarOp, CigarOpKind, Position, SamFlags}; +use rosalind::{PileupColumn, PileupEngine, PileupParams, SliceSource}; + +fn read(pos: u32, seq: &[u8]) -> AlignedRead { + AlignedRead { + contig: 0, + pos: Position(pos), + mapq: 60, + flags: SamFlags(0), + cigar: vec![CigarOp::new(CigarOpKind::Match, seq.len() as u32)], + seq: Arc::from(seq.to_vec().into_boxed_slice()), + qual: Arc::from(vec![40u8; seq.len()].into_boxed_slice()), + } +} + +fn main() { + let reference: Arc<[u8]> = Arc::from(b"ACGTACGTACGT".to_vec().into_boxed_slice()); + let reads = vec![read(0, b"ACGT"), read(2, b"GTAC"), read(4, b"ACGT")]; + + // The substrate: one PileupColumn per covered position, bounded by coverage — + // not by input size. PileupEngine is an Iterator. + let engine = PileupEngine::new( + SliceSource::new(reads), + Arc::clone(&reference), + 0, + 0..reference.len() as u32, + PileupParams::default(), + ); + + // A custom per-locus metric — here, depth — computed without any calling. + println!("pos\tref\tdepth"); + let mut total_depth = 0u64; + for column in engine { + let col: PileupColumn = column.expect("pileup column"); + total_depth += col.depth() as u64; + println!("{}\t{}\t{}", col.locus.pos.0, col.ref_base as char, col.depth()); + } + println!("# total observed depth across covered positions: {total_depth}"); +} From ee3448cafa6b7dc47ab8ab90133909a3921b093d Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 18:24:41 -0700 Subject: [PATCH 31/32] test: smoke-test the README in-house contract demo end-to-end (Move #4) --- tests/frontdoor_demo.rs | 92 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 tests/frontdoor_demo.rs diff --git a/tests/frontdoor_demo.rs b/tests/frontdoor_demo.rs new file mode 100644 index 0000000..2e0caba --- /dev/null +++ b/tests/frontdoor_demo.rs @@ -0,0 +1,92 @@ +//! Proves the README's in-house contract demo actually runs end-to-end on the +//! bundled single-contig fixture: index → sort → plan → variants --enforce → verify. + +use std::path::PathBuf; +use std::process::Command; +use std::sync::atomic::{AtomicU64, Ordering}; + +fn bin() -> &'static str { + env!("CARGO_BIN_EXE_rosalind") +} + +fn unique_dir() -> PathBuf { + static C: AtomicU64 = AtomicU64::new(0); + let n = C.fetch_add(1, Ordering::Relaxed); + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let d = std::env::temp_dir().join(format!("rosalind-frontdoor-{nanos}-{n}")); + std::fs::create_dir_all(&d).unwrap(); + d +} + +fn run(args: &[&str]) -> std::process::Output { + Command::new(bin()) + .args(args) + .output() + .expect("spawn rosalind") +} + +#[test] +fn readme_inhouse_contract_demo_runs_end_to_end() { + // CARGO_MANIFEST_DIR points at the crate root; the fixture is bundled there. + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let fa = root.join("examples/data/illumina_toy/reference.fa"); + let bam = root.join("examples/data/illumina_toy/alignments.bam"); + assert!(fa.exists(), "bundled reference missing: {}", fa.display()); + assert!(bam.exists(), "bundled alignments missing: {}", bam.display()); + + let dir = unique_dir(); + let idx = dir.join("toy.idx"); + let sorted = dir.join("toy.sorted.bam"); + let vcf = dir.join("toy.vcf"); + let manifest = dir.join("toy.vcf.manifest.json"); + + let out = run(&[ + "index", + "--reference", + fa.to_str().unwrap(), + "--output", + idx.to_str().unwrap(), + ]); + assert!(out.status.success(), "index: {}", String::from_utf8_lossy(&out.stderr)); + + let out = run(&[ + "sort", + "--input", + bam.to_str().unwrap(), + "--output", + sorted.to_str().unwrap(), + ]); + assert!(out.status.success(), "sort: {}", String::from_utf8_lossy(&out.stderr)); + + let out = run(&["plan", "--index", idx.to_str().unwrap(), "--budget-mb", "512"]); + assert!(out.status.success(), "plan: {}", String::from_utf8_lossy(&out.stderr)); + assert!(String::from_utf8_lossy(&out.stdout).contains("predicted peak")); + + let out = run(&[ + "variants", + "--index", + idx.to_str().unwrap(), + "--alignments", + sorted.to_str().unwrap(), + "--memory-budget-mb", + "512", + "--enforce", + "-o", + vcf.to_str().unwrap(), + ]); + assert!( + out.status.success(), + "variants --enforce: {}", + String::from_utf8_lossy(&out.stderr) + ); + assert!(manifest.exists(), "receipt sidecar must be written"); + + let out = run(&["verify", "--manifest", manifest.to_str().unwrap()]); + assert!(out.status.success(), "verify: {}", String::from_utf8_lossy(&out.stderr)); + assert!(String::from_utf8_lossy(&out.stdout).contains("verify: OK")); + + std::fs::remove_dir_all(&dir).ok(); +} From fe594b91ceb5e246935b2b226733befae099c393 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 18:25:38 -0700 Subject: [PATCH 32/32] style: rustfmt fixups (Move #4) --- examples/custom_pileup_analytics.rs | 7 +++++- tests/frontdoor_demo.rs | 38 ++++++++++++++++++++++++----- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/examples/custom_pileup_analytics.rs b/examples/custom_pileup_analytics.rs index 406b062..e183113 100644 --- a/examples/custom_pileup_analytics.rs +++ b/examples/custom_pileup_analytics.rs @@ -39,7 +39,12 @@ fn main() { for column in engine { let col: PileupColumn = column.expect("pileup column"); total_depth += col.depth() as u64; - println!("{}\t{}\t{}", col.locus.pos.0, col.ref_base as char, col.depth()); + println!( + "{}\t{}\t{}", + col.locus.pos.0, + col.ref_base as char, + col.depth() + ); } println!("# total observed depth across covered positions: {total_depth}"); } diff --git a/tests/frontdoor_demo.rs b/tests/frontdoor_demo.rs index 2e0caba..c5a2a67 100644 --- a/tests/frontdoor_demo.rs +++ b/tests/frontdoor_demo.rs @@ -35,7 +35,11 @@ fn readme_inhouse_contract_demo_runs_end_to_end() { let fa = root.join("examples/data/illumina_toy/reference.fa"); let bam = root.join("examples/data/illumina_toy/alignments.bam"); assert!(fa.exists(), "bundled reference missing: {}", fa.display()); - assert!(bam.exists(), "bundled alignments missing: {}", bam.display()); + assert!( + bam.exists(), + "bundled alignments missing: {}", + bam.display() + ); let dir = unique_dir(); let idx = dir.join("toy.idx"); @@ -50,7 +54,11 @@ fn readme_inhouse_contract_demo_runs_end_to_end() { "--output", idx.to_str().unwrap(), ]); - assert!(out.status.success(), "index: {}", String::from_utf8_lossy(&out.stderr)); + assert!( + out.status.success(), + "index: {}", + String::from_utf8_lossy(&out.stderr) + ); let out = run(&[ "sort", @@ -59,10 +67,24 @@ fn readme_inhouse_contract_demo_runs_end_to_end() { "--output", sorted.to_str().unwrap(), ]); - assert!(out.status.success(), "sort: {}", String::from_utf8_lossy(&out.stderr)); + assert!( + out.status.success(), + "sort: {}", + String::from_utf8_lossy(&out.stderr) + ); - let out = run(&["plan", "--index", idx.to_str().unwrap(), "--budget-mb", "512"]); - assert!(out.status.success(), "plan: {}", String::from_utf8_lossy(&out.stderr)); + let out = run(&[ + "plan", + "--index", + idx.to_str().unwrap(), + "--budget-mb", + "512", + ]); + assert!( + out.status.success(), + "plan: {}", + String::from_utf8_lossy(&out.stderr) + ); assert!(String::from_utf8_lossy(&out.stdout).contains("predicted peak")); let out = run(&[ @@ -85,7 +107,11 @@ fn readme_inhouse_contract_demo_runs_end_to_end() { assert!(manifest.exists(), "receipt sidecar must be written"); let out = run(&["verify", "--manifest", manifest.to_str().unwrap()]); - assert!(out.status.success(), "verify: {}", String::from_utf8_lossy(&out.stderr)); + assert!( + out.status.success(), + "verify: {}", + String::from_utf8_lossy(&out.stderr) + ); assert!(String::from_utf8_lossy(&out.stdout).contains("verify: OK")); std::fs::remove_dir_all(&dir).ok();