From 45751b9232bbd44d9096abbbc0d2214ba6a38277 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Tue, 2 Jun 2026 09:06:08 -0700 Subject: [PATCH 1/5] docs(design): hardening fast-follow spec (sort tie-break, @SQ check, BSD RSS, honesty pass) Co-Authored-By: Claude Opus 4.8 (1M context) --- ...2026-06-02-hardening-fast-follow-design.md | 185 ++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 docs/superpowers/specs/2026-06-02-hardening-fast-follow-design.md diff --git a/docs/superpowers/specs/2026-06-02-hardening-fast-follow-design.md b/docs/superpowers/specs/2026-06-02-hardening-fast-follow-design.md new file mode 100644 index 0000000..de9655c --- /dev/null +++ b/docs/superpowers/specs/2026-06-02-hardening-fast-follow-design.md @@ -0,0 +1,185 @@ +# Hardening Fast-Follow (design) + +**Status:** DESIGN SPEC — 2026-06-02. The cross-cutting hardening items deferred from the keystone +(`2026-06-02-trust-correctness-hardening-keystone-design.md` §7). **Branch:** +`rosalind/hardening-fast-follow` (off `main` `7ffe6b0` = keystone + D0 merged). Source: the 2026-06-02 +reflection audit. User chose "finish the hardening fast-follow" as the increment. + +## 1. Goal + +Close the remaining trust/correctness edges the keystone left, and align the front-door surfaces with +the shipped reality. Four independent fixes, one coherent PR (each small, each tested where it's code). + +## 2. The four items + +### Item 1 — sort k-way-merge tie-break (finding #5, high) + +**File:** `src/genomics/sort.rs`. + +`HeapItem::cmp` (sort.rs:188-192) orders only by `SortKey` (tid, pos, is_reverse, qname); `source_idx` +is stored but unused. For records with a fully-equal key (duplicate-marked reads; a primary + an +overlapping mate/split sharing qname+pos+strand), `BinaryHeap::pop` returns an unspecified one, +depending on the chunk partition — i.e. on `--memory-mb`. So the same input sorted at two budgets can +produce byte-different sorted BAMs (violates `docs/determinism.md` Rule 2/3). + +**Fix:** add `source_idx` as the final tie-break in `HeapItem::cmp`, reversed to match the max-heap +convention `SortKey::cmp` already uses (so the **lower** `source_idx` pops first). Because records are +read in input order and assigned to chunks sequentially, and each chunk is stable-sorted, "ties broken +by ascending `source_idx`" reproduces **input order** for equal-key records — independent of how they +were partitioned into chunks (i.e. budget-invariant). Make `PartialEq` consistent with the now-total +`Ord` (compare `key` AND `source_idx`); at most one item per chunk is in the heap at once, so +`source_idx` is a strict tie-break and `cmp` never returns `Equal` for distinct heap items. + +```rust +impl Ord for HeapItem { + fn cmp(&self, other: &Self) -> Ordering { + // Total order: key first (reversed for the min-key-first max-heap), then + // source_idx (also reversed) so equal-key records pop in input order — + // making the merge output independent of the chunk partition (--memory-mb). + self.key + .cmp(&other.key) + .then_with(|| other.source_idx.cmp(&self.source_idx)) + } +} +impl PartialEq for HeapItem { + fn eq(&self, other: &Self) -> bool { + self.key == other.key && self.source_idx == other.source_idx + } +} +``` + +**Test (new `#[cfg(test)] mod tests` in sort.rs):** build two `Record`s with identical (tid, pos, +strand, qname), wrap as `HeapItem::new(0, a)` and `HeapItem::new(3, b)`, push both into a +`BinaryHeap`, pop twice, assert the first popped has `source_idx == 0` (lower pops first = input +order). A third record with a smaller pos pops before both regardless of `source_idx`. + +### Item 2 — `@SQ` contig-length cross-check (finding #7, high) + +**File:** `src/io/bam.rs`. + +`record_to_aligned_read` maps the reference by **name only** (`contigs.by_name`, bam.rs:35) — the BAM +header's `LN` is never compared to the index contig length. A BAM aligned to a different-length +`chr1` (a different assembly/patch) is silently accepted, producing coordinate-shifted/truncated calls +with no error. samtools/bcftools reject `@SQ` length mismatches; Rosalind does not. + +**Fix:** in `StreamingBamSource::new` (the bounded `--index` contract path), after opening, validate +every header `@SQ` entry whose name is also in the index `ContigSet`: assert `header.target_len(tid) +== Some(contig.length as u64)`; on mismatch return `CoreError::MalformedRecord` naming the contig and +both lengths. Names in the header but absent from the index keep the existing skip behavior (the +records are skipped downstream). One-time check in `new()` (not per record). The `ContigSet` API is +`by_name(&str) -> Option<&Contig>` with `Contig.length: u32` (core/locus.rs); header lengths via +`HeaderView::target_count()` / `tid2name(tid)` / `target_len(tid) -> Option`. + +```rust +fn validate_contig_lengths(header: &bam::HeaderView, contigs: &ContigSet) -> Result<(), CoreError> { + for tid in 0..header.target_count() { + let name = std::str::from_utf8(header.tid2name(tid)) + .map_err(|_| CoreError::MalformedRecord("BAM reference name is not UTF-8".into()))?; + if let Some(c) = contigs.by_name(name) { + let hlen = header.target_len(tid); + if hlen != Some(c.length as u64) { + return Err(CoreError::MalformedRecord(format!( + "BAM @SQ length for contig '{name}' ({}) disagrees with the index ({}); \ + the alignments were built against a different reference", + hlen.map(|l| l.to_string()).unwrap_or_else(|| "missing".into()), + c.length + ))); + } + } + } + Ok(()) +} +``` +Call it in `StreamingBamSource::new` before `Ok(Self { … })`. + +**Test (bam.rs tests):** write a BAM whose `@SQ` `LN` for `chr1` is 2000 against a `ContigSet` where +`chr1` is 1000 → `StreamingBamSource::new` returns `Err`; the matching-length case still constructs Ok +(the existing `streaming_source_*` tests cover the Ok path — they use matching lengths). + +### Item 3 — BSD `ru_maxrss` unit fix (finding #6, high) + +**File:** `src/util/rss.rs`. + +The `#[cfg(not(target_os = "linux"))]` branch returns `ru_maxrss` raw, assuming macOS-style **bytes**. +That is wrong on the BSDs (FreeBSD/NetBSD/OpenBSD/DragonFly), which report **KiB** — a silent 1024× +**under-count** (the dangerous direction: `--enforce` would say "within" / `verify` pass for a job +that blew its budget 1000×). The edge/field/clinical appliances the README courts include BSD-based +NAS/storage boxes. + +**Fix (minimal + correct):** invert the cfg. Darwin (macOS/iOS) is the only common platform reporting +bytes; Linux + all BSDs + other unix report KiB. So: + +```rust +let raw = usage.ru_maxrss as u64; +// ru_maxrss units differ by platform: bytes on Darwin (macOS/iOS), KiB on Linux +// and the BSDs. Treat Darwin as bytes; everything else as KiB (×1024). For an +// unenumerated target this defaults to KiB — the common case, and the safe +// (never-undercount) direction for the memory contract. +#[cfg(any(target_os = "macos", target_os = "ios"))] +{ + raw +} +#[cfg(not(any(target_os = "macos", target_os = "ios")))] +{ + raw.saturating_mul(1024) +} +``` + +Update the doc comment accordingly. + +**Test (rss.rs tests):** allocate ~64 MiB, touch one byte per 4 KiB page (force residency), +`black_box` it, then assert `peak_rss_bytes() >= 32 * 1024 * 1024`. On the run platform (dev macOS = +bytes; CI Linux = KiB×1024) the value reflects ~64 MiB; a dropped `×1024` on Linux would yield ~64 KiB +< 32 MiB and fail. (Lower-bound assertion only — peak RSS is a process-global high-water mark, so +other tests can only raise it.) + +### Item 4 — front-door √t honesty pass (low, but the most jarring inconsistency) + +**Files:** `Cargo.toml`, `src/main.rs`, `scale_test_results.txt` (delete), `CONTRACT.md`. + +The README/`OPEN_PROBLEMS` honestly demote √t to "not yet load-bearing," but three skimmable surfaces +still sell it as shipped/verified: + +- `Cargo.toml:6` description `"Accessible genomics engine with O(√t) space complexity …"` → rewrite + contract-first, e.g. `"Deterministic, low-memory genomics engine: memory as a verifiable contract + (declare → predict → honor → verify) for alignment and variant calling"`. +- `src/main.rs:25` CLI about-string `"Genomic analysis engine using O(√t) space"` → e.g. + `"Deterministic low-memory genomics engine with a verifiable memory contract"`. +- `scale_test_results.txt` (repo root, 278 lines, **unreferenced** by any code/doc/script) — lines + 188-278 are an "O(√t) Space Complexity Verification" with 11× `✓ Space scales as O(√t)`, measured + on the tautological `SpaceTracker` counter (per the strategy note, the counter is self-incremented, + so it verifies the counter, not real RSS). A browsing builder reads it as evidence √t is shipped. + **Delete it** (`git rm`): it is a stray captured-output artifact, not referenced anywhere, and it + contradicts the README's own honest demotion. +- `CONTRACT.md:107` `"identical inputs produce a byte-identical VCF and a byte-identical manifest"` is + false — the manifest embeds the machine-dependent realized `peak_rss_bytes` (varies run-to-run). + Reword to: `"identical inputs produce a byte-identical VCF and a manifest identical except for the + realized peak_rss_bytes (a machine-dependent measurement)"`. + +No test (docs/strings); the build + the existing CLI `--help` smoke remain green. + +## 3. Cross-cutting requirements + +- Per-item commit; `cargo fmt --check` clean; 0 warnings (debug + release); full `cargo test` green at + each boundary. +- The sort tie-break must not change output for distinct-key inputs (only equal-key ordering becomes + deterministic). The `@SQ` check must not break the existing `streaming_source_*` tests (they use + matching lengths). +- Deleting `scale_test_results.txt`: confirmed unreferenced (grep across `.rs`/`.md`/`.toml`/`.sh` + found no references) and contradicts the shipped story — safe to remove. + +## 4. Out of scope (further fast-follow, not this PR) + +- **cgroup-awareness + manifest os/arch provenance** (finding #6 remainder) — read cgroup memory + limits (where the OOM-killer fires in containers) and stamp os/arch into the receipt. Larger design + surface (enforcement semantics under a cgroup limit); belongs with a deployment/portability story. +- The bigger strategic bets (adoption on-ramp; ML feature substrate) — separate increments. + +## 5. Self-review + +- **Coverage:** 4 items, each with file + fix + (where code) a test. ✓ +- **Type consistency:** sort `HeapItem` Ord/PartialEq stay consistent; `validate_contig_lengths` + uses the confirmed `ContigSet`/`HeaderView` APIs; rss cfg branches both return `u64`. ✓ +- **No placeholders.** The BSD cfg-inversion judgment call (bytes iff Darwin; unknown → KiB = safe + direction) is stated explicitly. ✓ +- **Scope:** one coherent PR; cgroup/provenance + strategic bets explicitly deferred (§4). ✓ From ff7a4ddfc03277339341437e396882bf4672b6a1 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Tue, 2 Jun 2026 09:06:50 -0700 Subject: [PATCH 2/5] fix(sort): total-order k-way merge tie-break (budget-invariant output) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HeapItem::cmp now tie-breaks equal sort keys by source_idx (reversed for the max-heap, so the lower index pops first = input order). Equal-key records (duplicate-marked reads, primary+overlapping-split sharing qname/pos/strand) therefore merge in a deterministic order independent of how they were partitioned into spill chunks — i.e. the sorted BAM is byte-identical across --memory-mb budgets. PartialEq made consistent with the now-total Ord. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/genomics/sort.rs | 55 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/src/genomics/sort.rs b/src/genomics/sort.rs index a1dcbc2..8e73a1f 100644 --- a/src/genomics/sort.rs +++ b/src/genomics/sort.rs @@ -181,13 +181,22 @@ impl Eq for HeapItem {} impl PartialEq for HeapItem { fn eq(&self, other: &Self) -> bool { - self.key == other.key + // Consistent with the now-total `Ord` (key + source_idx). + self.key == other.key && self.source_idx == other.source_idx } } impl Ord for HeapItem { fn cmp(&self, other: &Self) -> Ordering { - self.key.cmp(&other.key) + // Total order: key first (reversed for the min-key-first max-heap, as + // `SortKey::cmp` already is), then `source_idx` — also reversed so the + // LOWER source_idx pops first. Records are read in input order and + // assigned to chunks sequentially, so equal-key records pop in input + // order regardless of how they were partitioned into chunks — making the + // merge output independent of the `--memory-mb` budget. + self.key + .cmp(&other.key) + .then_with(|| other.source_idx.cmp(&self.source_idx)) } } @@ -204,3 +213,45 @@ fn sort_key_cmp(a: &Record, b: &Record) -> Ordering { .then_with(|| a.is_reverse().cmp(&b.is_reverse())) .then_with(|| a.qname().cmp(b.qname())) } + +#[cfg(test)] +mod tests { + use super::*; + use rust_htslib::bam::record::{Cigar, CigarString}; + + fn rec(qname: &[u8], tid: i32, pos: i64) -> Record { + let mut r = Record::new(); + let cigar = CigarString(vec![Cigar::Match(1)]); + r.set(qname, Some(&cigar), b"A", &[30u8]); + r.set_tid(tid); + r.set_pos(pos); + r + } + + #[test] + fn merge_tie_break_pops_equal_key_records_in_source_index_order() { + // Two records with a fully-equal sort key (same tid/pos/strand/qname) must + // pop in ascending source_idx (= input order), independent of push order — + // this is what makes the merge output independent of the chunk partition + // (--memory-mb). A record with a smaller pos pops before both. + let a = HeapItem::new(0, rec(b"dup", 0, 100)); + let b = HeapItem::new(3, rec(b"dup", 0, 100)); // equal key, higher source_idx + let early = HeapItem::new(2, rec(b"dup", 0, 50)); // smaller pos -> pops first + + let mut heap: BinaryHeap = BinaryHeap::new(); + // Push in an order that does NOT match the desired pop order. + heap.push(b); + heap.push(early); + heap.push(a); + + let p1 = heap.pop().unwrap(); + let p2 = heap.pop().unwrap(); + let p3 = heap.pop().unwrap(); + assert_eq!(p1.record.pos(), 50, "smallest key pops first"); + assert_eq!( + p2.source_idx, 0, + "equal-key tie: lower source_idx pops first" + ); + assert_eq!(p3.source_idx, 3, "then the higher source_idx"); + } +} From e0566f27e2393cc6c7e2a14cb8d260a6aea9c5ca Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Tue, 2 Jun 2026 09:08:06 -0700 Subject: [PATCH 3/5] fix(io): reject @SQ contig-length mismatch at StreamingBamSource open record_to_aligned_read mapped the reference by NAME only, so a BAM aligned to a different-length contig (a different assembly/patch) was silently accepted and produced coordinate-shifted/truncated calls. validate_contig_lengths now checks every shared @SQ name's LN against the index ContigSet at open and errors on mismatch (naming the contig + both lengths), matching samtools/bcftools. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/io/bam.rs | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/src/io/bam.rs b/src/io/bam.rs index 0186e5f..77f12b9 100644 --- a/src/io/bam.rs +++ b/src/io/bam.rs @@ -74,6 +74,36 @@ pub(crate) fn record_to_aligned_read( })) } +/// Cross-check the BAM header's `@SQ` contig lengths against the index `ContigSet`. +/// For every header contig whose name is also in the index, the lengths must match; +/// a mismatch means the alignments were built against a different reference (a +/// different assembly/patch), which would silently produce coordinate-shifted or +/// truncated calls. Names present in the header but absent from the index are left +/// alone (their records are skipped downstream). +pub(crate) fn validate_contig_lengths( + header: &bam::HeaderView, + contigs: &ContigSet, +) -> Result<(), CoreError> { + for tid in 0..header.target_count() { + let name = std::str::from_utf8(header.tid2name(tid)) + .map_err(|_| CoreError::MalformedRecord("BAM reference name is not UTF-8".into()))?; + if let Some(c) = contigs.by_name(name) { + let header_len = header.target_len(tid); + if header_len != Some(c.length as u64) { + return Err(CoreError::MalformedRecord(format!( + "BAM @SQ length for contig '{name}' ({}) disagrees with the index ({}); \ + the alignments were built against a different reference", + header_len + .map(|l| l.to_string()) + .unwrap_or_else(|| "missing".into()), + c.length + ))); + } + } + } + Ok(()) +} + /// Read all mapped records of a BAM into canonical `core::AlignedRead`s, mapping /// each record's reference name to a contig id via `contigs`. Records that are /// unmapped, have no tid, or whose reference is absent from `contigs` are @@ -139,6 +169,9 @@ impl<'a> StreamingBamSource<'a> { let reader = bam::Reader::from_path(path) .map_err(|e| CoreError::MalformedRecord(format!("open BAM {}: {e}", path.display())))?; let header = reader.header().to_owned(); + // Reject a BAM aligned to a different-length reference up front — names + // matching the index must agree on length, or every coordinate is suspect. + validate_contig_lengths(&header, contigs)?; Ok(Self { reader, header, @@ -298,6 +331,37 @@ mod tests { std::fs::remove_dir_all(&dir).ok(); } + #[test] + fn streaming_source_rejects_sq_length_mismatch() { + // BAM header says chr1 is 2000 bp; the index ContigSet says 1000 bp — the + // alignments were built against a different reference. Reject at open. + let dir = std::env::temp_dir().join(format!( + "rosalind-stream-sqlen-{}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&dir).unwrap(); + let bam_path = dir.join("wrongref.bam"); + let header = test_header(&[("chr1", 2000)]); + { + let mut writer = bam::Writer::from_path(&bam_path, &header, bam::Format::Bam).unwrap(); + let hv = writer.header().clone(); + push_record(&mut writer, &hv, 0, 10, b"ACGT"); + } + let mut contigs = ContigSet::new(); + contigs.push("chr1", 1000); + let err = StreamingBamSource::new(&bam_path, &contigs); + assert!(err.is_err(), "length mismatch must be rejected at open"); + let msg = format!("{}", err.err().unwrap()); + assert!( + msg.contains("disagrees with the index"), + "clear message: {msg}" + ); + std::fs::remove_dir_all(&dir).ok(); + } + #[test] fn reads_bam_into_core_reads_with_flags_and_cigar() { let path = tmp("basic"); From 859eae2c6c8d9577d0fb68d5c6fb80aef6b41aad Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Tue, 2 Jun 2026 09:08:43 -0700 Subject: [PATCH 4/5] fix(rss): correct ru_maxrss units on the BSDs (bytes iff Darwin, else KiB) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The old not(linux) branch returned ru_maxrss raw, assuming macOS-style bytes — a silent 1024x UNDER-count on the BSDs (which report KiB), the dangerous direction: --enforce/verify would pass a job that breached its budget. Inverted the cfg: bytes only on Darwin (macos/ios), KiB->bytes everywhere else, so the unknown-target default is the safe over-estimate direction. Added a magnitude test that catches a dropped conversion on the run platform. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/util/rss.rs | 46 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/src/util/rss.rs b/src/util/rss.rs index fa2d419..679e8c5 100644 --- a/src/util/rss.rs +++ b/src/util/rss.rs @@ -7,8 +7,13 @@ use libc::rusage; /// Return the process peak RSS in bytes (best-effort, platform-dependent). /// -/// - On Linux, `ru_maxrss` is reported in KiB. -/// - On macOS, `ru_maxrss` is reported in bytes. +/// `ru_maxrss` units differ by platform: **bytes** on Darwin (macOS/iOS), +/// **KiB** on Linux and the BSDs (FreeBSD/NetBSD/OpenBSD/DragonFly). We treat +/// Darwin as bytes and everything else as KiB (×1024). For an unenumerated +/// target this defaults to KiB — the common case for `getrusage`, and the safe +/// (never-undercount) direction for the memory contract: under-reporting peak +/// RSS would let `--enforce`/`verify` pass a job that actually breached its +/// budget, which is exactly the failure the contract forbids. pub fn peak_rss_bytes() -> u64 { let mut usage: rusage = unsafe { std::mem::zeroed() }; let rc = unsafe { libc::getrusage(libc::RUSAGE_SELF, &mut usage as *mut rusage) }; @@ -17,12 +22,41 @@ pub fn peak_rss_bytes() -> u64 { } let raw = usage.ru_maxrss as u64; - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "macos", target_os = "ios"))] { - raw.saturating_mul(1024) + raw // already bytes } - #[cfg(not(target_os = "linux"))] + #[cfg(not(any(target_os = "macos", target_os = "ios")))] { - raw + raw.saturating_mul(1024) // KiB -> bytes (Linux, the BSDs, other unix) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn peak_rss_reflects_a_real_allocation_in_bytes() { + // Allocate ~64 MiB and touch one byte per 4 KiB page to force residency, + // then assert the reported peak is on the order of MiB (>= 32 MiB). This + // catches a dropped/incorrect KiB->bytes conversion on the run platform: + // a 1024x undercount would report ~64 KiB, far below the bound. (Lower + // bound only — peak RSS is a process-global high-water mark, so other + // tests can only raise it.) + let n = 64usize * 1024 * 1024; + let mut buf = vec![0u8; n]; + let mut i = 0; + while i < n { + buf[i] = 1; + i += 4096; + } + std::hint::black_box(&buf); + let peak = peak_rss_bytes(); + assert!( + peak >= 32 * 1024 * 1024, + "peak RSS {peak} bytes implausibly small after a 64 MiB allocation \ + (a dropped KiB->bytes conversion would land here)" + ); } } From 12a48081e83ae57340e6f37f46d53a433c85a4ec Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Tue, 2 Jun 2026 09:09:40 -0700 Subject: [PATCH 5/5] =?UTF-8?q?docs:=20front-door=20honesty=20pass=20?= =?UTF-8?q?=E2=80=94=20align=20skimmable=20surfaces=20with=20the=20contrac?= =?UTF-8?q?t?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README/OPEN_PROBLEMS honestly demote √t to 'not yet load-bearing', but three skimmable surfaces still sold it as shipped. Fixed: Cargo.toml description and the CLI --help about-string now lead with the memory contract (not 'O(√t) space'); deleted scale_test_results.txt (an unreferenced root artifact whose 'O(√t) Space Complexity Verification' measured the tautological SpaceTracker counter, not real RSS); corrected CONTRACT.md's false 'byte-identical manifest' claim (the manifest embeds the machine-dependent realized peak_rss_bytes). Co-Authored-By: Claude Opus 4.8 (1M context) --- CONTRACT.md | 3 +- Cargo.toml | 2 +- scale_test_results.txt | 278 ----------------------------------------- src/main.rs | 5 +- 4 files changed, 7 insertions(+), 281 deletions(-) delete mode 100644 scale_test_results.txt diff --git a/CONTRACT.md b/CONTRACT.md index 9003b54..ae76bec 100644 --- a/CONTRACT.md +++ b/CONTRACT.md @@ -104,4 +104,5 @@ A complete, runnable example: [`examples/custom_pileup_analytics.rs`](examples/c Every receipt is canonical JSON (sorted keys, no timestamps) with BLAKE3 content hashes of the index, the alignments, and the output VCF, plus the realized `peak_rss_bytes` / `max_working_set_bytes` and the contract params (`memory_budget_mb`, `contract_verdict`, `enforced`, `max_depth`, `max_read_len`). Identical -inputs produce a byte-identical VCF and a byte-identical manifest — and `rosalind verify` proves it. +inputs produce a byte-identical VCF, and a manifest identical except for the realized `peak_rss_bytes` (a +machine-dependent measurement) — `rosalind verify` re-checks the recorded hashes and peak against the budget. diff --git a/Cargo.toml b/Cargo.toml index 9319f81..c3bcc97 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ name = "rosalind" version = "0.1.0" edition = "2021" authors = ["Logan Nye "] -description = "Accessible genomics engine with O(√t) space complexity for genome alignment and variant calling" +description = "Deterministic, low-memory genomics engine: memory as a verifiable contract (declare → predict → honor → verify) for alignment and variant calling" repository = "https://github.com/logannye/rosalind" license = "MIT OR Apache-2.0" keywords = ["genomics", "alignment", "variant-calling", "bioinformatics", "bwt", "fm-index"] diff --git a/scale_test_results.txt b/scale_test_results.txt deleted file mode 100644 index 6e7834f..0000000 --- a/scale_test_results.txt +++ /dev/null @@ -1,278 +0,0 @@ - Compiling proc-macro2 v1.0.103 - Compiling unicode-ident v1.0.22 - Compiling quote v1.0.41 - Compiling libc v0.2.177 - Compiling cfg-if v1.0.4 - Compiling find-msvc-tools v0.1.4 - Compiling autocfg v1.5.0 - Compiling shlex v1.3.0 - Compiling zerocopy v0.8.27 - Compiling stable_deref_trait v1.2.1 - Compiling pkg-config v0.3.32 - Compiling version_check v0.9.5 - Compiling syn v1.0.109 - Compiling once_cell v1.21.3 - Compiling memchr v2.7.6 - Compiling serde_core v1.0.228 - Compiling writeable v0.6.2 - Compiling typenum v1.19.0 - Compiling regex-syntax v0.8.8 - Compiling getrandom v0.3.4 - Compiling litemap v0.8.1 - Compiling icu_properties_data v2.1.1 - Compiling rand_core v0.6.4 - Compiling generic-array v0.14.9 - Compiling serde v1.0.228 - Compiling aho-corasick v1.1.4 - Compiling num-traits v0.2.19 - Compiling icu_normalizer_data v2.1.1 - Compiling quick-error v1.2.3 - Compiling either v1.15.0 - Compiling syn v2.0.108 - Compiling crossbeam-utils v0.8.21 - Compiling heck v0.5.0 - Compiling vcpkg v0.2.15 - Compiling smallvec v1.15.1 - Compiling regex-automata v0.4.13 - Compiling rustversion v1.0.22 - Compiling jobserver v0.1.34 - Compiling cc v1.2.44 - Compiling semver v0.1.20 - Compiling utf8parse v0.2.2 - Compiling rustc_version v0.1.7 - Compiling anstyle-parse v0.2.7 - Compiling itertools v0.10.5 - Compiling fs-utils v1.1.4 - Compiling ahash v0.8.12 - Compiling radium v0.7.0 - Compiling anstyle-query v1.1.4 - Compiling lazy_static v1.5.0 - Compiling thiserror v1.0.69 - Compiling crypto-common v0.1.6 - Compiling glob v0.3.3 - Compiling rustix v1.1.2 - Compiling cmake v0.1.54 - Compiling paste v1.0.15 - Compiling anstyle v1.0.13 - Compiling colorchoice v1.0.4 - Compiling is_terminal_polyfill v1.70.2 - Compiling num-integer v0.1.46 - Compiling anstream v0.6.21 - Compiling digest v0.10.7 - Compiling num-bigint v0.4.6 - Compiling newtype_derive v0.1.6 - Compiling rand_core v0.9.3 - Compiling errno v0.3.14 - Compiling tracing-core v0.1.34 - Compiling lzma-sys v0.1.20 - Compiling libz-sys v1.1.23 - Compiling bzip2-sys v0.1.13+1.0.8 - Compiling synstructure v0.13.2 - Compiling crossbeam-epoch v0.9.18 - Compiling regex v1.12.2 - Compiling hts-sys v2.2.0 - Compiling utf8_iter v1.0.4 - Compiling strsim v0.11.1 - Compiling rayon-core v1.13.0 - Compiling percent-encoding v2.3.2 - Compiling tap v1.0.1 - Compiling serde_json v1.0.145 - Compiling bitflags v2.10.0 - Compiling clap_lex v0.7.6 - Compiling wyz v0.5.1 - Compiling form_urlencoded v1.2.2 - Compiling clap_builder v4.5.51 - Compiling crossbeam-deque v0.8.6 - Compiling blake3 v1.8.2 - Compiling anyhow v1.0.100 - Compiling log v0.4.28 - Compiling ark-serialize-derive v0.4.2 - Compiling derivative v2.2.0 - Compiling ark-ff-asm v0.4.2 - Compiling ciborium-io v0.2.2 - Compiling plotters-backend v0.3.7 - Compiling itoa v1.0.15 - Compiling pin-project-lite v0.2.16 - Compiling ark-ff-macros v0.4.2 - Compiling fastrand v2.3.0 - Compiling ryu v1.0.20 - Compiling zerocopy-derive v0.8.27 - Compiling zerofrom-derive v0.1.6 - Compiling yoke-derive v0.8.1 - Compiling zerovec-derive v0.11.2 - Compiling displaydoc v0.2.5 - Compiling serde_derive v1.0.228 - Compiling zeroize_derive v1.4.2 - Compiling thiserror-impl v1.0.69 - Compiling derive-new v0.6.0 - Compiling tracing-attributes v0.1.30 - Compiling clap_derive v4.5.49 - Compiling strum_macros v0.26.4 - Compiling zeroize v1.8.2 - Compiling funty v2.0.0 - Compiling zerofrom v0.1.6 - Compiling tempfile v3.23.0 - Compiling plotters-svg v0.3.7 - Compiling bitvec v1.0.1 - Compiling tracing-log v0.2.0 - Compiling derive-new v0.5.9 - Compiling tracing v0.1.41 - Compiling yoke v0.8.1 - Compiling test-case-core v3.3.1 - Compiling matchers v0.2.0 - Compiling sharded-slab v0.1.7 - Compiling wait-timeout v0.2.1 - Compiling clap v4.5.51 - Compiling thread_local v1.1.9 - Compiling fnv v1.0.7 - Compiling constant_time_eq v0.3.1 - Compiling custom_derive v0.1.7 - Compiling same-file v1.0.6 - Compiling bit-vec v0.8.0 - Compiling linear-map v1.2.0 - Compiling arrayvec v0.7.6 - Compiling byteorder v1.5.0 - Compiling ieee754 v0.2.6 - Compiling bio-types v1.0.4 - Compiling cast v0.3.0 - Compiling subtle v2.6.1 - Compiling arrayref v0.3.9 - Compiling nu-ansi-term v0.50.3 - Compiling bit-set v0.8.0 - Compiling walkdir v2.5.0 - Compiling criterion-plot v0.5.0 - Compiling test-case-macros v3.3.1 - Compiling rusty-fork v0.3.1 - Compiling rayon v1.11.0 - Compiling plotters v0.3.7 - Compiling ff v0.13.1 - Compiling rand_xorshift v0.4.0 - Compiling rand v0.9.2 - Compiling tracing-subscriber v0.3.20 - Compiling is-terminal v0.4.17 - Compiling anes v0.1.6 - Compiling oorandom v11.1.5 - Compiling unarray v0.1.4 - Compiling ppv-lite86 v0.2.21 - Compiling half v2.7.1 - Compiling test-case v3.3.1 - Compiling zerovec v0.11.5 - Compiling zerotrie v0.2.3 - Compiling rand_chacha v0.3.1 - Compiling ciborium-ll v0.2.2 - Compiling hashbrown v0.13.2 - Compiling rand v0.8.5 - Compiling rand_chacha v0.9.0 - Compiling proptest v1.9.0 - Compiling ark-std v0.4.0 - Compiling ark-serialize v0.4.2 - Compiling ciborium v0.2.2 - Compiling tinytemplate v1.2.1 - Compiling criterion v0.5.1 - Compiling tinystr v0.8.2 - Compiling potential_utf v0.1.4 - Compiling icu_locale_core v2.1.1 - Compiling icu_collections v2.1.1 - Compiling ark-ff v0.4.2 - Compiling icu_provider v2.1.1 - Compiling icu_normalizer v2.1.1 - Compiling icu_properties v2.1.1 - Compiling idna_adapter v1.2.1 - Compiling idna v1.1.0 - Compiling url v2.5.7 - Compiling ark-poly v0.4.2 - Compiling rust-htslib v0.44.1 - Compiling rosalind v0.1.0 (/Users/logannye/rosalind) - Finished `dev` profile [unoptimized + debuginfo] target(s) in 10.26s - Running `target/debug/examples/scale_performance_test` -Scale Performance Test: O(√t) Space Complexity Verification -========================================================== - -Running scale tests... -Note: Large time bounds (t > 1,000,000) may take several minutes to complete. - -Time Bound | Block Size | Blocks | Space Used | √t Bound | Space/t | Time(s) | Scaling ------------------------------------------------------------------------------------------------------------------------- -100 | 10 | 10 | 17 | 20 | 0.17000000 | 0.002 | - - components: leaf=10 stack=5 ledger=2 -400 | 20 | 20 | 31 | 40 | 0.07750000 | 0.003 | 1.82x - components: leaf=20 stack=6 ledger=5 -1,600 | 40 | 40 | 57 | 80 | 0.03562500 | 0.010 | 1.84x - components: leaf=40 stack=7 ledger=10 -6,400 | 80 | 80 | 108 | 160 | 0.01687500 | 0.023 | 1.89x - components: leaf=80 stack=8 ledger=20 -25,600 | 160 | 160 | 209 | 320 | 0.00816406 | 0.074 | 1.94x - components: leaf=160 stack=9 ledger=40 -100,000 | 317 | 316 | 406 | 634 | 0.00406000 | 0.268 | 1.94x - components: leaf=317 stack=10 ledger=79 -400,000 | 633 | 632 | 802 | 1,266 | 0.00200500 | 1.035 | 1.98x - components: leaf=633 stack=11 ledger=158 -Running test for t=1600000 (this may take a while)... -1,600,000 | 1265 | 1,265 | 1,593 | 2,530 | 9.9562e-4 | 4.072 | 1.99x - components: leaf=1,265 stack=12 ledger=316 -Running test for t=6400000 (this may take a while)... -6,400,000 | 2530 | 2,530 | 3,175 | 5,060 | 4.9609e-4 | 16.254 | 1.99x - components: leaf=2,530 stack=13 ledger=632 -Running test for t=25600000 (this may take a while)... -25,600,000 | 5060 | 5,060 | 6,339 | 10,120 | 2.4762e-4 | 64.045 | 2.00x - components: leaf=5,060 stack=14 ledger=1,265 -Running test for t=100000000 (this may take a while)... -100,000,000 | 10000 | 10,000 | 12,515 | 20,000 | 1.2515e-4 | 250.747 | 1.97x - components: leaf=10,000 stack=15 ledger=2,500 -Running test for t=400000000 (this may take a while)... -400,000,000 | 20000 | 20,000 | 25,016 | 40,000 | 6.2540e-5 | 1005.999 | 2.00x - components: leaf=20,000 stack=16 ledger=5,000 -Running test for t=1600000000 (this may take a while)... -1,600,000,000 | 40000 | 40,000 | 50,017 | 80,000 | 3.1261e-5 | 4013.011 | 2.00x - components: leaf=40,000 stack=17 ledger=10,000 - -Summary: --------- -✓ All tests satisfy space bound (space ≤ O(√t)) -✓ Space scaling: 1.95x average (expected: ~2.00x for 4x time increase) -✓ Scaling is sublinear (closer to √t than linear) -✓ Efficiency: 100.00% space savings vs naive O(t) at t=1600000000 -✓ Largest test: t=1600000000 completed in 4013.01s - -O(√t) Verification: -------------------- -t: 100 → 400 (4x), space: 17 → 31 (1.82x), √t ratio: 2.00x - ✓ Space scales as O(√t), not O(t) -t: 400 → 1600 (4x), space: 31 → 57 (1.84x), √t ratio: 2.00x - ✓ Space scales as O(√t), not O(t) -t: 1600 → 6400 (4x), space: 57 → 108 (1.89x), √t ratio: 2.00x - ✓ Space scales as O(√t), not O(t) -t: 6400 → 25600 (4x), space: 108 → 209 (1.94x), √t ratio: 2.00x - ✓ Space scales as O(√t), not O(t) -t: 25600 → 100000 (3x), space: 209 → 406 (1.94x), √t ratio: 1.98x - ✓ Space scales as O(√t), not O(t) -t: 100000 → 400000 (4x), space: 406 → 802 (1.98x), √t ratio: 2.00x - ✓ Space scales as O(√t), not O(t) -t: 400000 → 1600000 (4x), space: 802 → 1593 (1.99x), √t ratio: 2.00x - ✓ Space scales as O(√t), not O(t) -t: 1600000 → 6400000 (4x), space: 1593 → 3175 (1.99x), √t ratio: 2.00x - ✓ Space scales as O(√t), not O(t) -t: 6400000 → 25600000 (4x), space: 3175 → 6339 (2.00x), √t ratio: 2.00x - ✓ Space scales as O(√t), not O(t) -t: 25600000 → 100000000 (3x), space: 6339 → 12515 (1.97x), √t ratio: 1.98x - ✓ Space scales as O(√t), not O(t) -t: 100000000 → 400000000 (4x), space: 12515 → 25016 (2.00x), √t ratio: 2.00x - ✓ Space scales as O(√t), not O(t) -t: 400000000 → 1600000000 (4x), space: 25016 → 50017 (2.00x), √t ratio: 2.00x - ✓ Space scales as O(√t), not O(t) - -CSV Export: -time_bound,block_size,num_blocks,space_used,sqrt_t_bound,space_efficiency,execution_time,scaling_ratio -100,10,10,17,20,0.1700,0.002,- -400,20,20,31,40,0.0775,0.003,1.8235294117647058 -1600,40,40,57,80,0.0356,0.010,1.8387096774193548 -6400,80,80,108,160,0.0169,0.023,1.894736842105263 -25600,160,160,209,320,0.0082,0.074,1.9351851851851851 -100000,317,316,406,634,0.0041,0.268,1.9425837320574162 -400000,633,632,802,1266,0.0020,1.035,1.9753694581280787 -1600000,1265,1265,1593,2530,0.0010,4.072,1.986284289276808 -6400000,2530,2530,3175,5060,0.0005,16.254,1.9930947897049591 -25600000,5060,5060,6339,10120,0.0002,64.045,1.9965354330708662 -100000000,10000,10000,12515,20000,0.0001,250.747,1.974286165010254 -400000000,20000,20000,25016,40000,0.0001,1005.999,1.998881342389133 -1600000000,40000,40000,50017,80000,0.0000,4013.011,1.9994003837543972 diff --git a/src/main.rs b/src/main.rs index 99aed88..a0d909f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,7 +22,10 @@ use rust_htslib::bam::{ }; #[derive(Parser, Debug)] -#[command(name = "rosalind", about = "Genomic analysis engine using O(√t) space")] +#[command( + name = "rosalind", + about = "Deterministic low-memory genomics engine with a verifiable memory contract" +)] struct Cli { #[command(subcommand)] command: Commands,