From 95c489ba5b9882c9e944bbc98fb99a4ce103af61 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 19:53:16 -0700 Subject: [PATCH 1/7] =?UTF-8?q?docs(spec):=20Phase=20D0=20=E2=80=94=20meas?= =?UTF-8?q?ure-first=20build-memory=20probe=20+=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First Phase-D increment after the kernel pivot (√t/Cook-Mertz retired as the construction mechanism, kept as framing; native external-memory constructor is the target). D0 instruments the existing build to emit a build receipt (realized peak RSS + a logical n-scale SA-IS accountant), runs it on E. coli + yeast, and records a pre-registered verdict (±25% of ~185 MiB AND ≥70% SA-workspace attribution) that gates D1. Pure instrumentation + measurement + a focused OPEN_PROBLEMS reframing. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../2026-06-02-phase-d0-build-probe-design.md | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 docs/superpowers/specs/2026-06-02-phase-d0-build-probe-design.md diff --git a/docs/superpowers/specs/2026-06-02-phase-d0-build-probe-design.md b/docs/superpowers/specs/2026-06-02-phase-d0-build-probe-design.md new file mode 100644 index 0000000..33f434a --- /dev/null +++ b/docs/superpowers/specs/2026-06-02-phase-d0-build-probe-design.md @@ -0,0 +1,109 @@ +# Phase D0 — measure-first build-memory probe + gate (design) + +**Status:** Spec for review — 2026-06-02. The first increment of **Phase D** (sublinear-space index +construction), after a scoping workflow (`wlb1l1924`) **pivoted the kernel**: the literal √t/Cook–Mertz +combiner is retired as the construction mechanism (it is stubbed *and* discarded on the correctness path; +SA-IS produces a permutation with no low-degree-extension structure to compress; Cook–Mertz is +super-polynomial with no systems realization). Phase D's real target is a **native, polynomial-time, +budget-tunable external-memory SA/BWT constructor** along a *measured* space/time curve, wrapped in the +shipped `plan`/`--enforce`/`verify` contract; **√t/Cook–Mertz is kept as honest theoretical framing only.** +D0 is the cheap, decisive **measure-first gate** that confirms the build is SA-workspace-bound before any +external-memory work begins. Built on a fresh branch off the merged `main`. + +## 1. Goal + the precise claim + +Instrument the existing index build to emit a **build receipt** (realized peak RSS + a logical accountant +that attributes the peak to the simultaneously-live n-scale SA-IS arrays), run it on a small and a +~3× larger real genome, and record a **pre-registered verdict** that gates the rest of Phase D: + +> **CONFIRM** — on E. coli (4.64 Mbp): realized peak RSS within **±25%** of the audited ~185 MiB **AND** +> ≥**70%** of the realized peak attributable to the modeled SA/text/workspace arrays → the build is +> intermediate-state-bound (OPEN_PROBLEMS §3.1), the external-memory path (D1) is **greenlit**. +> +> **NULL** — peak is dominated by something else (allocator slack, the FM-index rank bitvectors, the +> persistence buffers), or attribution < 70% → **stop and re-scope**: a block-tunable SA build would not +> reduce the realized peak; this is a publishable honest-NULL that redirects Phase D and saves months. + +D0 changes **no construction algorithm** — it is instrumentation + measurement + the recorded verdict +(plus a small docs reframing, §6). It also ships a real *build* receipt the contract was missing on the +index path. + +## 2. The logical build-memory accountant (`genomics/index/report.rs`) + +The current `estimate_build_working_set` is a coarse "~12 bytes/base" scalar. Replace/augment it with a +structured, pure, unit-tested model derived from the **actual `sais_impl` allocations** +(`src/genomics/suffix_array.rs`), so the model and the kernel cannot silently drift (the `plan.rs` +discipline). The dominant simultaneously-live n-scale terms (n = reference length in bases): + +| Component | Modeled bytes | Source | +|---|---|---| +| `u32` text (SA-IS input alphabet) | 4n | `sais_u32(text: &[u32], …)` | +| `types` (`Vec`, 1 byte/elem) | 1n | `classify_types` | +| induce-sort SA (`Vec`) | 4n | `induce_sort` → `vec![-1i32; n]` | +| `lms_name` (`Vec`) | 4n | `sais_impl` line ~223 | +| reduced string + recursion (geometric, ≤ n/2 first level) | ~2n | `reduced` + `sais_impl(&reduced, …)` | +| returned SA (`Vec`) | 4n | `sais_impl` return | +| FM-index structures (BWT + rank/C-table) | ~2n | `BlockedFMIndex` build | + +`BuildMemoryModel { total_bytes, components: Vec<(name, bytes)> }` with a `from_reference_len(n)` +constructor and a `render()` for the receipt. (The model is an estimate of the *peak set*, not a sum of +all allocations ever made; D0 measures whether it captures the realized peak — that is the experiment.) + +## 3. The build receipt (`rosalind index`) + +`run_index` (`src/main.rs`) currently prints `build peak RSS: {} MiB`. Extend it to emit a structured +build receipt — to stderr and into the existing index build report (`IndexBuildReport::render` / +`report.rs`): + +- realized **peak RSS** (`util::rss::peak_rss_bytes()`, captured after the build) and realized **bytes/base** + (peak ÷ total_bp); +- the **`BuildMemoryModel` breakdown** (per-component bytes + total) and the modeled bytes/base; +- the **attribution ratio** = modeled total ÷ realized peak, with a `[CONFIRM ≥0.70]` / `[below 0.70]` tag. + +The realized peak is machine-dependent; the breakdown + attribution ratio are the analysis payload. + +## 4. The probe script (`scripts/build_memory_probe.sh`) + +A reproducible bash script (curl + the release binary): + +1. **E. coli** (gate genome) — reuse the cached `results/flagship-ecoli/ecoli.fa` if present (from Move #5), + else fetch GCF_000005845.2 (4.64 Mbp). `rosalind index` → capture the build receipt. +2. **A ~3× larger reference** — *S. cerevisiae* R64 (NCBI RefSeq GCF_000146045.2, ~12.1 Mbp, multi-contig; + the index handles multi-contig — build only, **no reads/align**). `rosalind index` → capture the receipt. +3. Print both receipts + the **bytes/base for each** (is it ~constant across the two → the build is n-scale) + and the **gate verdict** for E. coli. Results → `results/build-probe/` (gitignored). + +## 5. The findings doc (`docs/findings/2026-06-02-d0-build-memory-probe.md`) + +The recorded experiment: realized peak RSS + bytes/base for E. coli and yeast, the component attribution, +and the **pre-registered CONFIRM/NULL verdict** from §1 — which is the D1 go/no-go. Honest notes: realized +peak is machine-dependent; the model is a peak-set estimate; the verdict is what matters. If CONFIRM, +state the greenlight for D1 (blocked external-memory SA construction) and the baseline curve point +(b = n, full-RAM) every later increment must beat. + +## 6. Docs reframing (the pivot, made honest) + +A focused edit to `docs/OPEN_PROBLEMS.md` (§3.2 / §5-D) reflecting the accepted pivot: the √t/Cook–Mertz +machinery is **theoretical framing** for "recomputation along a √-shaped curve," **not** the literal +construction kernel; Phase D's mechanism is a **native budget-tunable external-memory SA/BWT constructor**; +the differentiation is the **contract + measured curve + verifiable receipt**, *not* a new complexity bound. +(One honest paragraph; no rewrite of the whole doc.) + +## 7. Testing + +- Unit tests for `BuildMemoryModel`: monotonic in n, the per-component breakdown sums to the total, no + overflow at u32::MAX-scale n, and the constants match the `sais_impl` allocation sizes. +- `cargo test --test index_cli` (the existing build CLI gates) stays green; the build receipt is additive. +- The probe script + findings doc **are the experiment** (not a permanent unit test — realized RSS is + machine-dependent). The script asserts the build succeeds; the verdict is transcribed into the doc. +- Gates unchanged: `cargo fmt --check`, `cargo build` 0 warnings (debug + release), full suite. + +## 8. Non-goals + +No construction-algorithm change; no blocked/external-memory build (that is D1, gated on D0's CONFIRM); no +u64 widening (D4); no `sa_sample_rate` knob (it shrinks *query*, not *build* peak — explicitly out, per the +guardrail); no new complexity-bound claim; no Cook–Mertz combiner work (retired). MSRV 1.72; no new deps. + +## 9. Branch + +`rosalind/phase-d0-build-probe`, off the merged `main`. Its own PR when done (the user decides merge). From 4159248c1e94d3dd5dbbc0cb4f524ab71eca5c04 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 20:00:33 -0700 Subject: [PATCH 2/7] =?UTF-8?q?docs(plan):=20Phase=20D0=20=E2=80=94=20meas?= =?UTF-8?q?ure-first=20build=20probe=20implementation=20plan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../plans/2026-06-02-phase-d0-build-probe.md | 362 ++++++++++++++++++ 1 file changed, 362 insertions(+) create mode 100644 docs/superpowers/plans/2026-06-02-phase-d0-build-probe.md diff --git a/docs/superpowers/plans/2026-06-02-phase-d0-build-probe.md b/docs/superpowers/plans/2026-06-02-phase-d0-build-probe.md new file mode 100644 index 0000000..f7b80bb --- /dev/null +++ b/docs/superpowers/plans/2026-06-02-phase-d0-build-probe.md @@ -0,0 +1,362 @@ +# Phase D0 — Measure-First Build Probe Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans (inline, chosen for this work). Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Instrument the index build to emit a build receipt (realized peak RSS + a code-grounded n-scale SA-IS memory accountant), run it on E. coli + yeast, and record a pre-registered verdict that gates the rest of Phase D — without changing the construction algorithm. + +**Architecture:** Add a structured `BuildMemoryModel` to `genomics/index/report.rs` that enumerates the real n-scale arrays SA-IS + the FM-index allocate (using true element sizes + a geometric recursion term); have `rosalind index` emit a build receipt (realized peak via `util/rss.rs` + the model breakdown + attribution ratio + bytes/base); a probe script builds E. coli + yeast and captures both; a findings doc records the CONFIRM/NULL verdict. Plus a focused `OPEN_PROBLEMS.md` reframing (√t = framing, native-EM = mechanism). + +**Tech Stack:** Rust 1.72, `cargo test`/`fmt`/`build`, bash + curl + python3. No new deps. Branch `rosalind/phase-d0-build-probe` off merged `main`. + +**Spec:** [`docs/superpowers/specs/2026-06-02-phase-d0-build-probe-design.md`](../specs/2026-06-02-phase-d0-build-probe-design.md). + +**Honesty guardrail (do not tune to the gate):** the accountant must be a *code-grounded enumeration* of the arrays `sais_impl`/`induce_sort`/the FM-index actually allocate, with their true byte sizes — NOT a coefficient reverse-engineered to clear ≥70%. If the honest enumeration lands well below the realized peak, that is a real signal (investigate / NULL), not a number to fudge. The most robust CONFIRM evidence is **bytes/base ≈ constant across E. coli and yeast** (the peak scales with n, not a fixed overhead). + +--- + +## File Structure + +- **Modify** `src/genomics/index/report.rs` — add `BuildMemoryModel` (the n-scale accountant) + render; keep `estimate_build_working_set`/`render_plan_line` (still used by `run_plan` + the `index --memory-budget-mb` line). Tests in-file. +- **Modify** `src/main.rs` — `run_index` emits the structured build receipt (realized peak + model + attribution + bytes/base). +- **Create** `scripts/build_memory_probe.sh` — build E. coli + yeast, capture both receipts + the verdict. +- **Modify** `.gitignore` — ignore `results/` (already ignored by the Move-#5 `/results/` rule; verify). +- **Create** `docs/findings/2026-06-02-d0-build-memory-probe.md` — the recorded experiment + verdict. +- **Modify** `docs/OPEN_PROBLEMS.md` — the focused √t-as-framing reframing. + +--- + +## Task 1: `BuildMemoryModel` — the code-grounded n-scale accountant + +**Files:** +- Modify: `src/genomics/index/report.rs` + +- [ ] **Step 1: Write the failing tests.** Append to the `tests` module in `src/genomics/index/report.rs`: + +```rust + #[test] + fn build_model_breaks_down_and_sums() { + let m = BuildMemoryModel::from_reference_len(1_000_000); + // Components sum to the total. + let sum: u64 = m.components.iter().map(|(_, b)| b).sum(); + assert_eq!(sum, m.total_bytes, "components must sum to total"); + // The major SA-IS arrays are present and n-scale. + let names: Vec<&str> = m.components.iter().map(|(n, _)| n.as_str()).collect(); + for needed in ["text(u32)", "suffix array", "lms_name"] { + assert!(names.iter().any(|n| n.contains(needed)), "missing component {needed}: {names:?}"); + } + // Honest sanity: SA-IS over a u32 text is many bytes/base, not a handful. + assert!(m.total_bytes >= 20_000_000, "model must be ≥20 B/base (~{} B/base)", m.total_bytes / 1_000_000); + } + + #[test] + fn build_model_is_monotonic_and_overflow_safe() { + assert!(BuildMemoryModel::from_reference_len(2_000_000).total_bytes + > BuildMemoryModel::from_reference_len(1_000_000).total_bytes); + let _ = BuildMemoryModel::from_reference_len(u64::MAX); // must not panic + } +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `cd ~/rosalind && cargo test -p rosalind --lib genomics::index::report::tests::build_model 2>&1 | tail -10` +Expected: FAIL — `BuildMemoryModel` not found. + +- [ ] **Step 3: Implement `BuildMemoryModel`.** In `src/genomics/index/report.rs`, add (after `estimate_build_working_set`): + +```rust +/// A code-grounded model of the **peak simultaneously-live n-scale memory** of the +/// SA-IS index build, broken down by the arrays `sais_impl`/`induce_sort`/the +/// FM-index actually allocate. Element sizes are the real Rust types: the SA-IS +/// input text and the suffix array are `u32`/`i32` (4 B); the LMS index vectors +/// are `usize` (8 B); `types` is one byte. `#LMS ≤ n/2` is modeled at n/2. The +/// recursion (`sais_impl(&reduced, …)`) keeps the parent's text/types/lms arrays +/// live while a ~n/2-size child runs, so its overhead is modeled as a geometric +/// tail (~1× the parent's live n-scale arrays). This is a peak-SET estimate, not a +/// sum of every allocation ever made — D0 MEASURES whether it captures the +/// realized peak (the gate); it is not tuned to the gate. +#[derive(Debug, Clone)] +pub struct BuildMemoryModel { + /// Per-component `(name, bytes)` of the modeled peak set. + pub components: Vec<(String, u64)>, + /// Sum of the components. + pub total_bytes: u64, +} + +impl BuildMemoryModel { + /// Model the peak n-scale build memory for a reference of `n` bases. + pub fn from_reference_len(n: u64) -> Self { + // Real element sizes. + const SA_ELEM: u64 = 4; // u32 / i32 suffix-array + text element + const USIZE: u64 = 8; // LMS index vectors are Vec + let lms = n / 2; // #LMS ≤ n/2 (upper-ish) + let comps: Vec<(&str, u64)> = vec![ + ("text(u32)", n.saturating_mul(SA_ELEM)), + ("types(1B)", n), + ("lms_positions(usize)", lms.saturating_mul(USIZE)), + ("induce-sort suffix array(i32)", n.saturating_mul(SA_ELEM)), + ("lms_in_sa_order(usize)", lms.saturating_mul(USIZE)), + ("lms_name(u32)", n.saturating_mul(SA_ELEM)), + ("reduced string(u32)", lms.saturating_mul(SA_ELEM)), + ("returned suffix array(u32)", n.saturating_mul(SA_ELEM)), + ("fm-index (bwt + rank + C-table)", n.saturating_mul(3)), + ]; + // Single-level n-scale footprint. + let level0: u64 = comps.iter().map(|(_, b)| *b).sum(); + // Recursion: parent text/types/lms_positions/lms_name/reduced stay live + // (~text 4n + types 1n + lms_positions 4n + lms_name 4n + reduced 2n = 15n) + // while a ~n/2 child runs; the geometric tail ≈ that parent-live amount. + let recursion = n.saturating_mul(15); + let mut components: Vec<(String, u64)> = + comps.into_iter().map(|(s, b)| (s.to_string(), b)).collect(); + components.push(("recursion (geometric tail)".to_string(), recursion)); + let total_bytes = level0.saturating_add(recursion); + Self { + components, + total_bytes, + } + } + + /// Render the model as a deterministic multi-line breakdown (bytes + per-base). + pub fn render(&self, total_bp: u64) -> String { + let mut out = String::from("build memory model (n-scale peak set):\n"); + let denom = total_bp.max(1); + for (name, bytes) in &self.components { + out.push_str(&format!( + " {name:<34} {:>6} MiB ({} B/base)\n", + bytes / (1 << 20), + bytes / denom + )); + } + out.push_str(&format!( + " {:-<34} {:>6} MiB ({} B/base)\n", + "total ", + self.total_bytes / (1 << 20), + self.total_bytes / denom + )); + out + } +} +``` + +- [ ] **Step 4: Run to verify pass** + +Run: `cd ~/rosalind && cargo test -p rosalind --lib genomics::index::report 2>&1 | tail -10` +Expected: PASS (the two new tests + the existing report tests). The model totals ~33 B/base (level0 ~18n + recursion 15n) — a code-grounded estimate, not tuned. + +- [ ] **Step 5: Commit** + +```bash +cd ~/rosalind && git add src/genomics/index/report.rs && git commit -m "feat(index): BuildMemoryModel — code-grounded n-scale SA-IS build accountant (D0)" +``` + +--- + +## Task 2: `rosalind index` emits the build receipt + +**Files:** +- Modify: `src/main.rs` (`run_index`) + +- [ ] **Step 1: Replace the bare peak-RSS line with a structured receipt.** In `run_index` (`src/main.rs`, ~line 439), the build report import + the final lines are: + +```rust + print!("{}", report.render()); + + // Realized peak RSS (per-run, informational) → stderr. + eprintln!("build peak RSS: {} MiB", peak_rss_bytes() / (1 << 20)); + Ok(()) +} +``` + +Replace those final lines (from `// Realized peak RSS` through `Ok(())`) with: + +```rust + // Build receipt: realized peak RSS vs the modeled n-scale SA-IS build memory — + // the D0 measure-first probe. The realized peak is machine-dependent; the + // breakdown + attribution ratio are the analysis payload. + let peak = peak_rss_bytes(); + let model = BuildMemoryModel::from_reference_len(total_bp); + let denom = total_bp.max(1); + eprintln!( + "build: realized peak RSS {} MiB ({} B/base) over {} bp", + peak / (1 << 20), + peak / denom, + total_bp + ); + eprint!("{}", model.render(total_bp)); + let ratio = if peak > 0 { + model.total_bytes as f64 / peak as f64 + } else { + 0.0 + }; + eprintln!( + "build: model/realized attribution = {:.2} [{}]", + ratio, + if ratio >= 0.70 { "CONFIRM ≥0.70" } else { "below 0.70" } + ); + Ok(()) +} +``` + +And add `BuildMemoryModel` to the `report` import. Find the existing `use` of the report items in `run_index` (the `estimate_build_working_set, render_plan_line, IndexBuildReport` import — at the top of `main.rs` or local to the file) and add `BuildMemoryModel`. (Check: `grep -n "estimate_build_working_set\|IndexBuildReport\|use rosalind::genomics::index" src/main.rs` and extend the matching `use` to include `BuildMemoryModel`.) + +- [ ] **Step 2: Build** + +Run: `cd ~/rosalind && cargo build 2>&1 | tail -5` +Expected: success, 0 warnings. + +- [ ] **Step 3: Smoke-test on the bundled toy** (the receipt renders + the build still works) + +Run: `cd ~/rosalind && cargo run -q -- index --reference examples/data/illumina_toy/reference.fa --output /tmp/d0toy.idx 2>&1 | tail -20` +Expected: the build report (contigs/total_bp/blake3/index_bytes) on stdout, then the build-memory model breakdown + `build: realized peak RSS … ` + the attribution line on stderr. (On a tiny toy the realized peak is dominated by the binary baseline, so the ratio will be tiny/`below 0.70` — that's expected for a toy; the gate is assessed on real genomes in Task 4.) + +- [ ] **Step 4: Run the existing index CLI gates** + +Run: `cd ~/rosalind && cargo test --test index_cli 2>&1 | tail -10` +Expected: PASS (the build receipt is additive; the deterministic stdout build report is unchanged). + +- [ ] **Step 5: Commit** + +```bash +cd ~/rosalind && git add src/main.rs && git commit -m "feat(cli): index emits a build receipt (realized peak vs modeled n-scale memory) (D0)" +``` + +--- + +## Task 3: The probe script + +**Files:** +- Create: `scripts/build_memory_probe.sh` +- Modify: `.gitignore` (verify `results/` already ignored) + +- [ ] **Step 1: Write the script.** Create `scripts/build_memory_probe.sh`: + +```bash +#!/usr/bin/env bash +# D0 measure-first probe: build the index for a small and a ~3x-larger REAL genome, +# capture each build receipt (realized peak RSS + modeled n-scale memory + bytes/base), +# and surface the pre-registered gate verdict. Build-only — no reads/alignment. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +RESULTS="$ROOT/results/build-probe" +mkdir -p "$RESULTS" +BIN="$ROOT/target/release/rosalind" +[ -x "$BIN" ] || (cd "$ROOT" && cargo build --release) + +# E. coli (gate genome) — reuse the Move-#5 cached reference if present. +ECOLI="$ROOT/results/flagship-ecoli/ecoli.fa" +if [ ! -s "$ECOLI" ]; then + ECOLI="$RESULTS/ecoli.fa" + if [ ! -s "$ECOLI" ]; then + curl -fsSL "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/005/845/GCF_000005845.2_ASM584v2/GCF_000005845.2_ASM584v2_genomic.fna.gz" -o "$RESULTS/ecoli.fa.gz" + gunzip -f "$RESULTS/ecoli.fa.gz" + fi +fi + +# S. cerevisiae R64 (~12.1 Mbp, ~3x; multi-contig — build handles it). +YEAST="$RESULTS/yeast.fa" +if [ ! -s "$YEAST" ]; then + curl -fsSL "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/146/045/GCF_000146045.2_R64/GCF_000146045.2_R64_genomic.fna.gz" -o "$RESULTS/yeast.fa.gz" + gunzip -f "$RESULTS/yeast.fa.gz" +fi + +SUMMARY="$RESULTS/SUMMARY.txt" +: > "$SUMMARY" +probe() { + local name="$1" ref="$2" + echo "================ $name ================" | tee -a "$SUMMARY" + "$BIN" index --reference "$ref" --output "$RESULTS/$name.idx" 2>&1 | tee -a "$SUMMARY" + echo | tee -a "$SUMMARY" +} +probe "ecoli" "$ECOLI" +probe "yeast" "$YEAST" + +echo ">> D0 gate (assess in the findings doc): realized peak within ±25% of ~185 MiB on E. coli" | tee -a "$SUMMARY" +echo ">> AND ≥70% model/realized attribution; bytes/base ~constant across E. coli & yeast = n-scale." | tee -a "$SUMMARY" +echo ">> Summary: $SUMMARY" +``` + +- [ ] **Step 2: Make executable + confirm gitignore.** + +```bash +cd ~/rosalind && chmod +x scripts/build_memory_probe.sh && git check-ignore results/build-probe/ecoli.fa && echo "(results ignored ✓)" +``` +(If `results/` is not ignored, append `/results/` to `.gitignore` — but the Move-#5 rule should already cover it.) + +- [ ] **Step 3: Commit** + +```bash +cd ~/rosalind && git add scripts/build_memory_probe.sh && git commit -m "feat(scripts): build_memory_probe.sh — D0 build-memory probe on real genomes (D0)" +``` + +--- + +## Task 4: Run the probe (produce the verdict numbers) + +**Files:** none (produces `results/build-probe/`, gitignored) + +- [ ] **Step 1: Run it.** (E. coli build is ~seconds; yeast ~12 Mbp build is the longer one — generous timeout. Downloads are cached after first run.) + +Run: `cd ~/rosalind && bash scripts/build_memory_probe.sh 2>&1 | tail -40` +Expected: a build report + build-memory model + realized peak + attribution for both E. coli and yeast. + +- [ ] **Step 2: Read the captured numbers + compute bytes/base scaling.** + +Run: `cd ~/rosalind && grep -E "realized peak RSS|attribution|^=====" results/build-probe/SUMMARY.txt` +Expected: the realized peak RSS + B/base + attribution ratio for each genome. Note for the findings doc: (a) E. coli realized peak vs ~185 MiB (±25%?), (b) the attribution ratio (≥0.70?), (c) E. coli B/base vs yeast B/base (≈ constant → n-scale → CONFIRM the build is SA-workspace-bound). + +(No commit — `results/` is gitignored.) + +--- + +## Task 5: Findings doc + `OPEN_PROBLEMS.md` reframing + +**Files:** +- Create: `docs/findings/2026-06-02-d0-build-memory-probe.md` +- Modify: `docs/OPEN_PROBLEMS.md` + +- [ ] **Step 1: Write the findings doc** with the **actual captured numbers** from Task 4. Sections: + 1. **Result + verdict (one line):** "E. coli build realized peak `` MiB (`` B/base); yeast `` MiB (`` B/base); model/realized attribution ``. Verdict: **CONFIRM/NULL**." + 2. **The pre-registered gate** (verbatim from the spec) and whether each criterion is met (±25% of 185 MiB; ≥70% attribution; bytes/base constant across the two genomes). + 3. **Table:** genome, bp, realized peak RSS, B/base, model total, attribution ratio. + 4. **Interpretation:** if CONFIRM — the build is intermediate-state-bound (the SA/text/workspace arrays), the baseline curve point (b=n, full-RAM) is set, **D1 (blocked external-memory SA construction) is greenlit**. If NULL — what dominates instead, and the re-scope. + 5. **Reproduce:** `bash scripts/build_memory_probe.sh`. Honest note: realized peak is machine-dependent; the model is a code-grounded peak-set estimate. + +- [ ] **Step 2: Reframe `OPEN_PROBLEMS.md`.** Add a short, dated note at the top of the §3.2 "beachhead" / §5-D section (do not rewrite the whole doc) stating the accepted pivot: the √t/Cook–Mertz machinery is **theoretical framing** for "recomputation along a √-shaped space/time curve," **not** the literal construction kernel (SA-IS produces a permutation with no low-degree-extension structure; Cook–Mertz is super-polynomial with no systems realization); Phase D's mechanism is a **native budget-tunable external-memory SA/BWT constructor** honoring a declared `MemoryBudget` along a measured curve, wrapped in the `plan`/`--enforce`/`verify` contract; the differentiation is the **contract + curve + verifiable receipt**, not a new complexity bound. Link the D0 findings doc. + +- [ ] **Step 3: Commit** + +```bash +cd ~/rosalind && git add docs/findings/2026-06-02-d0-build-memory-probe.md docs/OPEN_PROBLEMS.md && git commit -m "docs(findings): D0 build-memory probe verdict + OPEN_PROBLEMS √t-as-framing reframing (D0)" +``` + +--- + +## Task 6: Final verification + +**Files:** none + +- [ ] **Step 1: Format + zero-warning builds.** + +Run: `cd ~/rosalind && cargo fmt --all && cargo fmt --all -- --check && echo FMT_CLEAN; cargo build 2>&1 | grep -c "warning:"; cargo build --release 2>&1 | grep -c "warning:"` +Expected: `FMT_CLEAN`; `0`; `0`. + +- [ ] **Step 2: Full suite + tree clean of regenerated data.** + +Run: `cd ~/rosalind && cargo test 2>&1 | grep -E "FAILED|panicked|[1-9][0-9]* failed" || echo "no failures"; git status --short` +Expected: no failures; no `results/` tracked. + +- [ ] **Step 3: Commit any fmt fixups** + +```bash +cd ~/rosalind && git add -A && git commit -m "style: rustfmt fixups (D0)" || true +``` + +--- + +## Self-Review notes + +- **Spec coverage:** §2 accountant → Task 1; §3 build receipt → Task 2; §4 probe script → Task 3; §4 run → Task 4; §5 findings + §6 OPEN_PROBLEMS reframing → Task 5; §7 testing → Tasks 1/2/6. +- **Placeholder scan:** `//` in Task 5 are real numbers captured at run time (Task 4) for the findings doc — not plan placeholders. All code steps are complete. +- **Type consistency:** `BuildMemoryModel::from_reference_len(n) -> Self` with `.components: Vec<(String,u64)>`, `.total_bytes: u64`, `.render(total_bp)` — defined in Task 1, consumed in Task 2. The accountant constants (4 B SA elem, 8 B usize, 1 B types, n/2 LMS) are stated once and reused. +- **Honesty:** the model is a code-grounded enumeration (Task 1 comment + the guardrail at top), not tuned to the gate; the robust CONFIRM signal is bytes/base-constant-across-genomes (Task 4/5). A NULL is an acceptable, recorded outcome. From 29d6ac1b0f58717602f5cf415d85361cbbcd4072 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 20:02:03 -0700 Subject: [PATCH 3/7] =?UTF-8?q?feat(index):=20BuildMemoryModel=20=E2=80=94?= =?UTF-8?q?=20code-grounded=20n-scale=20SA-IS=20build=20accountant=20(D0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/genomics/index/report.rs | 100 +++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/src/genomics/index/report.rs b/src/genomics/index/report.rs index eb62672..0ac9660 100644 --- a/src/genomics/index/report.rs +++ b/src/genomics/index/report.rs @@ -25,6 +25,77 @@ pub fn estimate_build_working_set(reference_len: u64) -> WorkingSet { } } +/// A code-grounded model of the **peak simultaneously-live n-scale memory** of the +/// SA-IS index build, broken down by the arrays `sais_impl`/`induce_sort`/the +/// FM-index actually allocate. Element sizes are the real Rust types: the SA-IS +/// input text and the suffix array are `u32`/`i32` (4 B); the LMS index vectors +/// are `usize` (8 B); `types` is one byte. `#LMS ≤ n/2` is modeled at n/2. The +/// recursion (`sais_impl(&reduced, …)`) keeps the parent's text/types/lms arrays +/// live while a ~n/2-size child runs, so its overhead is modeled as a geometric +/// tail (~1× the parent's live n-scale arrays). This is a peak-SET estimate, not a +/// sum of every allocation ever made — D0 MEASURES whether it captures the +/// realized peak (the gate); it is not tuned to the gate. +#[derive(Debug, Clone)] +pub struct BuildMemoryModel { + /// Per-component `(name, bytes)` of the modeled peak set. + pub components: Vec<(String, u64)>, + /// Sum of the components. + pub total_bytes: u64, +} + +impl BuildMemoryModel { + /// Model the peak n-scale build memory for a reference of `n` bases. + pub fn from_reference_len(n: u64) -> Self { + const SA_ELEM: u64 = 4; // u32 / i32 suffix-array + text element + const USIZE: u64 = 8; // LMS index vectors are Vec + let lms = n / 2; // #LMS ≤ n/2 (upper-ish) + let comps: Vec<(&str, u64)> = vec![ + ("text(u32)", n.saturating_mul(SA_ELEM)), + ("types(1B)", n), + ("lms_positions(usize)", lms.saturating_mul(USIZE)), + ("induce-sort suffix array(i32)", n.saturating_mul(SA_ELEM)), + ("lms_in_sa_order(usize)", lms.saturating_mul(USIZE)), + ("lms_name(u32)", n.saturating_mul(SA_ELEM)), + ("reduced string(u32)", lms.saturating_mul(SA_ELEM)), + ("returned suffix array(u32)", n.saturating_mul(SA_ELEM)), + ("fm-index (bwt + rank + C-table)", n.saturating_mul(3)), + ]; + let level0: u64 = comps.iter().fold(0u64, |acc, (_, b)| acc.saturating_add(*b)); + // Recursion: parent text/types/lms_positions/lms_name/reduced stay live + // (~text 4n + types 1n + lms_positions 4n + lms_name 4n + reduced 2n = 15n) + // while a ~n/2 child runs; the geometric tail ≈ that parent-live amount. + let recursion = n.saturating_mul(15); + let mut components: Vec<(String, u64)> = + comps.into_iter().map(|(s, b)| (s.to_string(), b)).collect(); + components.push(("recursion (geometric tail)".to_string(), recursion)); + let total_bytes = level0.saturating_add(recursion); + Self { + components, + total_bytes, + } + } + + /// Render the model as a deterministic multi-line breakdown (bytes + per-base). + pub fn render(&self, total_bp: u64) -> String { + let mut out = String::from("build memory model (n-scale peak set):\n"); + let denom = total_bp.max(1); + for (name, bytes) in &self.components { + out.push_str(&format!( + " {name:<34} {:>6} MiB ({} B/base)\n", + bytes / (1 << 20), + bytes / denom + )); + } + out.push_str(&format!( + " {:-<34} {:>6} MiB ({} B/base)\n", + "total ", + self.total_bytes / (1 << 20), + self.total_bytes / denom + )); + out + } +} + /// The deterministic build receipt for a persisted index. Per-run fields (e.g. /// realized RSS) are intentionally excluded — the caller prints those separately. #[derive(Debug, Clone)] @@ -132,4 +203,33 @@ mod tests { assert!(render_plan_line(over, budget).ends_with("[OVER]")); assert!(render_plan_line(under, budget).contains("budget 100 MiB")); } + + #[test] + fn build_model_breaks_down_and_sums() { + let m = BuildMemoryModel::from_reference_len(1_000_000); + let sum: u64 = m.components.iter().map(|(_, b)| b).sum(); + assert_eq!(sum, m.total_bytes, "components must sum to total"); + let names: Vec<&str> = m.components.iter().map(|(n, _)| n.as_str()).collect(); + for needed in ["text(u32)", "suffix array", "lms_name"] { + assert!( + names.iter().any(|n| n.contains(needed)), + "missing component {needed}: {names:?}" + ); + } + // Honest sanity: SA-IS over a u32 text is many bytes/base, not a handful. + assert!( + m.total_bytes >= 20_000_000, + "model must be ≥20 B/base (~{} B/base)", + m.total_bytes / 1_000_000 + ); + } + + #[test] + fn build_model_is_monotonic_and_overflow_safe() { + assert!( + BuildMemoryModel::from_reference_len(2_000_000).total_bytes + > BuildMemoryModel::from_reference_len(1_000_000).total_bytes + ); + let _ = BuildMemoryModel::from_reference_len(u64::MAX); // must not panic + } } From 58bd50861e15a69944b06b54ea4f67390a60b87e Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 20:03:54 -0700 Subject: [PATCH 4/7] feat(cli): index emits a build receipt (realized peak vs modeled n-scale memory) (D0) --- src/genomics/index/mod.rs | 2 +- src/genomics/mod.rs | 4 ++-- src/main.rs | 29 +++++++++++++++++++++++++---- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/genomics/index/mod.rs b/src/genomics/index/mod.rs index c7d073f..fec81af 100644 --- a/src/genomics/index/mod.rs +++ b/src/genomics/index/mod.rs @@ -12,5 +12,5 @@ mod view; pub use format::IndexHeader; pub use io::{IndexReader, IndexWriter, ReferenceIndex}; -pub use report::{estimate_build_working_set, render_plan_line, IndexBuildReport}; +pub use report::{estimate_build_working_set, render_plan_line, BuildMemoryModel, IndexBuildReport}; pub use view::{FmIndexView, GenomeIndexView, ReferenceView}; diff --git a/src/genomics/mod.rs b/src/genomics/mod.rs index 9e5f625..5980b17 100644 --- a/src/genomics/mod.rs +++ b/src/genomics/mod.rs @@ -34,8 +34,8 @@ pub use fm_index::{ }; pub use genome_index::{GenomeIndex, GenomeIndexError, MAX_GENOME_LEN}; pub use index::{ - estimate_build_working_set, render_plan_line, FmIndexView, GenomeIndexView, IndexBuildReport, - IndexHeader, IndexReader, IndexWriter, ReferenceIndex, ReferenceView, + estimate_build_working_set, render_plan_line, BuildMemoryModel, FmIndexView, GenomeIndexView, + IndexBuildReport, IndexHeader, IndexReader, IndexWriter, ReferenceIndex, ReferenceView, }; pub use io::create_bam_writer; pub use pileup::{PileupNode, PileupProcessor, PileupSummary, PileupWorkload}; diff --git a/src/main.rs b/src/main.rs index 47705a8..0d9688c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,8 +9,8 @@ use clap::{Parser, Subcommand, ValueEnum}; use rosalind::core::MemoryBudget; use rosalind::genomics::{ compare_callsets, create_bam_writer, estimate_build_working_set, read_vcf_variants, - render_plan_line, sort_bam_deterministic, AlignedRead, BWTAligner, BedIndex, CigarOp, - CigarOpKind, GenomeIndex, IndexBuildReport, IndexReader, IndexWriter, + render_plan_line, sort_bam_deterministic, AlignedRead, BWTAligner, BedIndex, BuildMemoryModel, + CigarOp, CigarOpKind, GenomeIndex, IndexBuildReport, IndexReader, IndexWriter, }; use rosalind::io::decompress::open_input; use rosalind::io::fasta::{FastaReader, FastaRecord}; @@ -484,8 +484,29 @@ fn run_index(reference: PathBuf, output: PathBuf, memory_budget_mb: Option) }; print!("{}", report.render()); - // Realized peak RSS (per-run, informational) → stderr. - eprintln!("build peak RSS: {} MiB", peak_rss_bytes() / (1 << 20)); + // Build receipt: realized peak RSS vs the modeled n-scale SA-IS build memory — + // the D0 measure-first probe. The realized peak is machine-dependent; the + // breakdown + attribution ratio are the analysis payload. + let peak = peak_rss_bytes(); + let model = BuildMemoryModel::from_reference_len(total_bp); + let denom = total_bp.max(1); + eprintln!( + "build: realized peak RSS {} MiB ({} B/base) over {} bp", + peak / (1 << 20), + peak / denom, + total_bp + ); + eprint!("{}", model.render(total_bp)); + let ratio = if peak > 0 { + model.total_bytes as f64 / peak as f64 + } else { + 0.0 + }; + eprintln!( + "build: model/realized attribution = {:.2} [{}]", + ratio, + if ratio >= 0.70 { "CONFIRM ≥0.70" } else { "below 0.70" } + ); Ok(()) } From 7e20bbb21bb174b58fd5166fa8c0ea390577ce60 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 20:04:17 -0700 Subject: [PATCH 5/7] =?UTF-8?q?feat(scripts):=20build=5Fmemory=5Fprobe.sh?= =?UTF-8?q?=20=E2=80=94=20D0=20build-memory=20probe=20on=20real=20genomes?= =?UTF-8?q?=20(D0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/build_memory_probe.sh | 43 +++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100755 scripts/build_memory_probe.sh diff --git a/scripts/build_memory_probe.sh b/scripts/build_memory_probe.sh new file mode 100755 index 0000000..cd699f9 --- /dev/null +++ b/scripts/build_memory_probe.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# D0 measure-first probe: build the index for a small and a ~3x-larger REAL genome, +# capture each build receipt (realized peak RSS + modeled n-scale memory + bytes/base), +# and surface the pre-registered gate verdict. Build-only — no reads/alignment. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +RESULTS="$ROOT/results/build-probe" +mkdir -p "$RESULTS" +BIN="$ROOT/target/release/rosalind" +[ -x "$BIN" ] || (cd "$ROOT" && cargo build --release) + +# E. coli (gate genome) — reuse the Move-#5 cached reference if present. +ECOLI="$ROOT/results/flagship-ecoli/ecoli.fa" +if [ ! -s "$ECOLI" ]; then + ECOLI="$RESULTS/ecoli.fa" + if [ ! -s "$ECOLI" ]; then + curl -fsSL "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/005/845/GCF_000005845.2_ASM584v2/GCF_000005845.2_ASM584v2_genomic.fna.gz" -o "$RESULTS/ecoli.fa.gz" + gunzip -f "$RESULTS/ecoli.fa.gz" + fi +fi + +# S. cerevisiae R64 (~12.1 Mbp, ~3x; multi-contig — build handles it). +YEAST="$RESULTS/yeast.fa" +if [ ! -s "$YEAST" ]; then + curl -fsSL "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/146/045/GCF_000146045.2_R64/GCF_000146045.2_R64_genomic.fna.gz" -o "$RESULTS/yeast.fa.gz" + gunzip -f "$RESULTS/yeast.fa.gz" +fi + +SUMMARY="$RESULTS/SUMMARY.txt" +: > "$SUMMARY" +probe() { + local name="$1" ref="$2" + echo "================ $name ================" | tee -a "$SUMMARY" + "$BIN" index --reference "$ref" --output "$RESULTS/$name.idx" 2>&1 | tee -a "$SUMMARY" + echo | tee -a "$SUMMARY" +} +probe "ecoli" "$ECOLI" +probe "yeast" "$YEAST" + +echo ">> D0 gate (assess in the findings doc): realized peak within +/-25% of ~185 MiB on E. coli" | tee -a "$SUMMARY" +echo ">> AND >=70% model/realized attribution; bytes/base ~constant across E. coli & yeast = n-scale." | tee -a "$SUMMARY" +echo ">> Summary: $SUMMARY" From b4d3bc7885b39268a412bed9731274029ba599fc Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 20:06:20 -0700 Subject: [PATCH 6/7] =?UTF-8?q?docs(findings):=20D0=20build-memory=20probe?= =?UTF-8?q?=20=E2=80=94=20CONFIRM=20(build=20is=20SA-workspace-bound)=20+?= =?UTF-8?q?=20OPEN=5FPROBLEMS=20=E2=88=9At-as-framing=20reframing=20(D0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/OPEN_PROBLEMS.md | 16 +++++ .../2026-06-02-d0-build-memory-probe.md | 66 +++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 docs/findings/2026-06-02-d0-build-memory-probe.md diff --git a/docs/OPEN_PROBLEMS.md b/docs/OPEN_PROBLEMS.md index c2488b3..c551b45 100644 --- a/docs/OPEN_PROBLEMS.md +++ b/docs/OPEN_PROBLEMS.md @@ -69,6 +69,22 @@ things this framework will magically solve. ### 3.2 The beachhead: sublinear-space full-text index construction +> **Pivot — Phase D mechanism (2026-06-02; see [`findings/2026-06-02-d0-build-memory-probe.md`](findings/2026-06-02-d0-build-memory-probe.md)).** +> A scoping study established that the literal √t / Cook–Mertz machinery is **theoretical framing**, not the +> construction *kernel*: SA-IS produces a permutation (random-access induced sorting) whose memory is the +> suffix array itself — there is no low-degree-extension structure to compress, the in-repo combiner is a +> stub whose output is discarded on the correctness path, and Cook–Mertz is super-polynomial with no systems +> realization. So Phase D's **mechanism** is a **native, polynomial-time, budget-tunable external-memory +> SA/BWT constructor** (pSAscan/eSAIS/Big-BWT-style: SA-IS each budget-sized block in RAM → exact disk-backed +> merge) that honors a declared `MemoryBudget` along a **measured** space/time curve, **degrading to disk +> rather than refusing**, wrapped in the shipped `plan`/`--enforce`/`verify` contract. The √t bound (Williams +> 2025; Cook–Mertz 2024) is kept as the asymptotic *justification* for "recomputation along a √-shaped +> curve" — **not** as the runtime kernel, and Rosalind makes **no new space-complexity claim** (prior +> succinct-construction work already hits low-space fixed points). The defensible contribution is the +> **declared-budget contract + measured curve + verifiable receipt over a real memory-bound build.** The D0 +> probe (above link) confirmed the build is intermediate-state-bound by the SA workspace (41 B/base, constant +> across E. coli and yeast), greenlighting this path. + Building the FM-index/BWT is Rosalind's own headline memory limitation ("the FM-index is built in memory at the start of each run; memory proportional to the reference"). It is the ideal first target because it is: diff --git a/docs/findings/2026-06-02-d0-build-memory-probe.md b/docs/findings/2026-06-02-d0-build-memory-probe.md new file mode 100644 index 0000000..02612d5 --- /dev/null +++ b/docs/findings/2026-06-02-d0-build-memory-probe.md @@ -0,0 +1,66 @@ +# D0 — measure-first build-memory probe (the Phase-D gate) + +**2026-06-02.** Before committing the Phase-D external-memory construction arc, a cheap, decisive +measure-first probe: is the index *build* peak actually dominated by the SA-IS workspace (so a +budget-tunable / blocked build can reduce it), or by something a block-tunable build wouldn't touch? + +**Verdict: CONFIRM.** On the real *E. coli* K-12 MG1655 chromosome (4,641,652 bp), the realized build peak +is **182 MiB** — within **1.6%** of the audited ~185 MiB (gate: ±25%) — at **41 bytes/base**. On +*S. cerevisiae* R64 (12,157,105 bp, 17 contigs) it is **482 MiB**, also **41 bytes/base**. The bytes/base is +**identical across a 2.6× size jump**, and a code-grounded model of the n-scale SA-IS arrays envelopes the +realized peak (model/realized = 1.09 / 1.08). The build is **intermediate-state-bound** by the +suffix-array workspace, exactly as `docs/OPEN_PROBLEMS.md` §3.1 claimed → **D1 (blocked external-memory SA +construction) is greenlit**, with the baseline curve point set at **b = n (full-RAM) = 41 B/base**. + +## The pre-registered gate (and the result) + +| Criterion | Threshold | Result | | +|---|---|---|---| +| E. coli realized peak vs audited ~185 MiB | within ±25% | **182 MiB (−1.6%)** | ✅ | +| model/realized attribution (peak ⊆ modeled SA arrays) | ≥ 0.70 | **1.09** (model ≥ realized) | ✅ | +| bytes/base constant across genomes (peak is n-scale, not fixed overhead) | ≈ constant | **41 → 41** (E. coli → yeast) | ✅ | + +All three met. The peak is the SA construction workspace; it scales with reference length, not with a +fixed baseline — so reducing the *simultaneously-live* SA workspace (the D1+ external-memory/blocked plan) +will reduce the realized build peak. + +## The numbers + +| Genome | bp | realized peak RSS | B/base | model total | model B/base | attribution | +|---|---|---|---|---|---|---| +| *E. coli* K-12 MG1655 (NC_000913.3) | 4,641,652 | 182 MiB | 41 | 199 MiB | 45 | 1.09 | +| *S. cerevisiae* R64 (17 contigs) | 12,157,105 | 482 MiB | 41 | 521 MiB | 44 | 1.08 | + +The code-grounded `BuildMemoryModel` (from the real `sais_impl` allocations) breaks the ~45 B/base down as: +`u32` text 4 + `types` 1 + `lms_positions` (usize) 4 + induce-sort SA (i32) 4 + `lms_in_sa_order` (usize) 4 + +`lms_name` 4 + reduced string 2 + returned SA 4 + FM-index 3 + recursion (geometric tail) 15. The realized +41 B/base sitting just under the 45 B/base model confirms these arrays are the peak set (not, say, allocator +slack, the FM-index rank bitvectors, or the persistence buffers — any of which would have produced a +realized peak well above the n-scale model, i.e. a NULL). + +## What this greenlights (D1+) + +The realized peak is the **simultaneously-live** SA-IS workspace at full RAM (b = n). The D1 plan — SA-IS +each budget-sized **block** of the text in RAM, then an exact disk-backed merge into the full SA/BWT — keeps +only one block's workspace live at a time, so a declared `MemoryBudget` selects the block size and the +realized peak drops along a measured space/time curve (~pSAscan's O(n²/M) shape), degrading to disk rather +than refusing, wrapped in the shipped `plan`/`--enforce`/`verify` contract. D0 is the baseline every later +increment must beat. + +## Honest notes + +- The realized `peak_rss` is **machine-dependent**; the model is a code-grounded *peak-set* estimate, not a + byte-exact predictor. What is robust is the **bytes/base constancy** (n-scale) and that the realized peak + sits **within** the modeled SA-workspace envelope on two genomes. +- This is **not** a new complexity-bound claim and **not** the √t simulation made real — see the Phase-D + reframing in [`../OPEN_PROBLEMS.md`](../OPEN_PROBLEMS.md). The differentiation is the contract + measured + curve + verifiable receipt over a real memory-bound build. + +## Reproduce + +```bash +bash scripts/build_memory_probe.sh # builds E. coli + yeast, prints both build receipts + the gate +``` + +Build-only (no reads/alignment). Downloads cached after the first run; regenerated data lands in +`results/build-probe/` (gitignored). The build receipt is emitted by every `rosalind index` run. From f68935694785e5baaf4a194ce385947aa35d0a74 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Mon, 1 Jun 2026 20:07:18 -0700 Subject: [PATCH 7/7] style: rustfmt fixups (D0) --- src/genomics/index/mod.rs | 4 +++- src/genomics/index/report.rs | 4 +++- src/main.rs | 6 +++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/genomics/index/mod.rs b/src/genomics/index/mod.rs index fec81af..32b78c3 100644 --- a/src/genomics/index/mod.rs +++ b/src/genomics/index/mod.rs @@ -12,5 +12,7 @@ mod view; pub use format::IndexHeader; pub use io::{IndexReader, IndexWriter, ReferenceIndex}; -pub use report::{estimate_build_working_set, render_plan_line, BuildMemoryModel, IndexBuildReport}; +pub use report::{ + estimate_build_working_set, render_plan_line, BuildMemoryModel, IndexBuildReport, +}; pub use view::{FmIndexView, GenomeIndexView, ReferenceView}; diff --git a/src/genomics/index/report.rs b/src/genomics/index/report.rs index 0ac9660..ab70a0b 100644 --- a/src/genomics/index/report.rs +++ b/src/genomics/index/report.rs @@ -60,7 +60,9 @@ impl BuildMemoryModel { ("returned suffix array(u32)", n.saturating_mul(SA_ELEM)), ("fm-index (bwt + rank + C-table)", n.saturating_mul(3)), ]; - let level0: u64 = comps.iter().fold(0u64, |acc, (_, b)| acc.saturating_add(*b)); + let level0: u64 = comps + .iter() + .fold(0u64, |acc, (_, b)| acc.saturating_add(*b)); // Recursion: parent text/types/lms_positions/lms_name/reduced stay live // (~text 4n + types 1n + lms_positions 4n + lms_name 4n + reduced 2n = 15n) // while a ~n/2 child runs; the geometric tail ≈ that parent-live amount. diff --git a/src/main.rs b/src/main.rs index 0d9688c..97a8be9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -505,7 +505,11 @@ fn run_index(reference: PathBuf, output: PathBuf, memory_budget_mb: Option) eprintln!( "build: model/realized attribution = {:.2} [{}]", ratio, - if ratio >= 0.70 { "CONFIRM ≥0.70" } else { "below 0.70" } + if ratio >= 0.70 { + "CONFIRM ≥0.70" + } else { + "below 0.70" + } ); Ok(()) }