diff --git a/card_engine/src/cost.rs b/card_engine/src/cost.rs index e0ff55ae5..c3d95db99 100644 --- a/card_engine/src/cost.rs +++ b/card_engine/src/cost.rs @@ -160,6 +160,20 @@ pub(crate) struct PlanFeatures { pub limit: u32, /// Page offset. pub offset: u32, + /// The permutation segment `StreamedSelect`'s emission walk is actually bounded to — + /// `walk_bounds(...).len()` for `(sort_col, descending)`, computed once at acquire from the SAME + /// inputs the executor uses (`QueryParams::sort_bound`, the filter's own interval on the sort + /// column). `n_cards` when the filter constrains nothing about the sort column, and also when no + /// permutation exists for this `(sort_col, descending)` pair — `StreamedSelect` is inapplicable + /// there and never reads this field, but `mk_plan_feats` sets it uniformly for every acquire + /// branch (the shared feats have to cost a competing `StreamedSelect` honestly regardless of which + /// branch produced them), so the fallback must still be a value, not an absent one. + /// + /// Round 32 of the printing-varying-depth ledger + /// (docs/issues/local-engine-gathered-scan-card-printing-varying-depth.md): `perm_steps` used to + /// multiply by `n_cards` unconditionally, which is right only for the unbounded case. See + /// `perm_steps`'s own doc for the regrade this field closes. + pub perm_walk_span: u32, /// Printings the legality **broadcast-down** synthesizes (card ∃-plane → printing bitmap) in /// `PrintingCompose`. `0` for border/rarity (precomputed planes) and for bare ranges (no broadcast). /// Costed at `LINEAR_PASS_PER_PRINTING_NS`. @@ -210,6 +224,11 @@ pub(crate) struct PlanFeatures { /// in ~`page_span/selectivity` steps), while the permutation-free gather visits every match — so /// the formula branches on this rather than assuming one. Ignored by every other plan. pub compose_paging: super::ComposePaging, + /// Printings a CARD-SPACE collection leaf's build broadcasts (`ids_of` + + /// `broadcast_card_ids_to_printings`). See `ComposeEstimate::collection_broadcast`'s doc for why + /// this is not just folded into `scatter_printings`. `0` for everything except `PrintingCompose` + /// on a card-space `subtypes`/`keywords`/`oracle_tags` leaf. + pub collection_broadcast_printings: u32, } // ─── P1: PrintingRangeScan ────────────────────────────────────────────────── @@ -610,6 +629,34 @@ const GATHER_COLLECT_PER_PAGE_ROW_NS: f64 = 9.79; /// and drift as either query sizes or the corpus change. The fix is a term for the curvature -- see the /// corpus-size note in `bench_gather_loop` -- not a smaller constant. const GATHER_FIXED_COST_NS: f64 = 169.6; +/// `GATHER_FIXED_COST_NS`'s own value when `matches == 0` -- a `Prep::Candidates`-acquired zero-match +/// round, where every other term in this arm is already provably zero (`eval_domain`, `scan_units`, +/// `page_span`/`page_rows`, `artwork_seen_printings` all vanish with the candidate list itself), so +/// the whole prediction collapses to this one constant alone. `GATHER_FIXED_COST_NS` was fit against +/// the general population and reads 169.6 there; a zero-candidate round pays none of the loop/verify +/// work that constant was priced to cover, so charging it here is a straight ~4x over-charge, not a +/// rounding difference. +/// +/// Fit as the calibration half's median measured `plan_self_ns` (not a mean, and not per-mode -- +/// `PlanFeatures` carries no `unique`/mode field this arm can read, so one pooled constant is what +/// this branch can express; see the doc issue's Round 9 section for the residual mode split this +/// leaves on the table for card/printing vs. artwork). 9,890 sampled `GatheredScan`/`candidates` +/// zero-match rows (31.9% of the sampled `candidates` population), hash-of-query split: +/// +/// calibration (n=4,944): median measured_ns = 42.0 -> this constant +/// held-out (n=4,946): 4,577 improved / 369 regressed / 0 tied +/// total abs ns error 530,256 -> 103,110 (5.1x) +/// median ratio (measured/predicted) 0.248 -> 1.000 +/// within-25% 0.1% -> 57.7% +/// +/// The held-out gain is not uniform across mode: card/printing land almost exactly on 1.00 (83-90% +/// within 25%), while artwork's real zero-match cost reads a flat ~2x higher (84ns vs. card/printing's +/// ~42ns -- plausibly `exec_gathered_scan`'s unconditional per-printing dedupe check setup, per its own +/// comment on `artwork_seen_printings` above), so artwork's ratio moves from 0.495 (over-cost) to 2.0 +/// (under-cost) -- roughly the same LOG-ratio magnitude, just flipped sign, and still a net win on +/// absolute ns error (|84-169.6| = 85.6 -> |84-42| = 42.0). Splitting this properly by mode needs a +/// `PlanFeatures` field this arm does not have; out of scope for a `cost.rs`-only round. +const GATHER_FIXED_COST_ZERO_MATCH_NS: f64 = 42.0; // --- PrintingCompose's own rates ------------------------------------------------------------- // @@ -637,6 +684,22 @@ const GATHER_FIXED_COST_NS: f64 = 169.6; pub(crate) const COMPOSE_LINEAR_PASS_PER_PRINTING_NS: f64 = 1.93; /// Range-slice scatter into the printing bitmap during build. pub(crate) const COMPOSE_SCATTER_PER_PRINTING_NS: f64 = 0.48; + +/// A CARD-SPACE collection leaf's build (`ids_of` + `broadcast_card_ids_to_printings`) used to ride +/// `COMPOSE_SCATTER_PER_PRINTING_NS`, on the assumption that it was the same shape of operation as a +/// range's contiguous slice-scatter. It measurably is not: a card-cursor lookup per id (`offsets[c]`/ +/// `offsets[c+1]`) plus a variable-width printing-range fill, against a range's single contiguous +/// write. +/// +/// Backed out of `otag:triggered-ability`/`otag:cycle`/`otag:activated-ability` (`unique=printing`, +/// EDHREC): with `printings_walked` corrected (`WalkCheckpoints`) and every other term in +/// `PhysicalPlan::PrintingCompose`'s formula computed from measured features, the residual against real +/// wall time scaled cleanly with `collection_broadcast_printings` (not a flat offset), implying 1.41, +/// 1.30, and 1.31 ns/printing -- tight enough (2.7-2.9x `COMPOSE_SCATTER_PER_PRINTING_NS`, all three +/// within 8% of each other) to be a real rate and not sampling noise, but from 3 points on one corpus +/// size, not the corpus-scaling sweep the rates above this comment were fit with. Revisit if a wider +/// measurement disagrees. +pub(crate) const COMPOSE_COLLECTION_BROADCAST_PER_PRINTING_NS: f64 = 1.34; /// Result-space bitmap words popcounted for the total. const COMPOSE_POPCOUNT_PER_WORD_NS: f64 = 1.07; /// Per printing stepped over by the Perm / OrderbyWalk page fill. @@ -758,6 +821,7 @@ pub(crate) fn plan_cost(plan: PhysicalPlan, f: &PlanFeatures) -> f64 { PhysicalPlan::PrintingCompose => { let build = f64::from(f.broadcast_printings) * COMPOSE_LINEAR_PASS_PER_PRINTING_NS // legality broadcast-down into the printing bitmap (border/rarity read a plane → 0) + f64::from(f.scatter_printings) * COMPOSE_SCATTER_PER_PRINTING_NS // range-slice scatter into the printing bitmap (cheap: no card cursor) + + f64::from(f.collection_broadcast_printings) * COMPOSE_COLLECTION_BROADCAST_PER_PRINTING_NS // card-space collection leaf's build (ids_of + broadcast_card_ids_to_printings) — a card cursor per id, pricier than a range's contiguous scatter + f64::from(f.project_printings) * COMPOSE_LINEAR_PASS_PER_PRINTING_NS // second pass: project printing→card/artwork (0 for printing mode) — the pass CardRangePopcount fuses away + f64::from(f.popcount_words) * COMPOSE_POPCOUNT_PER_WORD_NS; // popcount the result-space bitmap for the total (printing/card/artwork words) let page = match f.compose_paging { @@ -833,19 +897,25 @@ pub(crate) fn plan_cost(plan: PhysicalPlan, f: &PlanFeatures) -> f64 { let walks_permutation = !runs_small_gather && f.matches > 0 && u64::from(f.offset) < u64::from(f.matches); let perm_steps = if walks_permutation { // Entries visited to accumulate `page_span` matches, when matches are spread uniformly - // through the permutation: one match per `n_cards / matches` entries. Bounded by the - // corpus, since the walk cannot step past the end of the permutation. + // through the WALKED SEGMENT: one match per `perm_walk_span / matches` entries. Bounded + // by that segment, since the walk cannot step past its end. + // + // The executor starts and ends the walk at the segment its filter's bound on the SORT + // COLUMN admits (`walk_bounds`), and `perm_walk_span` is that same segment's length, + // computed once at acquire by `mk_plan_feats` calling the identical `walk_bounds` helper + // over the identical `QueryParams::sort_bound` the executor reads -- not re-derived by a + // second path that could silently disagree with what dispatch actually walks. Before + // Round 32 this multiplied by `n_cards` unconditionally, which is right only when the + // filter constrains nothing about the sort column; `perm_walk_span` already collapses to + // `n_cards` in exactly that case (and when no permutation exists at all), so this is a + // strict generalization, not a second code path with its own edge cases. // - // The executor now starts and ends the walk at the segment its filter's bound on the - // SORT COLUMN admits (`walk_bounds`), which this cannot see -- no `PlanFeatures` field - // carries the filter's shape. The uniform-spread assumption absorbs it: the expected gap - // before the first match is one `n_cards / matches` stride, negligible against - // `page_span` of them. What the regrade showed is that the assumption's remaining error - // is a DIFFERENT shape. Realized/estimated over ~12.5k walking rows, same seed and - // sample length: + // Realized/estimated `perm_steps` over ~12.5k walking rows, same seed and sample length + // (docs/issues/local-engine-gathered-scan-card-printing-varying-depth.md, Round 32 for + // the held-out re-check against current traffic): // - // unbounded walk p10 0.13 median 1.00 p90 6.43 - // sort-column bound p10 0.11 median 0.96 p90 5.31 + // unbounded walk (pre-Round-32) p10 0.13 median 1.00 p90 6.43 + // sort-column bound (shipped) p10 0.11 median 0.96 p90 5.31 // realized inv_perm min/max p10 0.08 median 0.90 p90 4.26 // // The third row is not shipped -- it cost 0.51 ns per matching card -- but it bounds how @@ -854,7 +924,8 @@ pub(crate) fn plan_cost(plan: PhysicalPlan, f: &PlanFeatures) -> f64 { // catches only what the predicate names. What is left in BOTH is non-matching entries // INTERIOR to the walked segment, which no start position reaches by construction. That // is the popcount-skip mechanism's territory. - (page_span * n_cards / f64::from(f.matches)).min(n_cards) + let perm_walk_span = f64::from(f.perm_walk_span); + (page_span * perm_walk_span / f64::from(f.matches)).min(perm_walk_span) } else { 0.0 }; @@ -898,7 +969,7 @@ pub(crate) fn plan_cost(plan: PhysicalPlan, f: &PlanFeatures) -> f64 { + page_span * GATHER_SELECT_PER_PAGE_SLOT_NS + page_rows * GATHER_COLLECT_PER_PAGE_ROW_NS + f64::from(f.artwork_seen_printings) * GATHER_ARTWORK_PER_PRINTING_NS - + GATHER_FIXED_COST_NS + + if matches > 0.0 { GATHER_FIXED_COST_NS } else { GATHER_FIXED_COST_ZERO_MATCH_NS } } } } diff --git a/card_engine/src/lib.rs b/card_engine/src/lib.rs index 4860fa351..6de37639d 100644 --- a/card_engine/src/lib.rs +++ b/card_engine/src/lib.rs @@ -1725,6 +1725,53 @@ fn arith_tuple_narrow(filter: &FilterExpr, idx: &Archived, n_ca type TagIndex = HashMap>; +/// One O(n_printings) pass at load time, the same cost class as `set_codes`'s own build just above +/// it (not a second full-corpus scan on top of an existing one — this and `set_codes` are built from +/// two independent loops over `printings` today, each already O(n_printings) on its own). See +/// `SetCollectorRange`'s own doc for what this feeds and why. +fn build_set_collector_ranges(printings: &[T], set_code: impl Fn(&T) -> &str, collector_number_int: impl Fn(&T) -> Option) -> HashMap { + let mut ranges: HashMap = HashMap::new(); + for p in printings { + let code = set_code(p); + if code.is_empty() { + continue; + } + let Some(cn) = collector_number_int(p) else { continue }; + ranges + .entry(code.to_string()) + .and_modify(|r| { + r.min = r.min.min(cn); + r.max = r.max.max(cn); + r.count += 1; + }) + .or_insert(SetCollectorRange { min: cn, max: cn, count: 1 }); + } + ranges +} + +/// Per-set `collector_number_int` span, derived once from `set_codes`'s own postings at load time +/// (docs/issues/local-engine-gathered-scan-card-printing-varying-depth.md, Round 33). Lets an `And` +/// of `set:X` + a `collector_number_int` range answer with a density estimate +/// (`count / (max - min + 1)` scaled by the query's own overlap with `[min, max]`) instead of the +/// `compose_printing_estimate` `And` arm's plain min-fold, which has no tightening at all for this +/// pair today — `set` has no `compile_plane` arm and isn't in `ValueTotals`, and +/// `collector_number_int` isn't arith-tuple-eligible and has no `compile_plane` arm either, so the +/// fold picks whichever leaf's own (corpus-wide, not set-scoped) count happens to be smaller. +/// +/// Exact for a contiguously-numbered set (`density == 1.0`) and near-exact for one with a handful of +/// internal gaps; only a genuinely non-contiguous set (Secret Lair Drop, whose numbering resets per +/// drop rather than running sequentially) sees real residual error — still far smaller than either +/// alternative the fold could otherwise pick, per Round 33's held-out validation. +#[derive(Archive, Serialize, Deserialize, Default, Clone, Copy)] +struct SetCollectorRange { + min: u32, + max: u32, + /// Printings in the set with a `collector_number_int` value — almost always the set's own full + /// postings length, but tracked separately rather than assumed, since a handful of promo prints + /// omit it. + count: u32, +} + /// Build a tag/list index from interned collection ids. Accumulates postings by /// vocab id in the hot loop (integer keys, no per-element string hashing), then /// resolves each id to its owned String key once at the end. @@ -2709,6 +2756,24 @@ struct PairTotals { frame: HashMap, /// Keyed as `ValueTotals::legality` is, `(shift << 2) | status`. legality: HashMap, + /// `cmc`/`power`/`toughness` — single-valued per card (a card has exactly one of each), so a + /// per-value entry here can be SUMMED over a range exactly (`pair_range_sum`, Round 24), unlike + /// border/rarity/frame/legality above which only ever answer a single `Eq` value. Keyed by the + /// field's own raw integer value (not a bucket scheme): `NumericLayout` (planes.rs) buckets by PLANE + /// layout for bitmap compilation, a different job, and this table only ever needs the same + /// dense-but-small value→id shape the other three dimensions above already use. + cmc: HashMap, + power: HashMap, + toughness: HashMap, + /// Every DISTINCT `cmc`/`power`/`toughness` value observed in the corpus at all, regardless of + /// whether it cleared `PAIR_MIN_PRINTINGS` above — lets `pair_range_sum` tell "no card has this + /// value" (safe: contributes nothing to a range sum) apart from "some card has this value, but it + /// was pruned from the id map above" (unsafe: silently treating a pruned value as zero would + /// undercount any range that spans it). The id maps alone cannot make that distinction on their + /// own. Sorted, and small in practice (~14-21 entries per field on the production corpus). + cmc_seen: Vec, + power_seen: Vec, + toughness_seen: Vec, /// `min(a,b) * n_ids + max(a,b)` → the pair's exact totals. Complete over the stored ids: a present /// key is exact (possibly zero), a missing one means at least one value was pruned by the floor. pairs: HashMap, @@ -2726,6 +2791,14 @@ impl ArchivedPairTotals { fn get(&self, a: u16, b: u16, mode: Mode) -> Option { self.pairs.get(&self.key(a, b).into()).map(|t| t.get(mode)) } + + /// All three spaces at once, for `pair_range_sum` — which needs card/printing/artwork totals + /// together for every value in a range, and would otherwise hash the same key three times. + fn get_all(&self, a: u16, b: u16) -> Option<(usize, usize, usize)> { + self.pairs.get(&self.key(a, b).into()).map(|t| { + (u32::from(t.printings) as usize, u32::from(t.cards) as usize, u32::from(t.artworks) as usize) + }) + } } /// Exact 3-space totals per value, in one pass over printings. @@ -2793,6 +2866,8 @@ fn build_pair_totals( // Pass 1: per-value printing counts, to apply the selectivity floor. let (mut border_n, mut rarity_n, mut frame_n, mut legality_n) = (HashMap::new(), HashMap::new(), HashMap::new(), HashMap::new()); + let (mut cmc_n, mut power_n, mut toughness_n): (HashMap, HashMap, HashMap) = + (HashMap::new(), HashMap::new(), HashMap::new()); let shifts: Vec = (0..MAX_FORMATS as u8).map(|i| i * 2).collect(); for (pid, p) in printings.iter().enumerate() { let card = &cards[printing_to_card[pid] as usize]; @@ -2814,6 +2889,15 @@ fn build_pair_totals( *legality_n.entry(legality_totals_key(shift, status)).or_insert(0usize) += 1; } } + if let Some(v) = card.cmc { + *cmc_n.entry(v).or_insert(0usize) += 1; + } + if let Some(v) = card.creature_power { + *power_n.entry(v).or_insert(0usize) += 1; + } + if let Some(v) = card.creature_toughness { + *toughness_n.entry(v).or_insert(0usize) += 1; + } } // Assign compact ids to the survivors, one id space across all four dimensions. @@ -2854,6 +2938,32 @@ fn build_pair_totals( out.legality.insert(v, id); } } + // `_seen` records EVERY distinct value observed, before the floor prunes the id map -- see its own + // doc on `PairTotals` for why `pair_range_sum` needs that distinction. + let mut cmc_sorted: Vec<_> = cmc_n.into_iter().collect(); + cmc_sorted.sort_unstable(); + out.cmc_seen = cmc_sorted.iter().map(|(v, _)| *v).collect(); + for (v, n) in cmc_sorted { + if let Some(id) = assign(n, &mut next) { + out.cmc.insert(v, id); + } + } + let mut power_sorted: Vec<_> = power_n.into_iter().collect(); + power_sorted.sort_unstable(); + out.power_seen = power_sorted.iter().map(|(v, _)| *v).collect(); + for (v, n) in power_sorted { + if let Some(id) = assign(n, &mut next) { + out.power.insert(v, id); + } + } + let mut toughness_sorted: Vec<_> = toughness_n.into_iter().collect(); + toughness_sorted.sort_unstable(); + out.toughness_seen = toughness_sorted.iter().map(|(v, _)| *v).collect(); + for (v, n) in toughness_sorted { + if let Some(id) = assign(n, &mut next) { + out.toughness.insert(v, id); + } + } out.n_ids = next; let n = usize::from(next); if n == 0 { @@ -2897,6 +3007,21 @@ fn build_pair_totals( ids.push(id); } } + if let Some(v) = card.cmc + && let Some(&id) = out.cmc.get(&v) + { + ids.push(id); + } + if let Some(v) = card.creature_power + && let Some(&id) = out.power.get(&v) + { + ids.push(id); + } + if let Some(v) = card.creature_toughness + && let Some(&id) = out.toughness.get(&v) + { + ids.push(id); + } for (i, &a) in ids.iter().enumerate() { for &b in &ids[i + 1..] { let (lo, hi) = if a <= b { (a, b) } else { (b, a) }; @@ -3745,6 +3870,10 @@ struct CardIndexes { artists: ArtistIndex, // printing space (CSR by artist vocab id) flavor: FlavorIndex, // printing space (CSR by dense flavor text id) set_codes: TagIndex, // printing space + // printing space, keyed the same as `set_codes`: per-set collector_number_int min/max/count, for + // `compose_printing_estimate`'s `set:X` + `cn`-range density tightening (Round 33). Derived from + // `set_codes`'s own build pass, not a second corpus scan. + set_collector_ranges: HashMap, watermarks: TagIndex, // printing space released_at: PrintingValueIndex, // printing space price_usd: PrintingValueIndex, // printing space (integer cents, already order-preserving) @@ -4491,6 +4620,11 @@ fn probe_collection_k(filter: &FilterExpr, indexes: &Archived) -> O /// interval is never discovered — measured at 1,146.8 µs, against 26.7 µs for the one-sided /// `usd>=200`, which returns *more* rows. Fusing before ranking puts the two-sided form on the same /// sparse-vec path the one-sided form already takes. +// `Clone, Copy`: every field is itself a reference or a `Copy` scalar, so a caller that wants to +// inspect the fused list a second time (Round 33's `set:X` + `cn`-range density check, which reads it +// after `compose_printing_estimate`'s own fold already consumed one copy) can hold the `Vec` and +// iterate it by value instead of fighting reference-of-reference match ergonomics. +#[derive(Clone, Copy)] enum AndSource<'f, 'i> { Child(&'f FilterExpr), /// `[lo, hi)` on `idx`, the intersection of two or more children's intervals, holding `k` @@ -7035,6 +7169,25 @@ fn range_leaf_bits(idx: &Archived, lo: u32, hi: u32, n_print bits } +/// Exact total width across every set card in `card_bits`, against any per-card offsets array shaped +/// like `offsets`/`artwork_base` (`[card_offsets[c], card_offsets[c+1])` per card) -- the printing OR +/// artwork total of an exact card-space set, by direct summation instead of the corpus-average +/// `card_count * n_printings / n_cards` ratio. Same iteration `broadcast_card_bits_to_printings` (just +/// below) uses, summing span widths instead of setting bits, since only the total is needed here. +/// O(set cards), not O(n_cards). +fn card_bits_span_total(card_bits: &[u64], card_offsets: &AOffsets) -> usize { + let mut total = 0usize; + for (i, &word) in card_bits.iter().enumerate() { + let mut w = word; + while w != 0 { + let c = (((i as u32) << 6) | w.trailing_zeros()) as usize; + w &= w - 1; + total += u32::from(card_offsets[c + 1]) as usize - u32::from(card_offsets[c]) as usize; + } + } + total +} + /// Broadcast a card-space bitmap **down** to printing space: set every printing of each set card. The /// inverse of `printing_bits_to_card_bits`, used to lift a card-settled fact (a legality that doesn't /// diverge across the card's printings) into the printing domain for composition. Iterates set cards @@ -7373,6 +7526,49 @@ fn compose_printing_bits( } } +/// One count, in all three spaces at once -- `printing` is always known (every leaf/join here builds +/// or bounds a printing-space set, one way or another), `card`/`artwork` are `Some` exactly when this +/// same leaf/join has one for free. Replaces a design where `card`/`artwork` were bolted on next to a +/// bare `result: usize`/`candidate: usize` (each already implicitly printing-space, but nothing in the +/// TYPE said so): that shape made it possible -- and it happened, twice, checked against real data +/// before being caught -- to fold or consume the wrong field's value where a printing-space number was +/// expected, or vice versa. A `SpaceEstimate` can't be misread that way: the field name IS the space. +#[derive(Clone, Copy)] +struct SpaceEstimate { + printing: usize, + card: Option, + artwork: Option, +} + +impl SpaceEstimate { + fn printing_only(printing: usize) -> Self { + Self { printing, card: None, artwork: None } + } + + /// `And`'s fold: printing always narrows (min); card/artwork narrow too whenever EITHER side has + /// an answer for that space -- a one-sided answer is still a valid tightening (the other side is + /// unconstrained information, not a competing value), so this is not the same as requiring both. + fn min(self, other: Self) -> Self { + Self { + printing: self.printing.min(other.printing), + card: [self.card, other.card].into_iter().flatten().min(), + artwork: [self.artwork, other.artwork].into_iter().flatten().min(), + } + } + + /// `Or`'s fold: a valid (if loose, on overlap) upper bound per space, mirroring how printing's own + /// sum-then-clamp already worked. Unlike `min`, a card/artwork total here needs BOTH sides known -- + /// `Some(0) + None` must not silently drop the unknown side's real contribution and under-report + /// the union. + fn add(self, other: Self) -> Self { + Self { + printing: self.printing + other.printing, + card: self.card.zip(other.card).map(|(a, b)| a + b), + artwork: self.artwork.zip(other.artwork).map(|(a, b)| a + b), + } + } +} + /// Cheap cost-model estimate for a composable filter: `(matches, broadcast_printings, scatter_printings)` /// **without** paying legality's broadcast. The two synthesis kinds are returned separately because they /// cost different rates (`LINEAR_PASS_PER_PRINTING_NS` vs `RANGE_SCATTER_PER_PRINTING_NS`): a legality @@ -7395,36 +7591,50 @@ fn compose_printing_bits( /// distinction `exact_cards` vs `exact_total` draws one level down. #[derive(Clone)] struct ComposeEstimate { - result: usize, - candidate: usize, + result: SpaceEstimate, + candidate: SpaceEstimate, broadcast: usize, scatter: usize, - /// Best-effort CARD count `narrow_rec` really leaves for `GatheredScan`/`StreamedSelect` to walk, - /// when this estimate came from an `And` with plane-compilable children — `None` everywhere else - /// (leaves, `Or`, no plane-compilable children at all). Computed once alongside `result`'s own - /// plane-AND tightening (`compile_children_once`'s doc has the two-case mechanism) specifically so - /// `acquire_plan_features` never has to re-run `compile_plane`/`eval_planes` over the same children - /// a second time just to answer a different question about them — measured to matter: computing it - /// via a second independent pass over the same children was a real ~4x `acquire_ns` regression on - /// cheap queries, not just redundant-looking code. - domain_hint: Option, + /// Printings a CARD-SPACE collection leaf's build broadcasts (`ids_of` + + /// `broadcast_card_ids_to_printings`) -- kept apart from `scatter` because it is a different, + /// pricier operation (a card-cursor lookup per id, then a variable-width printing-range fill) than + /// a range's contiguous slice-scatter. Measured riding `scatter`'s rate on `otag:triggered-ability` + /// /`otag:cycle`/`otag:activated-ability`: 2.7-2.9x too cheap, consistently across all three (see + /// `COMPOSE_COLLECTION_BROADCAST_PER_PRINTING_NS`). Printing-space collection leaves (art_tags/ + /// is_tags) still ride `scatter`: their build IS a contiguous `bits()` copy, the same shape a + /// range's is, and measured fine. + collection_broadcast: usize, + /// The `And` arm's `best_other`/arith-tuple-merge intersection, in all three spaces, captured + /// before `result` could be tightened any further by `pair_bounded_min` -- `None` everywhere else + /// (leaves, `Or`, an `And` where neither mechanism fired). Unlike `result` (which folds in every + /// tightening this match found, so its printing count is not guaranteed to describe the same set + /// its card/artwork counts do), every field here is guaranteed to be the SAME set's count in each + /// space. `acquire_plan_features` uses this for `scan_units` -- the printing SPAN of the CANDIDATE + /// cards, which needs a printing/card pair known to match, not `result`'s tightest-found number. + exact_domain: Option, } impl ComposeEstimate { - /// A leaf: nothing to tighten, so both figures are the same count, and there is no `And` of - /// plane-compilable children to hint a domain from. + /// A leaf with no cheap exact card/artwork source: nothing to tighten, so `result`/`candidate` are + /// the same count, and there is no space beyond printing to report. fn leaf(k: usize, broadcast: usize, scatter: usize) -> Self { - Self { result: k, candidate: k, broadcast, scatter, domain_hint: None } + let space = SpaceEstimate::printing_only(k); + Self { result: space, candidate: space, broadcast, scatter, collection_broadcast: 0, exact_domain: None } + } + + /// `leaf`, plus whichever of the card/artwork spaces the caller already has in hand for free -- + /// every call site that has one is expected to pass it, not re-derive it via `result`'s own scale. + fn leaf_spaces(k: usize, broadcast: usize, scatter: usize, card: Option, artwork: Option) -> Self { + let space = SpaceEstimate { printing: k, card, artwork }; + Self { result: space, candidate: space, broadcast, scatter, collection_broadcast: 0, exact_domain: None } } -} -/// The breadth gate `narrow_rec`'s own `And` arm applies to each child before intersecting: a child -/// covering more than this fraction of its own domain gets dropped from the intersection entirely -/// (see the `len > domain - domain / 4` check in `narrow_rec`'s `And` arm) — kept as the identical -/// integer-division shape, not a `f64` fraction, so this can never round differently from the real -/// gate as domain sizes vary. -fn exceeds_own_domain_breadth(len: usize, domain: usize) -> bool { - len > domain - domain / 4 + /// A card-space collection leaf specifically -- see `collection_broadcast`'s doc for why this + /// isn't just `leaf(k, 0, k)`. + fn collection_leaf(k: usize, card: Option, artwork: Option) -> Self { + let space = SpaceEstimate { printing: k, card, artwork }; + Self { result: space, candidate: space, broadcast: 0, scatter: 0, collection_broadcast: k, exact_domain: None } + } } /// Exact count for a `ColorCmp` leaf (either polarity) via `ValueTotals`'s per-raw-combo table — @@ -7601,6 +7811,11 @@ fn arith_tuple_ids(bounds: &[&FilterExpr], indexes: &Archived) -> O Some(ids) } +/// One `And` child's compiled plane, paired with the original `FilterExpr` it came from -- the `And` +/// arm's `card_invariant`/`existential` partition keeps both so `pair_range_sum`'s preferred path can +/// ask `pair_leaf_id` about the lone existential leaf's own value without re-deriving it from the plane. +type CompiledLeaf<'a> = (&'a FilterExpr, PlaneExpr); + fn compose_printing_estimate( filter: &FilterExpr, indexes: &Archived, @@ -7609,21 +7824,35 @@ fn compose_printing_estimate( ) -> ComposeEstimate { let popcount = |bits: &[u64]| bits.iter().map(|w| w.count_ones() as usize).sum::(); match filter { - FilterExpr::True => ComposeEstimate::leaf(n_printings, 0, 0), + FilterExpr::True => { + let n_cards = offsets.len() - 1; + let n_artworks = u32::from(*indexes.artwork_base.last().expect("artwork_base has n_cards+1 entries")) as usize; + ComposeEstimate::leaf_spaces(n_printings, 0, 0, Some(n_cards), Some(n_artworks)) + } // The min-of-children fold is an intersection UPPER BOUND, and on a two-sided range it is a bad // one: `usd>=0.42 usd<=0.43` folded to min(33,862, 48,559) against a true 879, and the summed // scatter to 82,421 for an 879-row answer. Fusing same-index children first replaces both with // the interval's exact `k` — the same two `partition_point` calls the one-sided arm below // already makes, which is why a one-sided range estimates at 1.0x and this did not. FilterExpr::And(v) => { - // Collected (not folded straight through) so `domain_hint` below can look at each child's - // OWN result afterward, without re-deriving anything: every leaf arm in this match already - // answers cheaply and exactly now except `Devotion` (see that arm's own doc), so there is - // nothing left to recompute a second time the way `compile_children_once` used to. - let children_estimates: Vec = fuse_and_range_children(v, indexes, false) - .into_iter() + // Collected (not folded straight through) so the arith-tuple merge below can look at each + // child's OWN shape afterward (which ones are arith-eligible), without re-deriving anything: + // every leaf arm in this match already answers cheaply and exactly now except `Devotion` + // (see that arm's own doc), so there is nothing left to recompute a second time the way + // `compile_children_once` used to. + // Bound to a variable (not consumed straight into the fold below) so Round 33's + // `set:X` + `cn`-range check further down can inspect the same fused sources a second + // time — `AndSource` is `Copy`, so this costs nothing beyond holding the `Vec` a little + // longer, not a second call into `fuse_and_range_children`. + let and_sources = fuse_and_range_children(v, indexes, false); + let children_estimates: Vec = and_sources + .iter() + .copied() .map(|src| match src { AndSource::Child(c) => compose_printing_estimate(c, indexes, offsets, n_printings), + // `.card`/`.artwork` left `None`: `range_card_counts_for`'s `distinct_cards`/ + // `distinct_artworks` is the same structure the bare-range leaf arm below found + // unreliable for broad ranges -- see that arm's own doc. AndSource::FusedRange { k, .. } => ComposeEstimate::leaf(k, 0, k), }) .collect(); @@ -7632,7 +7861,8 @@ fn compose_printing_estimate( candidate: a.candidate.min(c.candidate), broadcast: a.broadcast + c.broadcast, scatter: a.scatter + c.scatter, - domain_hint: None, + collection_broadcast: a.collection_broadcast + c.collection_broadcast, + exact_domain: None, }); // Tighten the `min` bound with every PAIR of children the table stores. `min` over singles // lets the most selective leaf decide alone, which is why `f:modern r:rare border:white` @@ -7643,8 +7873,86 @@ fn compose_printing_estimate( // types; the two-leaf case is answered exactly one level up in `exact_result_total` and never // needs this. // Only `result` is tightened. `candidate` keeps the untightened `min`, because that is what - // narrowing leaves the alternatives to walk once its broad children decline. - let mut result = pair_bounded_min(v, indexes, folded.result); + // narrowing leaves the alternatives to walk once its broad children decline. Printing-space + // only from here down to the final `SpaceEstimate` construction: `result`/`exact_domain_*` + // stay bare `usize` locals through every tightening step below (unchanged from before this + // struct held three spaces), and get wrapped back into a `SpaceEstimate` only once, at the + // very end -- narrower diff, same values, against logic already checked with a paired diff. + let mut result = pair_bounded_min(v, indexes, folded.result.printing); + // Round 33 tightening: a bare `set:X` And'd with exactly one `collector_number_int` + // range (fused two-sided, e.g. `cn>=30 cn<=39`, or a bare one-sided child, e.g. `cn<=100`) + // gets a density estimate instead of the plain min-fold above. `set` has no `compile_plane` + // arm and isn't in `ValueTotals`, and `collector_number_int` isn't arith-tuple-eligible and + // has no `compile_plane` arm either -- so this pair gets NO tightening from any existing + // mechanism, and the fold picks whichever leaf's own (corpus-wide, not set-scoped) count + // happens to be smaller, frequently `set:X`'s own full postings length (`set:sld cn<=100` + // folds to 2,535 against a true 104). + // + // `set_collector_ranges` (built once per set at load time from `set_codes`'s own postings, + // O(1) to look up here) gives that set's own `[min, max]` collector-number span and how + // many of its printings carry a value -- `density = count / (max - min + 1)`. Scaling + // density by the query's own overlap with `[min, max]` is exact for a contiguously-numbered + // set (density == 1.0) and near-exact for one with a handful of internal gaps; only a + // genuinely non-contiguous set (Secret Lair Drop, numbered per-drop rather than + // sequentially) sees real residual error, and even there the estimate is a strict + // improvement over either marginal the fold could otherwise pick (Round 33's own + // `set:sld cn<=100`: density estimate 25.3 against a true 104, versus the fold's 2,535). + // + // Scoped to the strict 2-source shape only (after fusion): a `set:X` leaf and a lone + // `collector_number_int` source, nothing else in the `And`. `fuse_and_range_children` + // already reduces a two-sided cn bound to one `FusedRange` source, so this also covers + // `set:X cn>=30 cn<=39` despite it being 3 literal filter children. A third leaf + // (`set:sld id:g cn<=100`) is out of scope for this round -- `and_sources.len() != 2` + // simply skips it, same as any other shape this tightening doesn't recognize; the + // pre-existing fold still applies to it unchanged. + // Plain `fn`s, not closures: a closure returning a borrow tied to its argument's own + // lifetime needs an explicit HRTB Rust won't infer for a closure (unlike a `fn` item, + // which gets ordinary lifetime elision), so these are written the same way + // `bare_range_bounds` itself is. + fn set_code_eq_value<'f>(src: AndSource<'f, '_>) -> Option<&'f str> { + match src { + AndSource::Child(FilterExpr::TextExact { field: TextField::SetCode, op: CmpOp::Eq, value }) => Some(value.as_str()), + _ => None, + } + } + fn collector_number_bounds(src: AndSource<'_, '_>, indexes: &Archived) -> Option<(u32, u32)> { + match src { + AndSource::FusedRange { idx, lo, hi, .. } if std::ptr::eq(idx, &indexes.collector_number) => Some((lo, hi)), + AndSource::Child(c) => bare_range_bounds(c, indexes).and_then(|(idx, lo, hi)| std::ptr::eq(idx, &indexes.collector_number).then_some((lo, hi))), + _ => None, + } + } + if let [a, b] = and_sources.as_slice() { + let (a, b) = (*a, *b); + let shape = set_code_eq_value(a) + .zip(collector_number_bounds(b, indexes)) + .or_else(|| set_code_eq_value(b).zip(collector_number_bounds(a, indexes))); + if let Some((set_name, (q_lo, q_hi))) = shape + && let Some(range) = indexes.set_collector_ranges.get(set_name) + { + // Archived fields, not plain `u32` -- `range` is a reference into the persisted + // store, unlike `lo`/`hi`/`k` above, which are computed fresh from the query. + let (set_min, set_max, set_count) = (u32::from(range.min), u32::from(range.max), u32::from(range.count)); + if set_count > 0 && set_max >= set_min { + let span = f64::from(set_max - set_min + 1); + let density = f64::from(set_count) / span; + // Half-open `[q_lo, q_hi)` against the set's inclusive `[min, max]`: overlap + // is `min(q_hi - 1, max) - max(q_lo, min) + 1`, clamped to 0 when the + // intervals don't touch. `q_hi == 0` (an unsatisfiable fused range, see + // `fuse_and_range_children`'s own doc) is handled by the same clamp: `q_hi - 1` + // would underflow, so it's checked first. + let overlap = if q_hi == 0 { + 0 + } else { + let hi_incl = (q_hi - 1).min(set_max); + let lo_incl = q_lo.max(set_min); + if hi_incl >= lo_incl { hi_incl - lo_incl + 1 } else { 0 } + }; + let estimate = (density * f64::from(overlap)).round() as usize; + result = result.min(estimate); + } + } + } // Second tightening: 2+ cmc/power/toughness children get their TRUE joint card count from // one #743 scan (`arith_tuple_count`), not `min` of each one's own count — e.g. // `cmc<=5 power>=3` gets the real intersection, not `min(cmc<=5, power>=3)`. @@ -7709,11 +8017,14 @@ fn compose_printing_estimate( // leaf separately and ANDs printing-space bitmaps (unchanged), so this estimate-only shortcut // changes nothing about when or how the expensive work happens. let divergent_formats = u64::from(indexes.planes.divergent_formats); - let (card_invariant, existential): (Vec, Vec) = v + // Carries the original `FilterExpr` alongside each compiled `PlaneExpr` (not just the + // compiled form) so `pair_range_sum`'s preferred path below can ask `pair_leaf_id` about the + // lone existential leaf's own value directly, without re-deriving it from the plane. + let (card_invariant, existential): (Vec, Vec) = v .iter() .filter(|c| !is_arith_tuple_eligible(c)) - .filter_map(|c| compile_plane(c, &indexes.planes, &indexes.oracle_trigram.words)) - .partition(|pe| !plane_expr_is_existential(pe, divergent_formats)); + .filter_map(|c| compile_plane(c, &indexes.planes, &indexes.oracle_trigram.words).map(|pe| (c, pe))) + .partition(|(_, pe)| !plane_expr_is_existential(pe, divergent_formats)); // One trial per existential leaf present (each paired with ALL card-invariant leaves), not // "the first one, dropping the rest": every `(card-invariant leaves + this one existential // leaf)` combination is independently exact and safe (still only one existential fact, no @@ -7727,7 +8038,7 @@ fn compose_printing_estimate( // Bits are kept alongside the count (not just the scaled number) for the smaller-side merge // below, which needs to probe individual card ids against the winning combination's bitmap. let popcount_with_bits = |extra: Option<&PlaneExpr>| -> (usize, Vec) { - let mut children = card_invariant.clone(); + let mut children: Vec = card_invariant.iter().map(|(_, pe)| pe.clone()).collect(); if let Some(e) = extra { children.push(e.clone()); } @@ -7736,24 +8047,83 @@ fn compose_printing_estimate( let card_count = popcount(&bits); (card_count, bits) }; - let mut exact_domain: Option = None; + // Two DIFFERENT spaces come out of this intersection, and conflating them was a real bug + // (found while investigating `GatheredScan[printing_compose]`'s worst-in-the-survey spread, + // p90/p10 up to 40x): `exact_domain_cards` is the raw card-space popcount -- the CARD count + // this whole `And` finally exposes on `.card` (a real live query, `devotion:w c:u usd>5`, + // traced a 15.8x over-estimate to exactly this value being printing-scaled before reaching + // a card-space consumer). `result` (this function's own, PRINTING-space quantity) needs the + // printing-space total of the SAME exact card set, and that no longer means guessing it from + // the corpus's average printings-per-card (`card_count * n_printings / n_cards`) either -- + // `card_bits_span_total` sums each set card's OWN printing count directly from `offsets`, + // exact rather than average-case, since the exact bits are already in hand from the popcount + // above. Ditto `artwork` from `indexes.artwork_base`, the same shape one space over. + let mut exact_domain_cards: Option = None; let mut best_other: Option<(usize, Vec)> = None; - if existential.is_empty() { - if card_invariant.len() >= 2 { - best_other = Some(popcount_with_bits(None)); - } - } else if !card_invariant.is_empty() { - for e in &existential { - let candidate = popcount_with_bits(Some(e)); - if best_other.as_ref().is_none_or(|(c, _)| candidate.0 < *c) { - best_other = Some(candidate); + // Preferred path (Round 24) for a LONE existential leaf ANDed with a range over exactly one + // arith-tuple field (`cmc=1 border:white`, `cmc>=1 cmc<=5 r=mythic`, ...): sum the exact + // per-value pair-total over every value the arith bound(s) admit (`pair_range_sum`), rather + // than materializing a card-space bitmap and probing `arith_children`'s ids into it (Round + // 22's fix, kept below as the fallback). Exact whenever it answers, and strictly cheaper: + // bounded by the field's own distinct value count (~14-21), not `O(matching_ids)`. Declines + // to `None` (falling through to the existing logic unchanged) whenever a card-invariant leaf + // is ALSO present, whenever more than one existential leaf is present, whenever the arith + // children span more than one field, or whenever any admitted value was pruned from the pair + // table by its own selectivity floor -- see `pair_range_sum`'s own doc. + let pair_range_answer = if card_invariant.is_empty() + && let [(e_filter, _)] = existential.as_slice() + && !arith_children.is_empty() + && let Some(field) = single_arith_field(&arith_children) + && let Some(existential_id) = pair_leaf_id(e_filter, &indexes.pair_totals) + { + pair_range_sum(&arith_children, field, existential_id, &indexes.pair_totals) + } else { + None + }; + if pair_range_answer.is_none() { + if existential.is_empty() { + if card_invariant.len() >= 2 { + best_other = Some(popcount_with_bits(None)); + } + } else { + // A LONE existential leaf (no card-invariant partner at all -- `card_invariant` may be + // empty here) still gets its own exact joint via `popcount_with_bits(Some(e))`: the + // closure already handles an empty `card_invariant` correctly (`card_invariant.clone()` + // plus one pushed `e` is just `PlaneExpr::And(vec![e.clone()])`, a valid single-child + // AND, and `eval_planes` answers it exactly). This branch used to require + // `!card_invariant.is_empty()`, which meant a shape like `cmc=1 border:white` (an + // arith-tuple leaf ANDed with exactly one existential leaf and nothing else + // card-invariant) never ran this loop at all -- `best_other` stayed `None` for the rest + // of the function, so `exact_domain_cards`/`result`'s printing-space tightening below, + // and everything downstream that reads them (`domain_cards`'s `is_and` tightening, + // `card_invariant_domain_exact`, `est.result.card`), silently fell back to the + // untightened per-child `min` instead of an exact popcount. Root-caused and verified in + // docs/issues/local-engine-domain-cards-existential-arith-and.md (`eval_domain` off by up + // to ~9x for this shape); this loop is the fix. `pair_range_answer` above now answers + // most of this same population more cheaply; this loop is the fallback for the rest + // (multiple existential leaves, a card-invariant partner also present, or a pruned + // pair-table entry). + for (_, e) in &existential { + let candidate = popcount_with_bits(Some(e)); + if best_other.as_ref().is_none_or(|(c, _)| candidate.0 < *c) { + best_other = Some(candidate); + } } } } - if let Some((card_count, _)) = &best_other { - let scaled = (card_count * n_printings).checked_div(n_cards).unwrap_or(0); - result = result.min(scaled); - exact_domain = Some(scaled); + let mut exact_domain_printing: Option = None; + let mut exact_domain_artworks: Option = None; + if let Some((printings, cards, artworks)) = pair_range_answer { + result = result.min(printings); + exact_domain_cards = Some(cards); + exact_domain_printing = Some(printings); + exact_domain_artworks = Some(artworks); + } else if let Some((card_count, bits)) = &best_other { + let printing_span = card_bits_span_total(bits, offsets); + result = result.min(printing_span); + exact_domain_cards = Some(*card_count); + exact_domain_printing = Some(printing_span); + exact_domain_artworks = Some(card_bits_span_total(bits, &indexes.artwork_base)); } // Merge with the arith-tuple family (cmc/power/toughness) by ID probe, instead of compiling // their numeric-range planes into the SAME joint above: that was tried first and reverted @@ -7765,12 +8135,15 @@ fn compose_printing_estimate( // reuses the exact card ids `bare_numeric_field_ids`/`arith_tuple_ids` already have cheaply // in hand (`O(log n)` or `O(564 keys)`, no plane involved) and probes each one against // `best_other`'s bitmap directly -- cost `O(arith_count)`, adaptive to the actual data rather - // than a fixed worst case. Gated only on `best_other` existing at all: a query with rarity/ - // border/legality alongside arith leaves but nothing ELSE plane-compilable (`r<=rare - // tou>=4 tou<=5`, one of the regressed queries under the plane-based version) never reaches - // here at all, since `best_other` stays `None` for it (the existential branch above requires - // a card-invariant partner) -- measured back to the exact pre-this-commit acquire cost for - // that shape specifically, not just cheaper. Only handles "arith side probes into other's + // than a fixed worst case. Gated only on `best_other` existing at all: a query with NEITHER a + // card-invariant NOR an existential leaf compiling (an all-arith `And`, or one with a + // non-plane-compilable sibling like `year<=2017` and no rarity/border/legality anywhere) + // never reaches here, since `best_other` stays `None` for it -- measured back to the exact + // pre-this-commit acquire cost for that shape specifically, not just cheaper. A LONE + // existential leaf with no card-invariant partner (`r<=rare tou>=4 tou<=5`) now DOES reach + // here (the existential branch above no longer requires a card-invariant partner -- see its + // own comment) and gets the same arith-ID-probe tightening a card-invariant-paired + // existential leaf already got. Only handles "arith side probes into other's // bitmap", not the reverse (iterating `other`'s bits and checking the arith bound directly): // that direction would need a raw per-card cmc/power/toughness lookup this function has no // access to. Not a correctness gap -- probing arith into `best_other` is enough on its own to @@ -7783,61 +8156,91 @@ fn compose_printing_estimate( _ => arith_tuple_ids(&arith_children, indexes), }; if let Some(ids) = arith_ids { - let joint_count = ids + let joint_ids: Vec = ids .iter() - .filter(|&&id| { + .copied() + .filter(|&id| { let (word, bit) = (id as usize / 64, id as usize % 64); word < other_bits.len() && (other_bits[word] >> bit) & 1 == 1 }) - .count(); - let scaled = (joint_count * n_printings).checked_div(n_cards).unwrap_or(0); - result = result.min(scaled); - exact_domain = Some(exact_domain.map_or(scaled, |d| d.min(scaled))); + .collect(); + // Same exact-sum shape as `card_bits_span_total`, over an id LIST instead of a + // bitmap -- the joint set only exists as `joint_ids` here, never materialized as + // its own bitmap, so summing directly over the list avoids building one just to + // re-iterate it. + let span_of = |card_offsets: &AOffsets| -> usize { + joint_ids.iter().map(|&id| u32::from(card_offsets[id as usize + 1]) as usize - u32::from(card_offsets[id as usize]) as usize).sum() + }; + let joint_count = joint_ids.len(); + let joint_printing = span_of(offsets); + result = result.min(joint_printing); + exact_domain_cards = Some(exact_domain_cards.map_or(joint_count, |d| d.min(joint_count))); + exact_domain_printing = Some(exact_domain_printing.map_or(joint_printing, |d| d.min(joint_printing))); + exact_domain_artworks = Some(exact_domain_artworks.map_or_else(|| span_of(&indexes.artwork_base), |d| d.min(span_of(&indexes.artwork_base)))); } } - // `domain_hint`: what GatheredScan/StreamedSelect really see once narrow_rec intersects. - // Every leaf type in this match now already answers its OWN `.result` cheaply and exactly - // (border/rarity/legality/range/collection/color/cmc/power/toughness), so the breadth- - // filtered min over the children already collected above is real information, not a - // separate re-derivation — a child whose own result exceeds - // `exceeds_own_domain_breadth` of `n_printings` is dropped first, mirroring `narrow_rec`'s - // own breadth gate on its `And` arm, rather than left to dominate a `min` it would never - // survive to contribute to for real. (`Devotion`'s own `.result`, the one leaf type still - // priced via `eval_planes`, participates the same way as everything else here — nothing - // special-cased for it.) `exact_domain`, when present, is folded in too — it is the true - // value rather than an upper bound, so `min`-ing it in can only tighten, never regress: this - // is what actually reaches `acquire_plan_features`'s `domain_cards` (the `result`-only - // tightening above helps `PrintingCompose`'s own pricing, but `GatheredScan`/`StreamedSelect` - // are priced from `domain_hint`, not `result`, whenever `est.candidate != est.result`). - let domain_hint = [ - children_estimates.iter().filter(|c| !exceeds_own_domain_breadth(c.result, n_printings)).map(|c| c.result).min(), - exact_domain, - ] - .into_iter() - .flatten() - .min(); - ComposeEstimate { result, domain_hint, ..folded } + // `domain_hint` used to be its own field, folded from `children_estimates`'s own `.result` + // -- but `.result` is this function's PRINTING-space quantity for every leaf type (confirmed + // directly: `ColorCmp`'s arm calls `color_cmp_value_total(..., Mode::Printing)` explicitly), + // so that fold was handing a printing count to `domain_cards`, a card-space consumer. Because + // the corpus's average reprint rate inflates a card count into its printing equivalent + // (~3.08x here), the bug was largely self-masking -- an inflated `domain_hint` usually lost + // the outer `.min(calibrated)` at the call site to the correctly-scaled `calibrated`, so it + // only bit when the true intersection was selective enough that even the inflated number + // still undercut `calibrated`, which is exactly the long-tail-not-median shape this cell + // measured (median 1.00, p99 14-24x, p100 up to 382x). + // + // `domain_hint` is retired in favor of `.card` directly, but fed ONLY by + // `exact_domain_cards` (this intersection's own exact popcount from `best_other`/the + // arith-tuple merge) -- deliberately NOT `folded.card` (each child's OWN, individual card + // count, folded via `min`). That fold was tried and checked directly against real data: a + // single BROAD leaf's own card count (`border:black`, `f:duel`'s own legal-card count -- + // both individually correct numbers) is not a safe upper bound on the whole `And` unless + // `narrow_rec` would actually USE that leaf to narrow, and it does not for a broad one + // (`border:black` declining at 87% under `broad_ok: false` is the exact, already-documented + // precedent -- see `exceeds_own_domain_breadth`'s old callers). The retired `domain_hint` + // guarded against exactly this with a breadth filter before folding; dropping that filter + // along with the space-mismatch fix silently lost the guard too. `exact_domain_cards` has + // no such risk: it is a real INTERSECTION (`eval_planes` over the combined `PlaneExpr`), not + // a per-leaf bound, so it can only ever be <= the true joint count, never an overcount from + // a leaf `narrow_rec` would have declined to use alone. + let result_space = SpaceEstimate { printing: result, card: exact_domain_cards, artwork: exact_domain_artworks }; + // `exact_domain`: the SAME `best_other`/arith-merge intersection, but captured BEFORE + // `result` could be tightened any further by `pair_bounded_min` (above) -- so unlike + // `result`, `.printing`/`.card`/`.artwork` here are guaranteed to describe the exact same + // set, never a printing count over-tightened past what the card/artwork counts still + // describe. `result` doesn't have that guarantee (it takes the `min` across every + // tightening this match found, from whichever source got there first), which is exactly + // why `scan_units` -- the printing SPAN of the CANDIDATE cards, not the tightest known + // match count -- needs its own field instead of reusing `result`. + let exact_domain = exact_domain_printing.map(|printing| SpaceEstimate { printing, card: exact_domain_cards, artwork: exact_domain_artworks }); + ComposeEstimate { result: result_space, exact_domain, ..folded } } FilterExpr::Or(v) => { + let n_cards = offsets.len() - 1; + let n_artworks = u32::from(*indexes.artwork_base.last().expect("artwork_base has n_cards+1 entries")) as usize; + let clamp = |space: SpaceEstimate| SpaceEstimate { + printing: space.printing.min(n_printings), + card: space.card.map(|c| c.min(n_cards)), + artwork: space.artwork.map(|a| a.min(n_artworks)), + }; let summed = v .iter() .map(|c| compose_printing_estimate(c, indexes, offsets, n_printings)) .fold(ComposeEstimate::leaf(0, 0, 0), |a, c| ComposeEstimate { - result: a.result + c.result, - candidate: a.candidate + c.candidate, + result: a.result.add(c.result), + candidate: a.candidate.add(c.candidate), broadcast: a.broadcast + c.broadcast, scatter: a.scatter + c.scatter, - domain_hint: None, + collection_broadcast: a.collection_broadcast + c.collection_broadcast, + exact_domain: None, }); - ComposeEstimate { - result: summed.result.min(n_printings), - candidate: summed.candidate.min(n_printings), - ..summed - } + ComposeEstimate { result: clamp(summed.result), candidate: clamp(summed.candidate), ..summed } } // Precomputed planes: exact cheap popcount, nothing synthesized. FilterExpr::TextExact { field: TextField::Border, op: CmpOp::Eq, value } => { - ComposeEstimate::leaf(popcount(&border_leaf_bits(value.as_str(), &indexes.border_printing, n_printings)), 0, 0) + let k = popcount(&border_leaf_bits(value.as_str(), &indexes.border_printing, n_printings)); + ComposeEstimate::leaf_spaces(k, 0, 0, exact_result_total(filter, indexes, Mode::Card), exact_result_total(filter, indexes, Mode::Artwork)) } // #746: `set:`/`watermark:` postings — matches = the value's postings length `k` (each // posting is one distinct printing), synthesized by scattering `k` ids → rides `scatter` @@ -7863,17 +8266,27 @@ fn compose_printing_estimate( } // Collection containment leaf (`type:`/`kw:`/`otag:`/`art:`/`is:`, `Ge`): `k` = the exact // printing count the leaf matches (card-space sums the matching cards' printing ranges, - // printing-space is the postings length). The build scatters `k` printings → rides `scatter`, - // the cheap range-slice rate. + // printing-space is the postings length). Card-space fields build via `ids_of` + + // `broadcast_card_ids_to_printings` (a card-cursor lookup per id) -- `collection_broadcast`, + // not `scatter`. Printing-space fields build via a contiguous `bits()` copy, the same shape a + // range's scatter is, so they keep riding `scatter`'s rate. FilterExpr::CollectionCmp { field, op: CmpOp::Ge, value, .. } if collection_compose_index(indexes, *field).is_some() => { let src = collection_compose_index(indexes, *field).expect("guarded by the if"); let k = collection_leaf_printing_count(&src, value.as_str(), offsets); - ComposeEstimate::leaf(k, 0, k) - } - // Negated collection leaf: all printings minus the positive `k`; the scatter cost rides the - // (small) positive `k` cleared, not the (large) complement it produces — same shape as `-set:`. + // Every field reaching this arm is one of `exact_result_total`'s per-value `ValueTotals` + // dimensions (subtypes/keywords/oracle_tags/art_tags/is_tags/frame_data), which store all + // three spaces per value regardless of whether the field's own postings are card- or + // printing-space -- reusing it here for card/artwork is the exact same table lookup + // `collection_leaf_printing_count` uses for printing, not a second derivation. + let card = exact_result_total(filter, indexes, Mode::Card); + let artwork = exact_result_total(filter, indexes, Mode::Artwork); + if src.card_space { ComposeEstimate::collection_leaf(k, card, artwork) } else { ComposeEstimate::leaf_spaces(k, 0, k, card, artwork) } + } + // Negated collection leaf: all printings minus the positive `k`; the build cost rides the + // (small) positive `k` cleared, not the (large) complement it produces — same shape as `-set:`, + // and the same card-space-vs-printing-space split as the positive leaf above. FilterExpr::Not(inner) if matches!(inner.as_ref(), FilterExpr::CollectionCmp { field, op: CmpOp::Ge, .. } if collection_compose_index(indexes, *field).is_some()) => { @@ -7882,8 +8295,29 @@ fn compose_printing_estimate( }; let src = collection_compose_index(indexes, *field).expect("guarded by the matches!"); let k = collection_leaf_printing_count(&src, value.as_str(), offsets); - ComposeEstimate::leaf(n_printings.saturating_sub(k), 0, k) + let complement = n_printings.saturating_sub(k); + let n_cards = offsets.len() - 1; + let n_artworks = u32::from(*indexes.artwork_base.last().expect("artwork_base has n_cards+1 entries")) as usize; + // Safe to complement in card/artwork space too: every one of `exact_result_total`'s + // per-value tables is COMPLETE (holds every value present in the corpus, so absence is an + // exact zero, not an unknown), matching the same guarantee the printing-space complement + // above already relies on. + let card = exact_result_total(inner, indexes, Mode::Card).map(|c| n_cards.saturating_sub(c)); + let artwork = exact_result_total(inner, indexes, Mode::Artwork).map(|a| n_artworks.saturating_sub(a)); + if src.card_space { + let space = SpaceEstimate { printing: complement, card, artwork }; + ComposeEstimate { result: space, candidate: space, broadcast: 0, scatter: 0, collection_broadcast: k, exact_domain: None } + } else { + ComposeEstimate::leaf_spaces(complement, 0, k, card, artwork) + } } + // `.card`/`.artwork` deliberately left `None` here, not `exact_result_total(..., Mode::Card)`: + // checked directly against real data, `RangeCardCounts::distinct_cards` (the structure rarity's + // Mode::Card/Artwork arms in `exact_result_total` read, shared with the bare-range arm below) + // gave wrong-by-a-wide-margin answers for broad ranges (`r<=mythic` real 31,724 cards, this + // path 31,722; part of a broader pattern -- see the bare-range arm's own doc, and the `is_and` + // doc in `acquire_plan_features`, for the 250-query regression this traced to). A real, + // pre-existing bug in that structure's card/artwork counting, not something this pass fixes. FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::RarityInt), op, rhs: NumExpr::Const(c) } => { ComposeEstimate::leaf(popcount(&rarity_cmp_leaf_bits(*op, *c, &indexes.rarity_printing, n_printings)), 0, 0) } @@ -7900,7 +8334,10 @@ fn compose_printing_estimate( let legal = legality_candidate_bits(indexes, n_cards, *shift, *expected, false).map_or(0, |b| popcount(&b)); let illegal = legality_candidate_bits(indexes, n_cards, *shift, *expected, true).map_or(0, |b| popcount(&b)); let scale = |c: usize| (c * n_printings).checked_div(n_cards).unwrap_or(0); - ComposeEstimate::leaf(scale(legal), scale(legal.min(illegal)), 0) + // `legal` is already the exact CARD count this leaf matches -- reused directly rather than + // re-deriving it a second way through `exact_result_total`. Artwork has no equivalent + // cheap-popcount source here, so it comes from the `ValueTotals` lookup instead. + ComposeEstimate::leaf_spaces(scale(legal), scale(legal.min(illegal)), 0, Some(legal), exact_result_total(filter, indexes, Mode::Artwork)) } // Color family: exact via `ValueTotals`'s per-combo table — no `eval_planes`, no bitmap. // `.result`/`.broadcast` both ride the printing count: the REAL build (`compose_printing_bits`) @@ -7908,12 +8345,16 @@ fn compose_printing_estimate( // even though this estimate no longer computes it that way. FilterExpr::ColorCmp { field, op, mask } => { let k = color_cmp_value_total(*field, *op, *mask, false, indexes, Mode::Printing); - ComposeEstimate::leaf(k, k, 0) + let card = color_cmp_value_total(*field, *op, *mask, false, indexes, Mode::Card); + let artwork = color_cmp_value_total(*field, *op, *mask, false, indexes, Mode::Artwork); + ComposeEstimate::leaf_spaces(k, k, 0, Some(card), Some(artwork)) } FilterExpr::Not(inner) if matches!(inner.as_ref(), FilterExpr::ColorCmp { .. }) => { let FilterExpr::ColorCmp { field, op, mask } = inner.as_ref() else { unreachable!("guarded above") }; let k = color_cmp_value_total(*field, *op, *mask, true, indexes, Mode::Printing); - ComposeEstimate::leaf(k, k, 0) + let card = color_cmp_value_total(*field, *op, *mask, true, indexes, Mode::Card); + let artwork = color_cmp_value_total(*field, *op, *mask, true, indexes, Mode::Artwork); + ComposeEstimate::leaf_spaces(k, k, 0, Some(card), Some(artwork)) } // cmc/power/toughness: exact via the dedicated per-field sorted index (O(log n)) — no // `eval_planes`, and cheaper than the joint #743 index's O(distinct tuples) scan, which this @@ -7928,7 +8369,9 @@ fn compose_printing_estimate( .expect("gated by is_printing_composable"); let n_cards = offsets.len() - 1; let scaled = (card_count * n_printings).checked_div(n_cards).unwrap_or(0); - ComposeEstimate::leaf(scaled, scaled, 0) + // `card_count` is already exact and in hand -- no re-derivation needed. No artwork source + // for cmc/power/toughness (per `exact_result_total`'s own doc: card-space index only). + ComposeEstimate::leaf_spaces(scaled, scaled, 0, Some(card_count), None) } // Devotion: the one card-invariant broadcast leaf left with no cheaper exact source than a // real (cheap — O(n_cards/64)) `eval_planes` pass. No divergent-card fuzziness here (unlike @@ -7938,12 +8381,24 @@ fn compose_printing_estimate( _ if is_broadcast_leaf_shape(filter) => { let card_bits = broadcast_composable_card_bits(filter, indexes).expect("gated by is_printing_composable"); let n_cards = offsets.len() - 1; - let scaled = (popcount(&card_bits) * n_printings).checked_div(n_cards).unwrap_or(0); - ComposeEstimate::leaf(scaled, scaled, 0) + let card_count = popcount(&card_bits); + let scaled = (card_count * n_printings).checked_div(n_cards).unwrap_or(0); + // No artwork source here either -- devotion is synthesized from mana cost, not a raw + // corpus dimension `ValueTotals` has an artwork column for. + ComposeEstimate::leaf_spaces(scaled, scaled, 0, Some(card_count), None) } // Range (bare or negated — `-usd<50` etc., see `bare_range_bounds`'s doc): `k` in-range // printings from the index partition points (O(log n), no scatter here); matches ≈ k, and k // rides `scatter` — the cheap range-slice scatter into the printing bitmap. + // + // `.card`/`.artwork` deliberately left `None`, not `exact_result_total(..., Mode::Card/Artwork)` + // (which reads `RangeCardCounts::distinct_cards`/`distinct_artworks` for this same shape): + // checked directly against real data, that path gave wrong-by-a-wide-margin answers for broad + // ranges (`eur>0.16`, real 31,724 cards out of 31,724, this path 19,992 -- part of a 250-query + // regression pattern this traced to, all broad range/rarity queries; full detail on the `is_and` + // gate in `acquire_plan_features`). A real, pre-existing bug in that structure's card/artwork + // counting, not something this pass fixes -- `ValueTotals`-backed leaves (`ColorCmp`/`Legality`/ + // `CollectionCmp`/border), a different, simpler per-value lookup, show no sign of the same issue. FilterExpr::NumericCmp { .. } | FilterExpr::DateCmp { .. } | FilterExpr::YearCmp { .. } | FilterExpr::Not(_) if bare_range_bounds(filter, indexes).is_some() => { @@ -8349,10 +8804,11 @@ fn pair_bounded_min(children: &[FilterExpr], indexes: &Archived, si /// The pair-table id for a leaf, or `None` when the leaf's dimension is not covered or its value was /// pruned by the selectivity floor. /// -/// Deliberately the same four shapes `exact_result_total`'s singleton arms accept, and for the same +/// Deliberately the same shapes `exact_result_total`'s singleton arms accept, and for the same /// reasons: `Eq` only on the interned strings (the ordering ops are not a per-value question), `Ge` only -/// on the collection (`Eq`/`Gt` add a length condition containment does not prove), and rarity only at -/// `Eq` (any other op is a range over several values, which no per-value entry answers). +/// on the collection (`Eq`/`Gt` add a length condition containment does not prove), and rarity/cmc/ +/// power/toughness only at `Eq` (any other op is a range over several values, which no per-value entry +/// answers on its own — `pair_range_sum` is the range-capable counterpart for the arith-tuple three). fn pair_leaf_id(filter: &FilterExpr, pt: &ArchivedPairTotals) -> Option { let id = match filter { FilterExpr::TextExact { field: TextField::Border, op: CmpOp::Eq, value } => pt.border.get(value.as_str()), @@ -8365,11 +8821,114 @@ fn pair_leaf_id(filter: &FilterExpr, pt: &ArchivedPairTotals) -> Option { } pt.rarity.get(&(*v as u8)) } + FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::Cmc), op: CmpOp::Eq, rhs: NumExpr::Const(v) } + | FilterExpr::NumericCmp { lhs: NumExpr::Const(v), op: CmpOp::Eq, rhs: NumExpr::Field(NumField::Cmc) } => { + if v.fract() != 0.0 || *v < 0.0 || *v > f64::from(u8::MAX) { + return None; + } + pt.cmc.get(&(*v as u8)) + } + FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::Power), op: CmpOp::Eq, rhs: NumExpr::Const(v) } + | FilterExpr::NumericCmp { lhs: NumExpr::Const(v), op: CmpOp::Eq, rhs: NumExpr::Field(NumField::Power) } => { + if v.fract() != 0.0 || *v < f64::from(i8::MIN) || *v > f64::from(i8::MAX) { + return None; + } + pt.power.get(&(*v as i8)) + } + FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::Toughness), op: CmpOp::Eq, rhs: NumExpr::Const(v) } + | FilterExpr::NumericCmp { lhs: NumExpr::Const(v), op: CmpOp::Eq, rhs: NumExpr::Field(NumField::Toughness) } => { + if v.fract() != 0.0 || *v < f64::from(i8::MIN) || *v > f64::from(i8::MAX) { + return None; + } + pt.toughness.get(&(*v as i8)) + } _ => None, }?; Some(u16::from(*id)) } +/// The single `NumField` every one of `children` constrains, or `None` when they don't all agree (a +/// mixed `cmc>=1 power<=2`-shaped `And` — `pair_range_sum` sums over ONE field's own value axis per +/// call, not several at once) or `children` is empty. `children` is expected to already be +/// `is_arith_tuple_eligible`-filtered. +fn single_arith_field(children: &[&FilterExpr]) -> Option { + let mut field = None; + for c in children { + let f = match c { + FilterExpr::NumericCmp { lhs: NumExpr::Field(f), rhs: NumExpr::Const(_), .. } + | FilterExpr::NumericCmp { lhs: NumExpr::Const(_), rhs: NumExpr::Field(f), .. } => *f, + _ => return None, + }; + match field { + None => field = Some(f), + Some(existing) if existing == f => {} + Some(_) => return None, + } + } + field +} + +/// Exact (printing, card, artwork) totals for a RANGE over one arith-tuple field's values (`cmc`/ +/// `power`/`toughness`) ANDed with one existential leaf's own pair-table id (`pair_leaf_id`). +/// +/// `cmc`/`power`/`toughness` are single-valued per card — exactly one value per card, never several — +/// so partitioning the card space by that one value is genuinely exhaustive and non-overlapping. +/// Summing the per-value exact pair-total (`pt.get_all`) over every value `bounds` admits therefore +/// reproduces the TRUE joint total exactly: a sum over disjoint cells, not a product or a `min`, so +/// there is no independence assumption and no anti-correlation risk. Validated 429/429 against real +/// corpus data in Round 23's investigation before this was wired in — see +/// docs/issues/local-engine-domain-cards-existential-arith-and.md. +/// +/// Declines (`None`) the instant ANY value `bounds` admits was pruned from the field's own id map by +/// the selectivity floor — `*_seen` (on `PairTotals`) is what tells that apart from "this value never +/// occurs at all" (safe either way: contributes nothing), so a missing id here means "give up", never +/// "count it as zero". +/// +/// Bounded by the field's own distinct observed value count (~14-21 on the production corpus), not by +/// how wide `bounds` phrases the range: `cmc<=250` costs exactly what `cmc<=5` does. +fn pair_range_sum(bounds: &[&FilterExpr], field: NumField, existential_id: u16, pt: &ArchivedPairTotals) -> Option<(usize, usize, usize)> { + let admits = |v: f64| -> bool { + let cmc = matches!(field, NumField::Cmc).then_some(v); + let power = matches!(field, NumField::Power).then_some(v); + let toughness = matches!(field, NumField::Toughness).then_some(v); + bounds.iter().all(|b| { + let FilterExpr::NumericCmp { lhs, op, rhs } = b else { return false }; + matches!(eval_arith_tuple_tri(lhs, *op, rhs, cmc, power, toughness, None), Tri::True) + }) + }; + let mut total = (0usize, 0usize, 0usize); + let mut add = |field_id: u16| -> Option<()> { + let (p, c, a) = pt.get_all(field_id, existential_id)?; + total = (total.0 + p, total.1 + c, total.2 + a); + Some(()) + }; + match field { + NumField::Cmc => { + for &v in pt.cmc_seen.iter() { + if admits(f64::from(v)) { + add(u16::from(*pt.cmc.get(&v)?))?; + } + } + } + NumField::Power => { + for &v in pt.power_seen.iter() { + if admits(f64::from(v)) { + add(u16::from(*pt.power.get(&v)?))?; + } + } + } + NumField::Toughness => { + for &v in pt.toughness_seen.iter() { + if admits(f64::from(v)) { + add(u16::from(*pt.toughness.get(&v)?))?; + } + } + } + _ => return None, + } + Some(total) +} + /// Whether two leaves are provably disjoint: distinct values of a dimension that holds exactly ONE value /// per printing, so no printing can satisfy both. /// @@ -9373,6 +9932,52 @@ fn compose_leaf_nothing_to_verify(filter: &FilterExpr) -> bool { ) } +/// The PLANE half of the router's `tier`/`residual_tier_ns100` charge in the `PrintingCompose`-acquire +/// branch of `acquire_plan_features` -- the counterpart to `plane_leaves_nothing_to_verify`'s combined +/// (filter-must-be-True-too) check, factored out so it can be ANDed with the FILTER half +/// (`compose_leaf_nothing_to_verify`) independently instead of ORed as two whole-query claims. See +/// the call site for why the OR shape was unsound. +/// +/// Round 15 shipped this scoped to a special-cased `plane_touches_rarity_or_border` helper (a plane- +/// index-range check) instead of the general `plane_expr_is_existential` this now uses directly, on the +/// theory that legality's `Mode::Card` bypass (#667: "the card has some legal printing" already IS +/// `unique=card`'s semantics) was sound while rarity/border's wasn't. That theory doesn't survive +/// contact with the executor: `existential_plane_for` (this file) grants NO `Mode::Card`-vs-family +/// carveout at all -- it forces `push_card_matches` into the same per-candidate, per-printing +/// `eval_plane_expr_for_printing` walk for ANY plane where `plane_expr_is_existential` is true, rarity, +/// border, or a DIVERGENT legality format alike (row selection must still return an actual witnessing +/// printing -- see #667's "Row selection for `unique=card`"). A bare `f:oldschool` (Round 15's +/// production corpus's one divergent format), `unique=card`: `GatheredScan` predicted 10,358ns +/// (`residual_tier_ns100 == 0`, the Round-15 bypass firing because a pure-legality plane never +/// "touches rarity or border") against a measured 29,083ns median -- a 2.8x under-charge, and +/// `printings_examined` (6,037) over `cards_visited` (961) proves the per-printing walk is real, not a +/// costing artifact, exactly the same shape as Round 15's `border`/`rarity` finding +/// (docs/issues/local-engine-gathered-scan-undercosted-arith-existential-and.md). `f:oldschool +/// cmc>=1 cmc<=5` (folds to one plane the same way the shipped fix's `cmc`+`border` reproducer did): +/// predicted 18,519ns against measured 58,625ns, a 3.16x under-charge. +/// +/// So the right question was never "which FIELD is this" -- it's "does `existential_plane_for` force a +/// per-printing walk here", and `plane_expr_is_existential` (planes.rs) already answers that precisely, +/// per plane index, via the family-keyed `PLANE_BLOCKS`/`existential_leaf`/`needs_printing_verification` +/// table (legality: divergent-format-dependent; rarity/border: unconditional) -- the exact table +/// `existential_plane_for` itself reads. No `Mode::Card` term is needed here either: `split_planes` +/// (planes.rs) only ever folds an existential leaf into `plane` under `unique_is_card` (its whole-filter +/// guard and its `And`-child guard are both `unique_is_card || !plane_expr_is_existential`), so +/// `plane_expr_is_existential(plane)` being true already implies `mode == Mode::Card` -- there is no +/// live case for a `mode` check to distinguish here that the plane's own existential-ness doesn't +/// already settle. A future field (e.g. a new tracked value in a new `BlockKind`) inherits the right +/// answer automatically by which `PLANE_BLOCKS` entry it lands in, with no new arm needed here. +/// +/// `plane_leaves_nothing_to_verify` itself is deliberately left as-is (still used by the EXECUTOR's own +/// `all_match_known` in `prepare_candidates`): granting Mode::Card's bypass to EVERY existential family +/// there (legality included) is harmless because the real per-printing correctness work runs through +/// `existential_plane_for` regardless of what `all_match_known` says. The router's `tier` charge has no +/// such second mechanism: if `tier` is 0 ("nothing to verify"), `GatheredScan`/`StreamedSelect` are +/// priced as though that per-printing walk never happens, when for any existential plane it always does. +fn cost_plane_nothing_to_verify(plane: Option<&PlaneExpr>, indexes: &Archived) -> bool { + plane.is_none_or(|expr| !plane_expr_is_existential(expr, u64::from(indexes.planes.divergent_formats))) +} + /// The candidate materialization + filter rewriting shared by `StreamedSelect` /// and `GatheredScan`, extracted verbatim from `run_query`. Mutates `filter` via /// `memoize_text_predicates` + `order_children_by_verify_cost` under the same @@ -10295,6 +10900,21 @@ pub(crate) struct PhaseStats { /// `matches x EMIT + FIXED` ~ 397 ns throughout: under by 3.4x at the production corpus and 26x at /// 410k. Published so the estimate can be GRADED rather than assumed, like the other three counters. pub(crate) perm_steps: u64, + /// Printings `push_card_matches` re-examined in `run_query_streamed`'s `total <= STREAM_MIN_MATCHES` + /// branch's SECOND pass over every matching card -- the redo Round 30 of the printing-varying-leaf + /// depth ledger (docs/issues/local-engine-gathered-scan-card-printing-varying-depth.md) found + /// `printings_examined` structurally cannot see, because that counter is captured only from the + /// first, counting-only pass (`card_match_count`), which this second pass never touches. + /// + /// `push_card_matches` already computes and returns this per call (mirroring `card_match_count`'s + /// own `(c, examined)` pattern) -- capturing it here is free, not a new pass or a new computation, + /// just no longer discarding a value the redo loop already produces. + /// + /// Zero for every other executor and for P3's other two exits (the empty/past-the-end return and + /// the permutation walk, whose own per-step `push_card_matches` cost already flows into `ns_loop` + /// there and is not double-counted here -- see `perm_steps`'s own calibration). Nonzero only on the + /// exit this field exists to price: the small-total gather-and-quickselect branch, any mode. + pub(crate) redo_examined: u64, /// Per-query scratch setup, before the match loop starts. Split out because it is neither /// prepare nor match and it is NOT negligible: `run_query_streamed` zeroes an `n_cards`-long /// counts buffer here (~126 kB on the real corpus) no matter how few candidates it is about to @@ -10558,7 +11178,8 @@ thread_local! { /// owned elsewhere: `paging_taken` by `PAGING_TAKEN` below, `ns_round_total`/`result_total` by /// `explain_analyze`, which fills them after the take. static PHASE_STATS: std::cell::Cell = const { std::cell::Cell::new(PhaseStats { - cards_visited: 0, printing_span: 0, printings_examined: 0, matches_pushed: 0, set_printings: 0, perm_steps: 0, ns_setup: 0, + cards_visited: 0, printing_span: 0, printings_examined: 0, matches_pushed: 0, set_printings: 0, perm_steps: 0, + redo_examined: 0, ns_setup: 0, ns_loop: 0, ns_finish: 0, ns_round_total: 0, ns_prepare: 0, result_total: 0, paging_taken: PagingTaken::NotEntered, }) }; @@ -10794,6 +11415,7 @@ fn exec_gathered_scan<'a>( matches_pushed: n_matches_pushed, set_printings: 0, // PrintingCompose-only; GatheredScan never composes pbits perm_steps: 0, // GatheredScan never walks the permutation + redo_examined: 0, // StreamedSelect-only; GatheredScan pays one pass, never a redo ns_setup: (t_loop - t_start).as_nanos() as u64, ns_loop: (t_finish - t_loop).as_nanos() as u64, ns_finish: (t_end - t_finish).as_nanos() as u64, @@ -11008,6 +11630,204 @@ fn declined_sibling_fastpath<'a>( /// like 1.35 before the two populations were separated. const COMPOSE_CARD_ESTIMATE_BIAS: f64 = 1.78; +/// A SECOND clustering-bias divisor for `calibrated_balls_into_bins`, for a population +/// `COMPOSE_CARD_ESTIMATE_BIAS` was never fit against: an `And` of 2+ printing-varying range leaves +/// on DIFFERENT indexes (`price_usd`/`price_eur`/`price_tix`/`collector_number_int`/`released_at` +/// combinations, e.g. `usd<5 cn>100`, `eur<=0.3 tix>=0.03 tix<=0.97`) where `exact_cards` declines +/// because no pair-table / arith-tuple / plane-compile mechanism covers this combination (see +/// `is_cross_index_range_and`'s call site). `COMPOSE_CARD_ESTIMATE_BIAS`'s 1.78 was fit on a +/// single-leaf broadcast population (a card-invariant leaf setting a whole card's printings at +/// once) -- this population's `k` is instead the MIN-FOLD of 2-3 independently-indexed leaves' +/// own exact printing counts, and Round 2 of +/// `docs/issues/local-engine-gathered-scan-card-printing-varying-depth.md` proved directly that this +/// min-fold under-counts the true candidate span (866/1,500 sampled rows under-estimate +/// `scan_units` against only 450 over-estimate) -- the opposite direction from the single-leaf +/// broadcast population, where dividing `k` further (as 1.78 does) was the right correction. +/// +/// Fit as a plain bias sweep (same shape as `COMPOSE_CANDIDATE_SPAN_BIAS`'s Round 1 method): capture +/// each sampled query's per-leaf exact printing counts (one `unique=printing` sub-query per leaf) and +/// the real `printings_examined` GatheredScan counter, then re-derive `scan_units` in Python for any +/// candidate bias with no rebuild (`calibrated_balls_into_bins_with_bias` is a closed-form function +/// of `k`/`domain`/`bias`). Swept 0.20-1.78 in steps of 0.02 on a HELD-OUT split (hash of the query +/// string mod 2, fit on one half, graded on the other -- the two halves' error-vs-bias curves have +/// minima 0.04 apart, confirming the fit is not chasing split-specific noise) over 1,500 and2/and3 +/// RANGE_FAMILIES queries (`unique=card`), the same population and precedent size as Rounds 1-2: +/// +/// total abs scan_units error, calibration half (n=758): 8.70M (bias 1.78) -> 8.31M (bias 1.1) +/// total abs scan_units error, held-out half (n=742): 8.60M (bias 1.78) -> 8.02M (bias 1.1) +/// held-out improved/regressed/tied: 433 / 117 / 192 +/// +/// Restricted to the rows a bias can actually move -- `range_too_broad_to_narrow` resets +/// `eval_domain`/`scan_units` to the full corpus independently of any bias whenever the And's +/// min-folded `printing_matches` alone is already too broad a fraction of `n_printings` to trust +/// narrowing (found mid-investigation: a naive Python re-derivation missed this and only matched +/// 1,138/1,500 of the live build's own `scan_units`; modeling the guard brought that to 1,500/1,500) +/// -- the held-out NARROW subset alone (562/742 rows) moves 3.19M -> 2.61M, 433 improved / 117 +/// regressed / 12 tied. +/// +/// The held-out price-triple subset (`usd`/`eur`/`tix`, 2+ of them, 363/742 rows) moves 3.73M -> +/// 3.47M, 213 improved / 68 regressed / 82 tied -- proportionally in line with the whole population, +/// so the near-identical price columns' correlation (flagged as a risk for any independence-style +/// combination by Round 2) does not appear here: this is a flat multiplicative correction on +/// `calibrated_balls_into_bins`'s existing ball-count math, not a combination formula across the +/// leaves, so there is no per-leaf independence assumption for correlated fields to break. +/// +/// Smaller than 1.78 (less division, so a HIGHER effective ball count and a higher resulting +/// estimate) as the ledger's Round 3 assignment hypothesized: this population's undercount needed +/// raising, not the single-leaf population's saturating overcount that 1.78 corrects. +const COMPOSE_RANGE_AND_CLUSTER_BIAS: f64 = 1.1; + +/// Downscale for `scan_units` alone, on the ~25% of the SAME `is_cross_index_range_and` population +/// (see `COMPOSE_RANGE_AND_CLUSTER_BIAS`, just above) where `range_too_broad_to_narrow` -- a LATER, +/// independent guard just below in `acquire_plan_features` -- resets `eval_domain`/`scan_units` to +/// the full corpus (`n_cards`/`n_printings`) because the And's min-folded `printing_matches` alone +/// is too broad a fraction of `n_printings` to trust `domain_cards`. That reset is deliberately +/// conservative for `eval_domain`: real card-space narrowing gives up at the SAME threshold this +/// guard checks (`range_too_broad_to_narrow` is called from the real narrowing path too, not just +/// here), so a GatheredScan the router actually runs after this fires really does visit every card +/// -- confirmed directly against the real `cards_visited` counter, 0 total absolute error over 372 +/// sampled rows (see below), not merely assumed. `eval_domain` is therefore left untouched: it is +/// already exact for this population, and scaling it down would reintroduce the under-charge this +/// guard exists to prevent. +/// +/// `scan_units` is a different story. Every printing under every candidate card is NOT what +/// `exec_gathered_scan` actually bit-tests once card-space narrowing has given up -- the real +/// `printings_examined` counter (round-invariant; checked directly by rerunning 20 queries at +/// `num_warmups=0/trials=1` against `num_warmups=2/trials=5` with identical counters both times) +/// reads a stable ~70% of `n_printings`, not 100%, on this population specifically. `n_printings` +/// is still a sound UPPER bound (never measured over 1.0 on the sample below), so this is the same +/// "calibrated multiplicative correction on an already-conservative estimate" pattern as +/// `COMPOSE_RANGE_AND_CLUSTER_BIAS` and `COMPOSE_CANDIDATE_SPAN_BIAS` -- not a new exactness claim, +/// and not a 5th exemption added to the guard's own disjunction (the guard's reset still happens +/// unconditionally; only what it resets `scan_units` TO changes for this one shape). +/// +/// Fit as a plain scale sweep on 1,500 and2/and3 RANGE_FAMILIES queries (`unique=card`, same +/// population and precedent size as Rounds 1-3 of +/// `docs/issues/local-engine-gathered-scan-card-printing-varying-depth.md`), captured via +/// `explain_analyze`'s real `printings_examined` GatheredScan counter. Every sampled query is +/// `is_cross_index_range_and` by construction (`query()`'s family draws are distinct-without- +/// replacement over the 5 `RANGE_FAMILIES`, each its own index), so no separate shape filter was +/// needed on top of the guard-fired subset. 372/1,500 (24.8%) rows had the guard fire -- matching +/// the ~24-25% this constant's population was flagged at when `COMPOSE_RANGE_AND_CLUSTER_BIAS` was +/// fit. Split by `hash(query) % 2`: 191 calibration / 181 held-out. Swept 0.60-0.84 in steps of 0.01 +/// on the calibration half only; both halves' error-vs-scale curves are smooth, convex, and minimize +/// at the SAME 0.71 (closer agreement than `COMPOSE_RANGE_AND_CLUSTER_BIAS`'s two minima 0.04 apart). +/// Picked 0.7, inside the flat bottom of both curves and matching the sample's own mean/median +/// realized fraction (0.697 / 0.713) almost exactly. +/// +/// ```text +/// calibration (n=191) held-out (n=181) +/// scan_units total abs 5.64M (1.0) -> 1.92M (0.7) 5.40M (1.0) -> 1.90M (0.7) +/// improved / regressed 172 / 19 166 / 15 +/// price-triple subset (n) 79 71 +/// price-triple total abs 1.91M (1.0) -> 0.79M (0.7) 1.82M (1.0) -> 0.75M (0.7) +/// ``` +/// +/// The held-out price-triple subset (`usd`/`eur`/`tix`, 2+ of them) improves proportionally in line +/// with the whole population (62 improved / 9 regressed) -- same reasoning as +/// `COMPOSE_RANGE_AND_CLUSTER_BIAS`'s own price-triple check: this is a flat scale on an already- +/// computed ceiling, not a per-leaf independence combination, so the near-identical price columns' +/// correlation has nothing to break. +const COMPOSE_RANGE_AND_BROAD_SCAN_SCALE: f64 = 0.7; + +/// Downscale for `scan_units` alone, in the `CardRangePopcount` arm's own +/// `range_too_broad_to_narrow` broad-guard reset (see the arm below) -- a bare single range leaf +/// under `unique=card` (e.g. `usd>=0.24` alone, no `And` at all), a completely separate acquire +/// branch from `PrintingCompose`'s `is_cross_index_range_and` guard +/// (`COMPOSE_RANGE_AND_BROAD_SCAN_SCALE`, just above): confirmed live, `usd>=0.24` alone routes via +/// `count_source: card_range_popcount`, never `printing_compose`. +/// +/// Re-derived fresh rather than assumed to share `COMPOSE_RANGE_AND_BROAD_SCAN_SCALE`'s 0.7: Round +/// 5's diagnostic (`docs/issues/local-engine-gathered-scan-card-printing-varying-depth.md`) found +/// this arm's own broad-guard reset is the single largest bucket of pooled `GatheredScan`/`card` +/// error by magnitude (53.7% of the pooled total, median ratio 0.64) -- a materially different +/// realized fraction from the `PrintingCompose` sibling's 0.7, confirming the two arms should not +/// share one constant. +/// +/// Sampled 3,500 bare single-range `unique=card` queries where the guard fires +/// (`Shape(families=RANGE_FAMILIES, predicates=1, unique={"card"})`, filtered to `count_source == +/// "card_range_popcount"` and `eval_domain == n_cards && scan_units == n_printings`), captured +/// against the real `printings_examined` GatheredScan counter (GatheredScan is always tried as a +/// forced trial regardless of which plan wins, same trick Rounds 3/4 used). +/// +/// `eval_domain` (`n_cards`) is left untouched, same call as `COMPOSE_RANGE_AND_BROAD_SCAN_SCALE`, +/// but independently checked rather than assumed to transfer: 96.6% of rows read exactly 1.0 (mean +/// 0.975, not a clean 1.000 like the `PrintingCompose` sibling's 0 mismatches). The tail is real, +/// not sampling noise -- every row below 1.0 is a price field (`usd`/`eur`/`tix`) at an extreme +/// threshold, where most cards have no printing with that currency at all, so the real scan still +/// narrows out the null-complement even though the guard (correctly) judged the VALUE range itself +/// too broad to narrow on. Scaling the dominant 96.6%-exact regime down to chase that rare tail +/// would reintroduce the under-charge the guard exists to prevent, so this constant touches +/// `scan_units` only. +/// +/// `scan_units`'s realized fraction of `n_printings` is stable across all five `RANGE_FAMILIES` +/// (median 0.41-0.48 per field; overall median 0.43-0.45, mean ~0.45) -- close enough that a +/// per-field constant would only buy ~3.5% more total-error reduction than one flat scale (checked +/// directly: 31.4M vs an oracle 30.3M using each field's own median as its own best case), not worth +/// the extra parameterization. Split by `hash(query) % 2`: 1,765 calibration / 1,735 held-out. Swept +/// 0.30-0.60 in steps of 0.01 on the calibration half only; both halves' error-vs-scale curves are +/// smooth and convex, minimizing one step apart (0.43 calibration, 0.44 held-out). +/// +/// ```text +/// calibration (n=1,765) held-out (n=1,735) +/// scan_units total abs 96.0M (1.0) -> 15.4M (0.43) 93.3M (1.0) -> 16.0M (0.43) +/// improved / regressed 1,742 / 23 1,704 / 31 +/// ``` +/// +/// Verified against the real build, not just the Python-side sweep above: rebuilt with this +/// constant and re-ran the identical 3,500-query sample directly against it -- `scan_units` matched +/// `round(0.43 * n_printings)` exactly on every guard-fired row, and the real paired diff landed on +/// the exact same totals as the simulation above. +const COMPOSE_BARE_RANGE_BROAD_SCALE: f64 = 0.43; + +/// Downscale for `scan_units` alone, in `PrintingCompose`'s OWN `range_too_broad_to_narrow` +/// broad-guard reset (the arm below), for the shape `is_cross_index_range_and` was never meant to +/// cover: a bare single range leaf, or an `And` of range leaves that all share the SAME printing- +/// value index (a fused two-sided bound like `eur>=0.23 eur<=0.45`) -- see `is_same_index_range_only`. +/// +/// Round 5's diagnostic (`docs/issues/local-engine-gathered-scan-card-printing-varying-depth.md`) +/// found `GatheredScan`/`card`'s "single:range" bucket (a single family/predicate drawn from +/// `RANGE_FAMILIES`) at 3,041 rows, 53.7% of pooled error -- Round 6 fixed the slice of it that +/// reaches `CardRangePopcount` (`COMPOSE_BARE_RANGE_BROAD_SCALE`), but that arm is only ~4.7% of the +/// pooled `GatheredScan`/`card` cell by row count, smaller than the 3,041-row bucket. Round 7 traced +/// the rest: `card_range_popcount_applicable` requires BOTH sort-permutation directions for +/// `(sort_col, descending, cards.len())`, and a fused two-sided bound never satisfies +/// `bare_range_bounds` at all (it matches one comparison, not an `And` of two -- confirmed live, +/// `eur>=0.23 eur<=0.45` and a `cn>=8`-shaped query with an orderby lacking a permutation both route +/// via `count_source: printing_compose`) -- so both land here instead, in the ONE broad-guard branch +/// `is_cross_index_range_and`'s own doc already flagged as unscaled on purpose ("every other query +/// reaching this branch ... keeps today's unscaled `n_printings` ceiling"). +/// +/// Sampled 13,053 guard-fired rows from `Shape(families=RANGE_FAMILIES, predicates=1, +/// unique={"card"})` (varying orderby/direction/limit/offset the way `bench_cost_model_agreement.py` +/// does, not pinned, so the population is not an artifact of one page shape), filtered to +/// `count_source == "printing_compose"` and the guard signature (`scan_units == n_printings`), +/// against the real `printings_examined` GatheredScan counter. 5,300 of the 13,053 are a true bare +/// single leaf (no sort permutation for the drawn orderby/direction); 7,753 are a fused same-field +/// two-sided bound. `eval_domain` (`n_cards`) reads exactly 1.0 on every row (mean/median both +/// 1.000, cleaner than the `CardRangePopcount` sibling's 96.6%/0.975) -- left untouched, same call as +/// both existing broad-guard constants. +/// +/// `scan_units`'s realized fraction of `n_printings` is stable across the two sub-shapes (bare-single +/// median 0.473, fused-two-sided median 0.548 -- 0.075 apart, not worth two constants) and across all +/// five `RANGE_FAMILIES` (per-field median 0.46-0.58). Split by `hash(query|orderby|direction) % 2`: +/// 6,598 calibration / 6,455 held-out. Swept 0.20-0.80 in steps of 0.02 on the calibration half only; +/// both halves' error-vs-scale curves are smooth and convex, minimizing at the SAME 0.52 (each +/// sub-shape's own argmin, 0.48 and 0.55, brackets it tightly). +/// +/// ```text +/// calibration (n=6,598) held-out (n=6,455) +/// scan_units total abs 310.8M (1.0) -> 60.2M (0.52) 304.8M (1.0) -> 57.6M (0.52) +/// improved / regressed 6,560 / 38 6,422 / 33 +/// ``` +/// +/// Price-triple sanity (per-field, not cross-field correlation -- this is a flat scale on an +/// already-computed ceiling, not a per-leaf independence combination, so the near-identical price +/// columns' correlation has nothing to break, same reasoning as both sibling constants): `usd` +/// (0.534), `eur` (0.546), `tix` (0.574) all sit within the same band as `cn`/`date`/`year` +/// (0.46-0.49); `tix` reads highest but not an outlier. +const COMPOSE_SAME_RANGE_BROAD_SCAN_SCALE: f64 = 0.52; + /// Printings the gather BIT-TESTS per matching printing. /// /// `compose_scan_printings` was the composed bitmap's popcount, on the stated grounds that compose @@ -11038,7 +11858,28 @@ const COMPOSE_GATHER_SPAN_PER_MATCH: f64 = 1.47; /// lose, so it is a routing input even though compose never reads it. Calibrating the card estimate /// exposed it: `scan_units [printing_compose]` had been reading 0.75 with the two errors partly /// cancelling, and dividing `est_cards` by 1.78 moved it to 0.47. -const COMPOSE_CANDIDATE_SPAN_BIAS: f64 = 2.1; +/// +/// Round 1 of the printing-varying-leaf depth ledger +/// (`docs/issues/local-engine-gathered-scan-card-printing-varying-depth.md`) put a match-density +/// depth term (`(printings_per_card + 1) / (density + 1)`, see `scan_all`'s fallback arm) UNDER this +/// multiplier instead of the bare `printings_per_card` it used to scale — that term already prices +/// most of the "how deep does this query's own candidates get walked" question, so this constant's +/// remaining job is only the size bias described above (heavily-reprinted cards over-represented +/// among candidates), not the depth question too. +/// +/// Refit by a manual sweep (not `fit_cost_model.py`, which fits `cost.rs`'s per-unit NANOSECOND +/// rates against measured plan time -- a different regression than calibrating this dimensionless +/// FEATURE constant against the realized `printings_examined` counter): captured one build's raw +/// `domain_cards * expected_depth` (this constant pinned to 1.0) against measured +/// `printings_examined` over 1,500 and2/and3 queries on the RANGE_FAMILIES fields (unique=card), then +/// swept the multiplier in Python (no rebuild per candidate -- the cap at `n_printings` never binds +/// at 1.0 since `domain_cards <= n_cards` makes the uncapped product already `<= n_printings`, so +/// multiplying the raw capture by a candidate bias and re-applying the cap is equivalent to having +/// built with that bias). 0.7 minimized total absolute `scan_units` error on that sample (9.86M +/// against the old formula's 29.6M on the same rows, 946 improved / 544 regressed) -- lower than the +/// ~1.0 the ledger doc guessed going in, because the depth term alone still runs a bit hot relative +/// to the realized span, not because the size-bias premise above reversed sign. +const COMPOSE_CANDIDATE_SPAN_BIAS: f64 = 0.7; /// `balls_into_bins` with its measured clustering bias divided out of the BALL COUNT. See /// `COMPOSE_CARD_ESTIMATE_BIAS` for the 1.78 and why clustering causes it. @@ -11071,7 +11912,82 @@ const COMPOSE_CANDIDATE_SPAN_BIAS: f64 = 2.1; /// 2.2× as much. On the broad-residual class that inverted the pair — P3 measured 819.7 µs against P4's /// 1,308.4 with the model pricing them within 5 µs of each other. fn calibrated_balls_into_bins(k: usize, domain: usize) -> usize { - balls_into_bins_effective(k as f64 / COMPOSE_CARD_ESTIMATE_BIAS, domain).max(usize::from(k > 0)) + calibrated_balls_into_bins_with_bias(k, domain, COMPOSE_CARD_ESTIMATE_BIAS) +} + +/// `calibrated_balls_into_bins`, parameterized on which clustering bias divides `k` -- see +/// `COMPOSE_RANGE_AND_CLUSTER_BIAS`'s doc for the second population this exists for. +fn calibrated_balls_into_bins_with_bias(k: usize, domain: usize, bias: f64) -> usize { + balls_into_bins_effective(k as f64 / bias, domain).max(usize::from(k > 0)) +} + +/// Whether `composed` is an `And` of 2+ printing-varying range leaves spanning 2+ DIFFERENT printing +/// value indexes (`price_usd`/`price_eur`/`price_tix`/`collector_number_int`/`released_at`) -- the +/// shape `COMPOSE_RANGE_AND_CLUSTER_BIAS` is fit against, see that constant's doc. +/// +/// Reuses `bare_range_bounds` per child rather than re-deriving anything: it is the same leaf +/// dispatch `fuse_and_range_children` already runs over this And's children one call up, just +/// counting distinct index POINTERS instead of building intervals. Same-index children (a two-sided +/// `usd>=a usd<=b`) collapse to one entry here exactly as they fuse to one exact `k` there, so a +/// single-field bound never counts as "2+ different indexes" -- this is deliberately narrower than +/// "2+ range leaves", matching the population `est.result.card` never covers for (no pair-table / +/// arith-tuple / plane-compile tightening reaches a price/cn/date combination, so `exact_cards` +/// declines and `calibrated_balls_into_bins` is what actually answers it). +/// +/// O(children), pure `FilterExpr` matches and float comparisons (`bare_range_bounds` computes bounds +/// from the op/threshold, no index probe) -- cost is bounded by query length, never by match count, +/// and this only runs after `exact_cards` has already declined. +fn is_cross_index_range_and(composed: &FilterExpr, indexes: &Archived) -> bool { + let FilterExpr::And(children) = composed else { return false }; + let mut seen: Vec<*const Archived> = Vec::new(); + for child in children { + if let Some((idx, ..)) = bare_range_bounds(child, indexes) { + let ptr: *const Archived = idx; + if !seen.contains(&ptr) { + seen.push(ptr); + } + } + } + seen.len() >= 2 +} + +/// A bare range leaf, or an `And` of 2+ range leaves that all share the SAME printing-value index -- +/// the same-index counterpart `is_cross_index_range_and` deliberately excludes (see its own doc: "a +/// single-field bound never counts as 2+ different indexes"). Neither shape ever reaches +/// `CardRangePopcount`: a bare leaf can still land there when a sort permutation exists for the +/// query's orderby/direction (`card_range_popcount_applicable`), but a fused two-sided bound like +/// `usd>=a usd<=b` never does regardless of permutation (`bare_range_bounds`, `CardRangePopcount`'s +/// own gate, matches one comparison, not an `And` of two) -- both fall through to `PrintingCompose` +/// instead, where this identifies them for `COMPOSE_SAME_RANGE_BROAD_SCAN_SCALE`'s own broad-guard +/// scale. +/// +/// A mixed `And` (one range leaf plus a collection/rarity/numeric_other leaf) returns `false` here -- +/// every child must independently satisfy `bare_range_bounds`, which a non-range leaf never does -- +/// deliberately, since Round 5's diagnostic found that population's own broad-guard realized +/// fractions (0.21-0.36) read nothing like this shape's (0.46-0.58), so it needs its own future +/// investigation rather than silently sharing this constant. +/// +/// Same complexity bound as `is_cross_index_range_and`: O(children), no index probe, only runs after +/// `exact_cards` has already declined. +fn is_same_index_range_only(composed: &FilterExpr, indexes: &Archived) -> bool { + if bare_range_bounds(composed, indexes).is_some() { + return true; + } + let FilterExpr::And(children) = composed else { return false }; + if children.is_empty() { + return false; + } + let mut shared: Option<*const Archived> = None; + for child in children { + let Some((idx, ..)) = bare_range_bounds(child, indexes) else { return false }; + let ptr: *const Archived = idx; + match shared { + None => shared = Some(ptr), + Some(seen) if seen == ptr => {} + Some(_) => return false, + } + } + true } fn balls_into_bins(k: usize, domain: usize) -> usize { @@ -11114,7 +12030,7 @@ fn compose_gather_declines( mode: Mode, ) -> Option { // The gather's own decline is about the composed set it would page over, so it reads `result`. - let printing_matches = compose_printing_estimate(filter, indexes, offsets, printings.len()).result; + let printing_matches = compose_printing_estimate(filter, indexes, offsets, printings.len()).result.printing; // Artwork's domain is n_artworks, not n_cards. That used to be approximated by `cards.len()` // because the exact figure meant prefix-summing `artwork_groups` here -- real O(n_cards) work // paid just to maybe decline. It is a stored index now, so read the truth: the stand-in is @@ -11215,6 +12131,29 @@ fn compose_paging_with_total( } } +/// The segment `StreamedSelect`'s emission walk is bounded to, computed the SAME way +/// `exec_streamed_select` derives its own walk (`walk_bounds` over the identical +/// `(sort_col, descending, sort_bound)` triple) rather than by a second path that could silently +/// disagree with what dispatch actually walks. +/// +/// Free when the filter bounds nothing: `walk_bounds` returns the whole permutation on the +/// `bound.is_unbounded()` check before ever probing it, so the common case (most queries do not +/// filter on their own sort column) pays one branch, not a search. O(log n_cards) -- two binary +/// searches, nothing per candidate, nothing per printing -- when it does, mirroring +/// `CardRangePopcount`'s acquire-time range lookup a few branches up in `acquire_plan_features` +/// ("two binary searches, no scan"), the same style of cheap-exact acquire-time probe this cost +/// model already relies on elsewhere. `n_cards` when this `(sort_col, descending)` pair has no +/// permutation at all: `StreamedSelect` is inapplicable there and never reads this field, but +/// `mk_plan_feats` sets it uniformly for every acquire branch, since the shared feats have to cost +/// a competing `StreamedSelect` honestly regardless of which branch produced them. +/// +/// docs/issues/local-engine-gathered-scan-card-printing-varying-depth.md, Round 32. +fn perm_walk_span(ctx: &QueryCtx, params: &QueryParams) -> u32 { + ctx.indexes.sort_perms.get(params.sort_col, params.descending).map_or(ctx.n_cards(), |perm| { + walk_bounds(perm, ctx.cards, params.sort_col, params.descending, params.sort_bound).len() as u32 + }) +} + /// Cost features: the query-invariant fields filled once; the four that vary by /// count source passed in. Collapses each acquire branch's 8-field literal to one call. fn mk_plan_feats( @@ -11237,11 +12176,13 @@ fn mk_plan_feats( residual_tier_ns100, limit: params.limit as u32, offset: params.page_offset as u32, + perm_walk_span: perm_walk_span(ctx, params), broadcast_printings: 0, // PrintingCompose's legality broadcast-down (0 for ranges / precomputed planes) scatter_printings: 0, // range-slice k — set by both range-plan acquire branches (costed per-plan) project_printings: 0, // PrintingCompose's card/artwork projection pass; CardRangePopcount sets it too (for costing compose) popcount_words: 0, // PrintingCompose overrides this (result-space bitmap words) compose_paging: ComposePaging::Gather, // PrintingCompose overrides this (which paging strategy it'll actually use) + collection_broadcast_printings: 0, // PrintingCompose overrides this for a card-space collection leaf // `run_query_streamed`'s per-card artwork overhead applies to every candidate it visits, in // artwork mode only — so it rides `eval_domain` there and vanishes elsewhere. See // STREAM_ARTWORK_SEEN_PER_CARD_NS for the mechanism and the measurement. @@ -11484,8 +12425,14 @@ fn acquire_plan_features( // two: measured 31,508 cards / 97,206 printings visited against a `card_est` of 12,450, a // 3.2x gap no rate constant can absorb. The sibling `PrintingRangeScan` branch below assumes // the opposite (always unnarrowed) and its cells agree to within 1% -- this makes both exact. + // + // `n_printings` is still a sound upper bound here (never measured over 1.0), but loose: the + // real `printings_examined` GatheredScan counter reads a stable ~43% of it on this bare + // single-range population -- see `COMPOSE_BARE_RANGE_BROAD_SCALE`'s doc for the calibration. + // `eval_domain` is left at the full `n_cards`, same reasoning as that constant's doc: the + // dominant regime already reads exact there. let (eval_domain, scan_units) = if range_too_broad_to_narrow(k as usize, idx.len()) { - (n_cards, n_printings) + (n_cards, (f64::from(n_printings) * COMPOSE_BARE_RANGE_BROAD_SCALE).round() as u32) } else { (card_est, card_est) }; @@ -11545,7 +12492,7 @@ fn acquire_plan_features( // `plan_cost`. let composed_card_invariant = !touches_printing_field(composed); let est = compose_printing_estimate(composed, indexes, offsets, n_printings as usize); - let (printing_matches, broadcast, scatter) = (est.result, est.broadcast, est.scatter); + let (printing_matches, broadcast, scatter, collection_broadcast) = (est.result.printing, est.broadcast, est.scatter, est.collection_broadcast); // Two build kinds, charged at different rates: `broadcast` = legality broadcast-down (linear // pass), `scatter` = range-slice scatter (cheap). `project` = the second pass (printing→ // card/artwork), 0 for printing mode. Keeping all three separate is what lets a bare range's @@ -11570,8 +12517,30 @@ fn acquire_plan_features( } else { exact_result_total(composed, indexes, mode) }; - let est_cards = - exact_cards.unwrap_or_else(|| calibrated_balls_into_bins(printing_matches, n_cards as usize)); + // `is_cross_index_range_and` routes an And of 2+ different-index printing-varying range + // leaves to its OWN clustering-bias constant -- see `COMPOSE_RANGE_AND_CLUSTER_BIAS`'s doc + // for why `COMPOSE_CARD_ESTIMATE_BIAS` (fit on a single-leaf broadcast population) undershoots + // here instead. Checked only once `exact_cards` has already declined, so a query this + // mechanism never reaches (a single leaf, or a shape the pair table/arith-tuple/plane-compile + // paths already answer exactly) pays nothing extra. + let est_cards = exact_cards.unwrap_or_else(|| { + if is_cross_index_range_and(composed, indexes) { + calibrated_balls_into_bins_with_bias(printing_matches, n_cards as usize, COMPOSE_RANGE_AND_CLUSTER_BIAS) + } else { + calibrated_balls_into_bins(printing_matches, n_cards as usize) + } + }); + // Exact PRINTING total for the same composed filter -- valid as the candidate cards' full + // printing SPAN (what `scan_all` below needs) only when the filter is CARD-INVARIANT + // (`composed_card_invariant`): for a card-invariant field, every printing of a matching card + // matches too, so the exact MATCH count and the matching cards' full span are the same number. + // For a printing-VARYING field (border/rarity/legality/year/date/price/cn) they are not: a + // card can have printings that don't match the leaf's own value but are still part of that + // card's range, which `GatheredScan` walks in full. Checked directly against real data before + // adding the gate: applied unconditionally, this regressed 1,077 bare-leaf queries, every one + // printing-varying (`border:borderless`, `r>=mythic`, `year:2006`); scoped to card-invariant + // leaves only, it is exact by construction, not an approximation with a lucky population. + let exact_printing_span = composed_card_invariant.then(|| exact_result_total(composed, indexes, Mode::Printing)).flatten(); // The card count the MATERIALIZING alternatives walk, which stops being `est_cards` once the // estimate has been tightened. `est.candidate` is the untightened `min` over single leaves, and // that is what narrowing actually leaves them: it declines broad children (`border:black` at 87% @@ -11584,16 +12553,65 @@ fn acquire_plan_features( // // For an `And`, that "declines broad children" premise is false specifically for // `ColorCmp`/`NumericCmp(Cmc|Power|Toughness)`/`Devotion`: `narrow_rec` genuinely intersects - // them (see `compile_children_once`'s doc). `est.domain_hint`, computed alongside `est.result` - // in the SAME pass over the And's children (not re-derived here), can only be >= the real - // domain (it ignores what non-plane siblings would additionally narrow), so `min`-ing it with - // the existing estimate is a strict tightening, never a regression. - let domain_cards = if est.candidate == est.result { + // them (see `compile_children_once`'s doc). `est.card`, computed alongside `est.result` in the + // SAME pass over the And's children (not re-derived here) -- a real CARD count now, not a + // printing-scaled one wearing a card-shaped name (`domain_hint_is_card_space_not_printing_scaled`) + // -- can only be >= the real domain (it ignores what non-plane siblings would additionally + // narrow), so `min`-ing it with the existing estimate is a strict tightening, never a regression. + let domain_cards_before_card = if est.candidate.printing == est.result.printing { est_cards } else { - let calibrated = calibrated_balls_into_bins(est.candidate, n_cards as usize); - est.domain_hint.map_or(calibrated, |dc| dc.min(calibrated)) + calibrated_balls_into_bins(est.candidate.printing, n_cards as usize) }; + // `est.card` additionally tightens `domain_cards`, beyond the `else` branch above, but ONLY for + // a genuine `And` -- found live (`id:g border:white`, artwork mode): `est.candidate == est.result` + // (both 5131, border:white's own printing count) took the `est_cards` branch, and `est_cards` + // falls back to `exact_result_total`'s OWN, narrower 2-child-pair-table check, which doesn't + // cover a `ColorCmp`+`TextExact(Border)` combination and returned `None` -- landing on + // `calibrated_balls_into_bins`'s 2,756 guess despite `compose_printing_estimate`'s own And arm + // (a DIFFERENT, newer mechanism, `best_other`'s existential+card-invariant plane intersection) + // already knowing the exact answer, 576, on `est.card`. + // + // Scoped to `And` specifically, NOT applied to bare leaves too, after that broader version was + // checked directly against real data and found to regress 809 queries -- every one a genuinely + // broad bare leaf (`eur>0.16`, `tix<0.04`, `border:black` alone, ~100% of the corpus) where + // `est.card` (from `exact_result_total`'s OWN bare-leaf Mode::Card arms, the SAME function + // `est_cards` already calls) came back wrong by a wide margin -- e.g. `eur>0.16` real 31,724, + // `est.card` 19,992. That mismatch predates this session (it is `exact_result_total`'s own + // Mode::Card answer, not `compose_printing_estimate`'s), and was invisible before only because + // `range_too_broad_to_narrow`'s full-corpus fallback happened to override it every time a bare + // leaf was this broad -- a real, separate bug, worth its own investigation, but out of scope + // for this pass. Restricting the extra tightening to `And` keeps today's fix to the case it was + // actually verified against, without newly trusting a leaf-level answer nothing here checked. + let is_and = matches!(composed, FilterExpr::And(_)); + let domain_cards = if is_and { est.result.card.map_or(domain_cards_before_card, |dc| dc.min(domain_cards_before_card)) } else { domain_cards_before_card }; + // Card mode's `push_card_matches`/`Prefer::Default` loop settles a card in exactly ONE printing + // when the composed field is card-invariant: either the first printing checked satisfies the + // residual (found, done) or it does not -- and since every OTHER printing of that card carries + // the identical value, none of them would satisfy it either, so the loop has nothing left to + // gain by continuing (confirmed directly against the executor: `found` is set or the loop simply + // runs out at `end`, one check per printing, never a rescan). Declared here, not inside the + // `Mode::Card` arm below, because the LATER `range_too_broad_to_narrow` guard (see its own doc) + // also needs it -- a query this exact and this broad (`cmc>=0`, `id:bgruw`) is exactly the shape + // that guard exists to catch, and without this exemption it clobbers the correct answer computed + // below back to the full corpus. + // + // Gated on `est.result.card == Some(domain_cards)`, not `composed_card_invariant` alone: the + // "one printing settles it" argument only holds when `domain_cards` carries ZERO false + // positives, since a false-positive candidate's residual is false on every printing and the + // `Prefer::Default` loop has no way to know that in advance -- it still walks every printing + // before giving up, `examined = span`. `est.result.card` is populated only for the leaf types + // already verified this session to be a real, exact card-space count (ColorCmp/Legality/ + // CollectionCmp/Border/Devotion/NumericCmp), and deliberately left `None` for bare ranges/ + // rarity precisely because their own card counts are NOT reliably exact + // (`RangeCardCounts::distinct_cards`'s broad-range undercount, `eur>0.16` real 31,724 vs + // computed 19,992) -- reusing that field instead of a fresh `exact_cards.is_some()` check keeps + // this scoped to the population already trusted, not the wider one `exact_result_total` alone + // would admit. Measured: bare `cmc`/`power`/`toughness`/`color_identity` read a median 3.08x + // over (== `printings_per_card` exactly) across their WHOLE bucket, not just the near-universal + // queries -- the gap is structural, not a selectivity artifact, because card-invariance makes + // depth-1 true regardless of how selective the predicate is. + let card_invariant_domain_exact = composed_card_invariant && est.result.card == Some(domain_cards); // What the MATERIALIZING alternatives scan if compose loses. Every mode narrows -- a // composable filter has an index for every leaf -- so all three are the NARROWED counts. // Printing mode took the unnarrowed universe while card/artwork took a narrowed count; only @@ -11612,8 +12630,94 @@ fn acquire_plan_features( // `printings_examined` of exactly 97,206. The clamp makes that cell exact. It matters for routing // because this feature is 76% of P3's arm on the broad-residual class, where it drove P3 to // pred/meas 1.53 while P4 sat at 0.88 — the pair inverted, with both plans over the same feature. - let scan_all = - |cards: usize| (((cards as f64) * printings_per_card * COMPOSE_CANDIDATE_SPAN_BIAS) as usize).min(n_printings as usize); + // An earlier version of this used `exact_result_total(composed, Mode::Printing)` here whenever + // `est.candidate.printing == est.result.printing`, on the theory that a bare leaf's own exact + // MATCH count is the candidate cards' printing span. Checked directly against real data (a + // paired diff on `scan_units` specifically, not just the pooled percentile view): 1,077 + // queries regressed, every one a bare leaf (`border:borderless`, `r>=mythic`, `year:2006`). + // The two quantities are not the same: a card can have OTHER printings that don't match the + // leaf's own value but are still part of that card's range, which `GatheredScan` walks in + // full -- `printings_examined` counts that whole span, not just the matching subset. Removed + // rather than fixed further; `exact_domain` below doesn't have this flaw because + // `card_bits_span_total` (which built it) already sums each candidate card's FULL span, not a + // filtered match count -- confirmed by the same paired diff, run on this fix: `id:br r<=uncommon` + // 97,812 -> 23,394 against a real 23,394, exact. + // + // Whether `est.exact_domain` is actually what won `domain_cards`' `is_and` tightening above -- + // `domain_cards` is `.min()`-ed from TWO independent sources (`domain_cards_before_card` and + // `est.result.card`, which is `est.exact_domain`'s own `.card`), so `exact_domain.printing` is + // only the CANDIDATE set's true span when its `.card` is the side that actually won. If the + // OTHER side was tighter, the real candidate set is smaller than what `exact_domain` describes, + // and using its printing span would overstate the span of a set that isn't the one being priced. + let exact_domain_won = is_and && est.exact_domain.is_some_and(|ed| ed.card == Some(domain_cards)); + let scan_all = |cards: usize| { + if exact_domain_won + && let Some(exact_domain) = est.exact_domain + { + // Bare, not `.min()`-ed against the statistical guess: tried that (real data first -- + // it always is), and it made things worse both by row count (732/119 improved/regressed + // vs 772/79 bare) and by total magnitude. Unlike `domain_cards` (where `est.card` is a + // mathematically guaranteed upper bound on the true joint count, so `.min()` can only + // tighten), there's no such guarantee here -- `exact_domain` only covers the children + // `best_other`/the arith-merge actually captured (a sibling those mechanisms skip, a + // residual range or an uncaptured second existential, narrows the REAL candidate set + // further without `exact_domain` knowing it), and the true value is SOMETIMES bigger + // than both candidates, not just smaller. "Take the smaller number" isn't a safety net + // for a quantity that can be wrong in either direction -- it just doubles down on + // whichever candidate under-shoots. + // + // The 79 residual regressions (`f:penny produces:u`, `border:black id:r`, ...) are + // accepted as a known, documented gap: summed by MAGNITUDE, not row count, they add + // 134,494 units of error against 6,805,680 shed by the 772 improved rows (total absolute + // error across the whole affected population: 8,008,106 -> 1,336,920, an 83% cut) -- + // the worst single regression is 6,235, the best single improvement 74,418 + // (`id:br r<=uncommon`: 97,812 predicted -> 23,394, an exact match). A raw + // improved/regressed row COUNT alone overstates how mixed this is: several of the + // "regressed" rows are swings like 3 -> 4 against a true 1, noise on a query this + // narrow already costs nothing to get wrong. + return exact_domain.printing.min(n_printings as usize); + } + // `est.candidate == est.result` mirrors `domain_cards`' own guard: nothing was tightened + // away from the leaf/pair-exact estimate, so `exact_printing_span` (already gated to + // card-invariant filters above, where match count IS the matching cards' full span) is the + // true span under exactly these candidates. + if est.candidate.printing == est.result.printing + && let Some(printings) = exact_printing_span + { + return printings.min(n_printings as usize); + } + // Printing-varying leaf (price/collector_number/released_at, or an And of them): no + // "first printing settles it" guarantee, so this candidate's card DOES get walked past + // its first printing when that printing doesn't happen to satisfy the residual. The old + // formula priced every candidate at the CORPUS-WIDE average reprint depth + // (`printings_per_card`) regardless of how selective the predicate is at the printing + // level -- as wrong for `price_usd<0.05` (near-universal, few candidates settle deep) as + // for `price_usd=99.99` (rare, most candidates that have ANY match burn their whole span + // finding it). + // + // `printing_matches / cards` is the query's own match DENSITY: how many of each + // candidate's printings match, on average, among cards known to have at least one. Model + // each candidate's matching printings as landing at uniformly random positions among its + // `printings_per_card` slots (order-statistics on the position of the FIRST match, not a + // per-card exact position this has no data for) -- expected position of the first hit + // among `printings_per_card` slots with `density` of them set is + // `(printings_per_card + 1) / (density + 1)`. `density -> printings_per_card` (every slot + // set, i.e. card-invariant) collapses this to 1 (first printing always hits, the case the + // sibling `card_invariant_domain_exact` fast path already prices exactly); `density -> 0` + // (barely any matching printings) pushes it toward the corpus-average span, never past it + // -- `.min(printings_per_card)` is a belt-and-suspenders cap for float edge cases, not a + // load-bearing clamp (the order-statistics formula is already bounded by construction). + // + // Both inputs are already-computed scalars (`printing_matches` = `est.result.printing`, + // `cards` = `domain_cards` at every call site, `printings_per_card` = a corpus-wide ratio + // computed once per acquire) -- no new index probe, no per-query scan, cost independent of + // match/printing/candidate count. `COMPOSE_CANDIDATE_SPAN_BIAS` still corrects the + // remaining "candidates are more reprinted than an average card" size bias (see its own + // doc) on top of the depth term, refit for this shape (see the constant's own doc). + let density = (printing_matches as f64) / (cards as f64).max(1.0); + let expected_depth = ((printings_per_card + 1.0) / (density + 1.0)).min(printings_per_card); + (((cards as f64) * expected_depth * COMPOSE_CANDIDATE_SPAN_BIAS) as usize).min(n_printings as usize) + }; let (result_total, project, popcount_words, eval_domain, scan_units) = match mode { Mode::Printing => { // `exact_total` for the RESULT, `printing_matches` for everything else. They are not the @@ -11632,7 +12736,14 @@ fn acquire_plan_features( // `printing_matches.min(n_cards)`, which reads a median 1.99x the deduped // `matches_pushed` counter -- p10 1.01, so it is over on nearly every query. Two // names for one quantity, one of them wrong. - (est_cards, printing_matches, (n_cards as usize).div_ceil(64), domain_cards, scan_all(domain_cards)) + // + // `card_invariant_domain_exact` (computed above, alongside `domain_cards` -- see its own + // doc) replaces the generic `printings_per_card * BIAS` fallback with `domain_cards` + // directly whenever the composed field is card-invariant and exact: the executor settles + // each such card in exactly one printing, so the average-reprint-rate multiplier is not + // an approximation here, it is pricing a rescan that never happens. + let scan_units = if card_invariant_domain_exact { domain_cards } else { scan_all(domain_cards) }; + (est_cards, printing_matches, (n_cards as usize).div_ceil(64), domain_cards, scan_units) } Mode::Artwork => { // `result_total` is consumed as a per-RESULT count (GatheredScan's push term, @@ -11708,21 +12819,77 @@ fn acquire_plan_features( // `And` and matches no bare-leaf shape at all, silently undoing the #1005 collection-leaf // exemption for every query combining it with one of these fields. `plane_leaves_nothing_to_verify` // reaches the `pow<=2` case through `filter == True` instead, without disturbing that one. - let (eval_domain, scan_units) = if !(compose_leaf_nothing_to_verify(filter) || plane_leaves_nothing_to_verify(filter, mode, plane, indexes)) + // + // A third exemption, `is_and && est.card.is_some()`: the same failure mode again, this time for + // an `And` whose EXACT card intersection this session's `compose_printing_estimate` work now + // knows (`f:timeless r>=rare`, card mode: `domain_cards` a correct, verified 5,854 against + // `est.result`/`printing_matches` of 36,623 -- 37% of the corpus, over `MAX_NARROW_FRACTION`, so + // this guard fired and threw the exact card count away for the full 31,724-card corpus). + // `domain_cards` is `.min()`-ed against `est.card` for exactly this `is_and` case (see just + // above), so an exact card count in hand there can only make `domain_cards` tighter than a + // fabricated one, the same non-regression argument the other two exemptions already rely on. + // NOT extended to bare leaves -- see `is_and`'s own doc for the 809-query regression that + // surfaced when this was tried unscoped. + // A fourth exemption, `card_invariant_domain_exact` (computed alongside `domain_cards` above -- + // see its own doc): the same failure mode again, this time for a BARE card-invariant leaf whose + // exact card count this session's `compose_printing_estimate` work now knows (`cmc>=0`, card + // mode: `domain_cards` a correct, verified 31,724, discarded back to the full corpus by this + // guard because a bare leaf never reaches the `is_and` exemption above it). Safe across every + // mode, not just `Mode::Card`: whatever `scan_units`/`eval_domain` the arm above already computed + // for a card-invariant, false-positive-free domain is either that same exact card count (Card + // mode) or `exact_printing_span`'s exact span of it (Printing/Artwork, gated identically on + // card-invariance) -- never a value this guard would improve on by falling back to `n_printings`. + let (eval_domain, scan_units) = if !(compose_leaf_nothing_to_verify(filter) + || plane_leaves_nothing_to_verify(filter, mode, plane, indexes) + || (is_and && est.result.card.is_some()) + || card_invariant_domain_exact) && range_too_broad_to_narrow(printing_matches, n_printings as usize) { - (n_cards as usize, n_printings as usize) + // `eval_domain` stays the full `n_cards` unconditionally -- see + // `COMPOSE_RANGE_AND_BROAD_SCAN_SCALE`'s doc for why that half of the reset is already + // exact for the one shape this narrows further. `scan_units` alone gets a downward scale, + // keyed on which range shape reached this branch: `is_cross_index_range_and` (2+ + // different-index range leaves) and `is_same_index_range_only` (a bare single range leaf, + // or a fused same-field two-sided bound -- see its own doc for why neither reaches + // `CardRangePopcount`) each have their own fitted constant. Anything else reaching this + // branch (a broadcast legality, a range mixed with a collection/rarity/numeric_other leaf, + // ...) never had either scale's calibration sample in it, so it keeps today's unscaled + // `n_printings` ceiling. + // + // BOTH scales are `Mode::Card`-only, and deliberately so (Round 28 fix, #costcell-28): + // both were fit exclusively against `unique=card` samples (Round 4's re-derivation and + // Round 7's fit are each explicit about this), because card mode's kernels short-circuit + // per candidate and printing/artwork mode's do not -- confirmed directly against a fresh + // `bench_feature_accuracy.py`-style sample of this exact guard-fired population: `scan_units + // == n_printings` held EXACTLY on every `Mode::Printing`/`Mode::Artwork` row (p10/p50/p90 of + // `printings_examined / n_printings` all landing on 1.000, zero spread -- those loops really + // do walk the full candidate-card printing span with no early exit), while `Mode::Card`'s + // same population reads the ~0.52/0.7 fraction that motivated the fit. Applying either scale + // to `Mode::Printing`/`Mode::Artwork` was silently manufacturing a `Mode::Card`-shaped + // under-count out of a population that had no such property, and is what tipped + // `bench_feature_accuracy.py`'s pooled `scan_units` cell from clean (main) to UNDER-COUNTS + // (this branch) -- see `docs/issues/local-engine-gathered-scan-card-printing-varying-depth.md`'s + // Round 28 entry. + let scan_units = if matches!(mode, Mode::Card) && is_cross_index_range_and(composed, indexes) { + ((n_printings as f64) * COMPOSE_RANGE_AND_BROAD_SCAN_SCALE).round() as usize + } else if matches!(mode, Mode::Card) && is_same_index_range_only(composed, indexes) { + ((n_printings as f64) * COMPOSE_SAME_RANGE_BROAD_SCAN_SCALE).round() as usize + } else { + n_printings as usize + }; + (n_cards as usize, scan_units) } else { (eval_domain, scan_units) }; // The tier is what the MATERIALIZING alternatives pay per candidate, so it must be asked about // the predicate THEY see (`filter` + `plane`), not about `composed` — and gated exactly as - // `prepare_candidates` gates it, or the router charges a `card_pass` the kernels will skip. On a - // card-invariant legality format `card_pass` resolves at card level for every card, so - // `printings_examined` reads 0 and both the per-card residual and the per-row scan are dead - // terms; charging them anyway was 92-94% of P3's predicted cost on `f:modern`, `f:gladiator`, - // `f:commander` and `f:predh`. `residual_exact` is unavailable here (this branch never narrows), - // so this is the conservative half of the executor's disjunction: it can over-charge, never under. + // `prepare_candidates` gates `all_match_known` (skipping `card_pass`), or the router charges a + // `card_pass` the kernels will skip. On a card-invariant legality format `card_pass` resolves at + // card level for every card, so `printings_examined` reads 0 and both the per-card residual and + // the per-row scan are dead terms; charging them anyway was 92-94% of P3's predicted cost on + // `f:modern`, `f:gladiator`, `f:commander` and `f:predh`. `residual_exact` is unavailable here + // (this branch never narrows), so this is the conservative half of the executor's disjunction: it + // can over-charge, never under. // // `compose_leaf_nothing_to_verify` closes the analogous gap for a bare compose-exact collection // leaf: `otag:triggered-ability` (47% dense) measured StreamedSelect predicted at 1026us against @@ -11730,8 +12897,33 @@ fn acquire_plan_features( // query to `PrintingCompose` (176.5us) despite it being ~4x slower (#1005). `plane_leaves_ // nothing_to_verify` cannot see this — it only recognizes the legality plane's rewrite to `True`, // and a compose-exact leaf is never rewritten that way. - let nothing_to_verify = - plane_leaves_nothing_to_verify(filter, mode, plane, indexes) || compose_leaf_nothing_to_verify(filter); + // + // ANDed, not ORed like `plane_leaves_nothing_to_verify`'s own combined (filter-must-be-True-too) + // test: `compose_leaf_nothing_to_verify` and `cost_plane_nothing_to_verify` each answer for their + // OWN half of the query (the residual `filter` and the compiled `plane` respectively), and each + // can be satisfied while the OTHER half still needs a per-printing check. Originally this OR'd + // `plane_leaves_nothing_to_verify(filter, mode, plane, indexes)` (which itself requires filter == + // `True`, so it only ever fired when there was no separate residual) with + // `compose_leaf_nothing_to_verify(filter)` alone — but the latter says nothing about `plane`, so + // whenever `plane` ALSO existed and was existential, a residual that happened to be a bare safe + // collection leaf (`t:swamp`, `otag:X`, `keyword:Y`) still forced `nothing_to_verify = true` for + // the WHOLE query, silently discarding the plane side's real per-printing existential work. + // Found live: `t:swamp tou=5 border:black`/card kept `residual_tier_ns100 == 0` even after + // `cost_plane_nothing_to_verify` alone was scoped to reject it, because the OR's other arm + // (`compose_leaf_nothing_to_verify(t:swamp)`) still fired on its own. + // + // The `all_match_known` claim in the comment above (matching `prepare_candidates`'s `card_pass` + // skip) is true for `card_pass` — both use the same shaped test on each half — but incomplete + // whenever `plane` is existential (`plane_expr_is_existential`, any family): the executor's + // `existential_plane_for` runs a SEPARATE per-candidate-printing walk in exactly that case (see + // `cost_plane_nothing_to_verify`'s doc), and that walk is real work neither half's own "I have + // nothing to verify" claim accounts for on its own. Found live: `cmc>=1 cmc<=5 border:black`/card + // — `residual_tier_ns100 == 0` priced `GatheredScan` at 326,263ns against a measured + // 1,015,209-1,290,166ns; `f:oldschool cmc>=1 cmc<=5`/card (a DIVERGENT legality format in the + // same shape, no rarity/border involved) — predicted 18,519ns against a measured 58,625ns + // (docs/issues/local-engine-gathered-scan-undercosted-arith-existential-and.md). + let filter_nothing_to_verify = matches!(filter, FilterExpr::True) || compose_leaf_nothing_to_verify(filter); + let nothing_to_verify = filter_nothing_to_verify && cost_plane_nothing_to_verify(plane, indexes); let tier = if nothing_to_verify { 0 } else { verify_cost_tier(composed) }; // `GatheredScan` walks every printing of every candidate card, so its scan feature is the candidate // SPAN. `scan_all` estimates that span as `est_cards x` the corpus-average printings-per-card `x 2.1`, @@ -11748,9 +12940,39 @@ fn acquire_plan_features( // the same boolean as the tier because the grading inverts on the other population — with a real // residual `scan_units` is right at p50 0.97 and `printing_matches` badly under at 0.39. // - // Fixes the BIAS, not the spread: both rows read p90/p10 4.5, so what remains is the candidate - // count's own variance (`eval_domain` grades p90/p10 3.1 here) and is not a scan-feature problem. - let scan_units = if nothing_to_verify { printing_matches } else { scan_units }; + // That comparison pooled every `prefer` together, and the two are not actually competing for the + // same rows: `push_card_matches`'s `Mode::Card`/`Prefer::Default` loop settles a card-invariant + // match in exactly ONE printing (the plane pass already proved `all_match`, so the very first + // printing checked is the pick, no rescan), while any OTHER `prefer` must score every printing to + // find the best one and genuinely does walk the full span -- the identical distinction the sibling + // `PlanePopcountOrder` branch above already makes explicit for the same reason. `printing_matches` + // was the better SINGLE number for a population blending both regimes (median 0.93 against 1.76), + // but it is the wrong one for the ~85% `Prefer::Default` share (`REALISTIC_PREFER_WEIGHTS`) once + // the two are told apart: `eval_domain` is what that regime's executor actually walks. + // `PlanFeatures` has no `prefer` field, but the NUMBER this call computes can still condition on + // the `prefer` this call was actually made with -- it is not predicting for an unknown future + // prefer, it already knows this one, same as the plane branch above. + // + // Gated on `card_invariant_domain_exact`, not `matches!(mode, Mode::Card) && + // matches!(prefer, Prefer::Default)` alone: `nothing_to_verify` can be true from + // `compose_leaf_nothing_to_verify` too, and neither flag says `eval_domain`/`domain_cards` is + // actually EXACT there -- checked directly against real data first (paired diff on `scan_units`): + // the looser condition regressed `cmc>=2 cmc<=3 pow>2` 5,463 -> 11,405 against a real 1,772, + // because an all-arith-tuple-eligible `And` is exactly the shape `best_other`'s intersection + // (the thing that makes `est.result.card` trustworthy) does not cover -- `is_arith_tuple_eligible` + // children are filtered OUT of `best_other`'s `card_invariant`/`existential` partition, and the + // arith-merge that would otherwise pick them up is itself gated on `best_other` already being + // `Some` (see the `And` arm's own doc) -- so `domain_cards` there falls back to + // `calibrated_balls_into_bins`, an ESTIMATE, not the exact answer this branch needs. + // `card_invariant_domain_exact` is the same proof already required for the bare-leaf `scan_all` + // shortcut and the `range_too_broad_to_narrow` exemption above -- reusing it here instead of a + // fresh, looser condition keeps all three tied to one verified population rather than three + // separately-trusted ones. + let scan_units = if nothing_to_verify { + if card_invariant_domain_exact { eval_domain } else { printing_matches } + } else { + scan_units + }; let mut feats = mk_plan_feats(ctx, params, result_total as u32, eval_domain as u32, scan_units as u32, tier); feats.residual_card_invariant = composed_card_invariant; // What `StreamedSelect` actually examines here, which is NOT `scan_units`. P4 walks a card's whole @@ -11792,10 +13014,96 @@ fn acquire_plan_features( // argued for -- the `tier == 0` arm above is where it used to be wrong. ((scan_units as f64) * share).max(eval_domain as f64) as u32 } else { - scan_units as u32 + // Round 30 of the printing-varying-leaf depth ledger + // (docs/issues/local-engine-gathered-scan-card-printing-varying-depth.md): the bare + // `scan_units` inheritance here was fine BEFORE Round 1 revised `scan_all`'s fallback + // downward for a printing-varying leaf (price/collector_number/released_at, or an And of + // them) -- not because it was pricing the right thing, but because the old, cruder + // `domain_cards * printings_per_card * 2.1` term happened to be large enough to also mask + // a SEPARATE cost this feature has never priced: `run_query_streamed` runs a counting pass + // over every candidate (the SAME per-card cost `scan_units` already estimates -- confirmed + // directly, `f:pioneer cn>=30 cn<=39` reports IDENTICAL `printings_examined` for both + // plans, 2,449), then a SECOND pass that re-derives `card_pass` and re-walks the printing + // span for every MATCHING card to select the page. `cost.rs`'s `StreamedSelect` arm already + // has a `runs_small_gather` term for this second pass's OWN `total <= *STREAM_MIN_MATCHES` + // branch (`STREAM_SMALL_TOTAL_FLOOR_PER_CARD_NS * n_cards`) -- but that floor is a per-CORPUS + // constant, fit on a population where the branch's `matches` count was small enough that the + // redo itself was negligible next to the O(n_cards) "scan every stored count" overhead the + // floor was measured against. It cannot vary with a DIFFERENT query's `matches`, so on a + // query sitting near the `STREAM_MIN_MATCHES` ceiling (853 matches on `f:pioneer + // cn>=30 cn<=39`) the floor alone materially undershoots: real `ns_finish` 65.3us against a + // 32.4us floor (`n_cards=31,724 * 1.02`), the remainder being exactly the redo this note + // describes. + // + // Round 30 calibrated this against `ns_finish` minus the floor's own contribution (a + // wall-clock RESIDUAL, converted to `stream_scan_units` units via the existing, untouched + // `STREAM_SCAN_PER_ROW_NS`), because no structural counter existed for the redo pass's real + // work -- `push_card_matches`'s return value was discarded in that loop. Fit: median 1.32, + // p10 -38.7, p90 12.7. + // + // Round 31 of the same ledger entry closed that gap: `PhaseStats::redo_examined` now + // captures the exact printing count the redo pass re-examines (see its own doc -- free, + // `push_card_matches` already computed and returned it). Refitting DIRECTLY against that + // real counter instead of the noisy wall-clock/rate-conversion chain, over 2,926 held-in / + // 3,019 held-out `unique=card` `printing_compose` rows (same `bench_regret_matrix.py + // --mode realistic` corpus, same real-dispatch gate: small-total branch confirmed to have + // actually run via `perm_steps == 0` AND `matches_pushed > 0`, PLUS a guard Round 30's own + // gate missed -- `page_offset < matches_pushed`, ruling out the OTHER `perm_steps == 0` exit + // where `page_offset >= total` returns before the redo loop ever runs but still reports the + // counting pass's `matches_pushed`) surfaced TWO real numbers, not one, and they disagree: + // + // - The per-row ratio's MEDIAN is 1.0, and it minimizes held-out total absolute error on the + // `redo_examined` counter itself (2.476e6 at the old 1.32 -> 2.388e6 at 1.0) -- the best + // POINTWISE fit. + // - The candidate-WEIGHTED mean (`sum(redo_examined) / sum(redo_candidates)` over the same + // calib half) is 2.237 -- a worse pointwise fit (held-out error 2.752e6) because the + // distribution is heavily right-skewed (p10 0.15, p90 10.2): most rows sit near/under 1.0, + // but the flip-query population this bias exists to fix draws disproportionately from the + // heavy tail (a query only flips to the wrong plan when its real redo cost was + // under-priced, which the tail-heavy rows are), so a pointwise-optimal median systematically + // UNDER-corrects exactly the rows that matter for routing. + // + // Checked directly, not assumed: replaying the SAME reproduced 118-query f3f4a017 flip set + // (see the round's own doc entry) against both candidates confirms the divergence is real -- + // 1.0 fixes 43/118 and actively REGRESSES 9 of the 52 Round 30's own 1.32 already fixed (0 + // newly fixed); 2.237 fixes 64/118, regresses ZERO of Round 30's 52, and gains 12 more. Same + // asymmetry Round 30's own bias sweep found (a false-positive/false-negative trade-off, not a + // free lunch), just resolved this time against a real ground-truth counter instead of a + // guessed multiplier: 2.237 is kept because it is the real, structurally-grounded statistic + // that does not regress the live routing outcome, not because it minimizes the ns-error metric + // in isolation -- the same "live outcome over pointwise ns-error" precedent Round 30 itself + // set. Ratio by candidate-count bucket reads flat at ~1.0-1.3 across the whole small-total + // range (0-50, 50-150, 150-400, 400-1024 candidates) -- no saturation or other non-linearity + // to chase, so a flat linear bias remains the right shape; the skew is in the PER-QUERY + // residual (as Round 30 itself already flagged), not in candidate count. + // + // Above the threshold the small-total branch never runs (the permutation walk does + // instead, bounded by `limit`), so the redo candidate count is capped at `limit` rather + // than dropped to zero outright -- a discontinuity at the threshold would make the acquire + // estimate's own noise (this exact query's OWN estimate, 1,983, sits just above the 1,024 + // threshold despite really landing in the small-total branch) an all-or-nothing coin flip + // instead of a graceful degradation. + // + // `Mode::Card` only: the calibration sample above is `unique=card` exclusively, and + // `Printing`/`Artwork` never take `push_card_matches`'s early-break arm (see its own doc), + // so `scan_units` already prices the full span there -- a population this round did not + // check. + const STREAM_SMALL_TOTAL_REDO_BIAS: f64 = 2.237; + let redo = if matches!(mode, Mode::Card) { + let redo_candidates = if result_total > 0 && result_total <= *STREAM_MIN_MATCHES { + result_total + } else { + (feats.limit as usize).min(result_total) + }; + (STREAM_SMALL_TOTAL_REDO_BIAS * redo_candidates as f64) as u32 + } else { + 0 + }; + scan_units as u32 + redo }; feats.broadcast_printings = broadcast as u32; feats.scatter_printings = scatter as u32; + feats.collection_broadcast_printings = collection_broadcast as u32; feats.project_printings = project as u32; feats.popcount_words = popcount_words as u32; feats.compose_scan_printings = (printing_matches as f64 * COMPOSE_GATHER_SPAN_PER_MATCH) as u32; @@ -12787,7 +14095,7 @@ fn run_query_streamed<'a>( // Publishing helper: the walk below has several early returns, and every one of them must leave // the stats behind or the accounting silently attributes this plan's work to nothing. Each takes // the closing instant itself, so the emit phase is bounded without a second start marker. - let publish = |end: std::time::Instant, perm_steps: u64| { + let publish = |end: std::time::Instant, perm_steps: u64, redo_examined: u64| { let prep_ns = PENDING_PREPARE_NS.with(|c| c.replace(0)); PHASE_STATS.with(|c| { c.set(PhaseStats { @@ -12797,6 +14105,7 @@ fn run_query_streamed<'a>( matches_pushed: n_matches_pushed, set_printings: 0, // PrintingCompose-only; StreamedSelect never composes pbits perm_steps, + redo_examined, ns_setup: (t_loop - t_start).as_nanos() as u64, ns_loop: (t_finish - t_loop).as_nanos() as u64, ns_finish: (end - t_finish).as_nanos() as u64, @@ -12808,7 +14117,7 @@ fn run_query_streamed<'a>( }); }; if total == 0 || page_offset >= total { - publish(std::time::Instant::now(), 0); + publish(std::time::Instant::now(), 0, 0); return (total, Vec::new()); } @@ -12821,6 +14130,14 @@ fn run_query_streamed<'a>( // Small totals: gather and quickselect — same result as the gathered path. if total <= *STREAM_MIN_MATCHES { let mut best: Vec = Vec::with_capacity(total); + // Round 31 of the printing-varying-leaf depth ledger + // (docs/issues/local-engine-gathered-scan-card-printing-varying-depth.md): this loop IS the + // redo Round 30 priced from a wall-clock residual because `printings_examined` (captured only + // from the FIRST, counting-only pass above) structurally cannot see it. `push_card_matches` + // already returns the printings it examined per call -- the same value the counting pass's + // `card_match_count` call captures into `n_printings_examined` -- so summing it here is free: + // no new computation, just no longer discarding a return value this loop already produces. + let mut n_redo_examined = 0u64; for cid in 0..cards.len() as u32 { if counts[cid as usize] == 0 { continue; @@ -12834,16 +14151,16 @@ fn run_query_streamed<'a>( }; let start = u32::from(offsets[cid as usize]) as usize; let end = u32::from(offsets[cid as usize + 1]) as usize; - push_card_matches( + n_redo_examined += u64::from(push_card_matches( card, cid, printings, artwork_group_col, start, end, all_match, &residual, residual_is_or, mode, prefer, sort_col, descending, strings, existential_plane, &mut best, &mut group_best, &mut touched, - ); + )); } let page = select_page(best, page_offset, limit) .into_iter() .map(|(cid, pid)| (&cards[cid as usize], &printings[pid as usize])) .collect(); - publish(std::time::Instant::now(), 0); + publish(std::time::Instant::now(), 0, n_redo_examined); return (total, page); } @@ -12898,7 +14215,7 @@ fn run_query_streamed<'a>( } skip = 0; } - publish(std::time::Instant::now(), n_perm_steps); + publish(std::time::Instant::now(), n_perm_steps, 0); (total, page) }) // COUNTS.with } @@ -13264,6 +14581,9 @@ fn plan_trial_to_pydict<'py>(py: Python<'py>, t: &PlanTrial) -> PyResult(py: Python<'py>, f: &AcquireFacts) -> PyResult bool { match existential_leaf(p) { Some(ExistentialLeaf::Legality { shift, .. }) => divergent_formats >> shift & 0b11 != 0, @@ -1624,3 +1652,80 @@ pub(crate) fn decode_bitmap_ids(words: impl Iterator, count: usize) } out } + +/// Property tests over `PLANE_BLOCKS`/`existential_leaf`/`needs_printing_verification` -- guards the +/// invariant the Round 16 fix relies on (docs/issues/local-engine-gathered-scan-undercosted-arith- +/// existential-and.md): the router's cost-tier check (`cost_plane_nothing_to_verify`, lib.rs) trusts +/// `plane_expr_is_existential` to answer "is this a printing-level property" uniformly across every +/// family, with NO per-field special case in the caller. That trust is only sound while every entry in +/// this table actually behaves one of the two ways a family can: STATICALLY printing-level (true no +/// matter what `divergent_formats` says -- rarity, border) or DYNAMICALLY printing-level, gated +/// EXACTLY on its own format's bits in `divergent_formats` (legality). A future family wired into the +/// wrong bucket -- e.g. a new always-varying field accidentally routed through the `Legality` arm, or +/// a new format-gated field accidentally landing in the `Some(_) => true` catch-all -- would silently +/// reproduce this round's bug shape (a real per-printing walk priced as free) for a different field, +/// without any caller of `plane_expr_is_existential` needing to change at all. Iterates every index the +/// table actually covers, not a hardcoded list, so a new `PlaneBlock` entry is exercised automatically. +#[cfg(test)] +mod plane_block_family_invariants { + use super::{existential_leaf, needs_printing_verification, ExistentialLeaf, PLANE_COUNT}; + + /// Representative `divergent_formats` masks: none divergent, every format divergent (the + /// conservative default a mask-less caller supplies), and a single arbitrary bit set (the + /// production shape -- one real format, `oldschool`, diverges in the shipped corpus). + const MASKS: [u64; 3] = [0, u64::MAX, 0b11 << 6]; + + #[test] + fn rarity_and_border_are_existential_regardless_of_divergent_formats() { + for p in 0..PLANE_COUNT { + let is_static_existential = matches!( + existential_leaf(p), + Some(ExistentialLeaf::RarityTracked(_) | ExistentialLeaf::RarityHi | ExistentialLeaf::BorderTracked(_) | ExistentialLeaf::BorderOther) + ); + if !is_static_existential { + continue; + } + for &mask in &MASKS { + assert!( + needs_printing_verification(p, mask), + "plane {p} is a rarity/border family member -- printing-varying by construction, so it must \ + need per-printing verification for every divergent_formats mask, not just some (got mask \ + {mask:#x})" + ); + } + } + } + + #[test] + fn legality_is_existential_exactly_when_its_own_format_diverges() { + for p in 0..PLANE_COUNT { + let Some(ExistentialLeaf::Legality { shift, .. }) = existential_leaf(p) else { continue }; + for &mask in &MASKS { + let format_diverges = mask >> shift & 0b11 != 0; + assert_eq!( + needs_printing_verification(p, mask), + format_diverges, + "plane {p} (legality shift {shift}) must need per-printing verification IFF its own \ + format's bits are set in divergent_formats (mask {mask:#x}) -- not unconditionally \ + (that's rarity/border's shape) and not never (that's a card-invariant plane's shape)" + ); + } + } + } + + #[test] + fn planes_outside_every_block_are_never_existential() { + for p in 0..PLANE_COUNT { + if existential_leaf(p).is_some() { + continue; + } + for &mask in &MASKS { + assert!( + !needs_printing_verification(p, mask), + "plane {p} belongs to no existential family (`existential_leaf` returned `None`) -- a \ + card-invariant plane must never need per-printing verification, for any mask (got {mask:#x})" + ); + } + } + } +} diff --git a/card_engine/src/tests.rs b/card_engine/src/tests.rs index 945f6bc20..1349e42b7 100644 --- a/card_engine/src/tests.rs +++ b/card_engine/src/tests.rs @@ -7,7 +7,8 @@ use super::{ build_artist_index, build_printing_value_index, build_arith_tuple_index, is_arith_tuple_route, range_candidates, narrow_candidates, narrow_candidates_exact, rarity_candidates, range_too_broad_to_narrow, run_query, run_query_routed, run_query_with_plan, explain, explain_analyze, AcquireFacts, PlanEstimate, PlanTrial, acquire_plan_features, take_phase_stats, PagingTaken, CountSource, NarrowedRepr, - EXACT_VALUE_TOTALS, RangeCardCounts, narrow_rec, ValueTotals, PairTotals, build_all_value_totals, build_pair_totals, build_range_card_counts, exact_result_total, + EXACT_VALUE_TOTALS, RangeCardCounts, narrow_rec, ValueTotals, PairTotals, SpaceTotals, build_all_value_totals, build_pair_totals, build_range_card_counts, exact_result_total, + pair_range_sum, pair_leaf_id, single_arith_field, PhysicalPlan, PlanScope, CandidatePlan, ComposePaging, trigram_candidates, finalize_trigram_index, PrintingValueIndex, NARROW_FLOOR, gathered_scan_applicable, streamed_select_applicable, plane_popcount_order_applicable, printing_range_scan_applicable, walk_printing_page, aligned_page, bare_range_bounds, probe_range_k, printing_compose_fastpath, printing_range_fastpath, sort_key_bits, orderby_to_col, SortCol, STREAM_MIN_MATCHES, @@ -5140,6 +5141,62 @@ fn generate_calibration_corpus(n: usize, seed: u64) -> Vec<(String, FuzzSpec)> { /// on a quiesced machine (benchmark-artifacts protocol) — otherwise directional. /// The `est` column is the estimator's card-space point estimate (meaningful vs /// `total` in card mode; printing-mode totals count printings). +/// Round 9 of the `GatheredScan`/`card` printing-varying-depth doc +/// (`docs/issues/local-engine-gathered-scan-card-printing-varying-depth.md`): a +/// `Prep::Candidates`-acquired zero-match round collapses every multiplicative term in +/// `PhysicalPlan::GatheredScan`'s cost arm to zero (`eval_domain`, `scan_units`, `page_span`/ +/// `page_rows`, `artwork_seen_printings` all derive from an empty candidate list), so the whole +/// prediction used to read as `GATHER_FIXED_COST_NS` (169.6) alone — a real, isolated ~4x +/// over-charge against the calibrated `GATHER_FIXED_COST_ZERO_MATCH_NS` (42.0; see that constant's +/// doc comment for the held-out numbers). This asserts the gate fires on `matches == 0` alone and +/// leaves every other case, including `matches > 0` with every OTHER term also zeroed (an edge a +/// naive `eval_domain == 0` gate would need to special-case), on `GATHER_FIXED_COST_NS` unchanged. +#[test] +fn gathered_scan_zero_match_uses_the_lower_fixed_cost() { + use super::cost::{plan_cost, PlanFeatures}; + + // limit/offset both 0 so page_span/page_rows (which derive from `matches` too, not just + // eval_domain) stay zero regardless of `matches` — isolates the fixed-cost gate from the + // page-collect terms instead of re-deriving the whole formula here. + let base = PlanFeatures { + n_cards: 30_000, n_printings: 90_000, + matches: 0, eval_domain: 0, scan_units: 0, stream_scan_units: 0, + residual_card_invariant: false, residual_tier_ns100: 0, + artwork_seen_cards: 0, artwork_seen_printings: 0, compose_scan_printings: 0, + limit: 0, offset: 0, perm_walk_span: 30_000, // unbounded (no sort predicate in this fixture) + broadcast_printings: 0, scatter_printings: 0, project_printings: 0, popcount_words: 0, + compose_paging: ComposePaging::Gather, collection_broadcast_printings: 0, + gather_group_printings: 0, + }; + + // matches == 0: every other term is already zero by construction above, so the whole + // prediction IS the fixed-cost term — must read the lower, zero-match constant. + let zero_match = plan_cost(PhysicalPlan::GatheredScan, &base); + assert!( + (zero_match - 42.0).abs() < 1e-9, + "zero-match GatheredScan should cost exactly GATHER_FIXED_COST_ZERO_MATCH_NS, got {zero_match}" + ); + + // matches == 1, eval_domain/scan_units still 0 — the gate reads `matches` itself, not a + // derived "is everything else zero" check, so this must NOT take the zero-match branch even + // though eval_domain/scan_units look identical to the zero-match case above. The one match + // still pays GATHER_PUSH_PER_MATCH_NS (2.24) on top of the higher fixed cost (169.6). + let one_match = plan_cost(PhysicalPlan::GatheredScan, &PlanFeatures { matches: 1, ..base }); + assert!( + (one_match - 171.84).abs() < 1e-9, + "a single-match round must still use GATHER_FIXED_COST_NS (169.6) plus its own push cost, got {one_match}" + ); + + // A non-trivial, real-shaped `candidates` round (nonzero eval_domain/scan_units/matches) also + // reads the higher constant — the gate must not leak into ordinary rows via some interaction + // with the other terms. + let real = plan_cost( + PhysicalPlan::GatheredScan, + &PlanFeatures { matches: 500, eval_domain: 500, scan_units: 500, stream_scan_units: 500, ..base }, + ); + assert!(real > one_match, "a populated round must cost more than the bare fixed cost, got {real}"); +} + #[test] #[ignore = "plan-cost calibration bench; needs real.store; cargo test --release plan_cost_calibration -- --ignored --nocapture"] fn plan_cost_calibration() { @@ -5363,7 +5420,8 @@ fn plan_cost_model_matches_gold() { residual_card_invariant: false, // diagnostic only; nothing in plan_cost reads it limit: limit as u32, offset: offset as u32, - broadcast_printings: 0, scatter_printings: 0, project_printings: 0, popcount_words: 0, compose_paging: ComposePaging::Gather, + perm_walk_span: n_cards, // fixed orderby=edhrec, no query here bounds it + broadcast_printings: 0, scatter_printings: 0, project_printings: 0, popcount_words: 0, compose_paging: ComposePaging::Gather, collection_broadcast_printings: 0, artwork_seen_cards: 0, // no artwork per-card dedupe bitmask in this fixture artwork_seen_printings: 0, // no artwork per-printing dedupe check in this fixture compose_scan_printings: 0, @@ -5606,7 +5664,8 @@ fn plan_cost_refit() { residual_tier_ns100: if prep.all_match_known { 0 } else { verify_cost_tier(&res) }, residual_card_invariant: false, // diagnostic only; nothing in plan_cost reads it limit: limit as u32, offset: offset as u32, - broadcast_printings: 0, scatter_printings: 0, project_printings: 0, popcount_words: 0, compose_paging: ComposePaging::Gather, + perm_walk_span: n_cards, // fixed orderby=edhrec, no query here bounds it + broadcast_printings: 0, scatter_printings: 0, project_printings: 0, popcount_words: 0, compose_paging: ComposePaging::Gather, collection_broadcast_printings: 0, artwork_seen_cards: 0, // no artwork per-card dedupe bitmask in this fixture artwork_seen_printings: 0, // no artwork per-printing dedupe check in this fixture compose_scan_printings: 0, @@ -5812,7 +5871,8 @@ fn printing_range_route_probe() { residual_tier_ns100, residual_card_invariant: false, // diagnostic only; nothing in plan_cost reads it limit: LIMIT as u32, offset: offset as u32, - broadcast_printings: 0, scatter_printings: 0, project_printings: 0, popcount_words: 0, compose_paging: ComposePaging::Gather, + perm_walk_span: n_cards, // fixed orderby=edhrec, no query here bounds it + broadcast_printings: 0, scatter_printings: 0, project_printings: 0, popcount_words: 0, compose_paging: ComposePaging::Gather, collection_broadcast_printings: 0, artwork_seen_cards: 0, // no artwork per-card dedupe bitmask in this fixture artwork_seen_printings: 0, // no artwork per-printing dedupe check in this fixture compose_scan_printings: 0, @@ -6176,7 +6236,8 @@ fn plan_regret_report() { residual_card_invariant: false, // diagnostic only; nothing in plan_cost reads it limit: limit as u32, offset: offset as u32, - broadcast_printings: 0, scatter_printings: 0, project_printings: 0, popcount_words: 0, compose_paging: ComposePaging::Gather, + perm_walk_span: n_cards, // fixed orderby=edhrec, no query here bounds it + broadcast_printings: 0, scatter_printings: 0, project_printings: 0, popcount_words: 0, compose_paging: ComposePaging::Gather, collection_broadcast_printings: 0, artwork_seen_cards: 0, // no artwork per-card dedupe bitmask in this fixture artwork_seen_printings: 0, // no artwork per-printing dedupe check in this fixture compose_scan_printings: 0, @@ -6307,7 +6368,8 @@ fn plan_regret_fuzz() { residual_tier_ns100: tier, residual_card_invariant: false, // diagnostic only; nothing in plan_cost reads it limit: limit as u32, offset: offset as u32, - broadcast_printings: 0, scatter_printings: 0, project_printings: 0, popcount_words: 0, compose_paging: ComposePaging::Gather, + perm_walk_span: n_cards, // fixed orderby=edhrec, no fuzz filter here bounds it + broadcast_printings: 0, scatter_printings: 0, project_printings: 0, popcount_words: 0, compose_paging: ComposePaging::Gather, collection_broadcast_printings: 0, artwork_seen_cards: 0, // no artwork per-card dedupe bitmask in this fixture artwork_seen_printings: 0, // no artwork per-printing dedupe check in this fixture compose_scan_printings: 0, @@ -6637,6 +6699,7 @@ fn bench_checked_vs_unchecked_access() { artists: ArtistIndex::default(), flavor: build_flavor_index(&printings, &strings), set_codes: HashMap::new(), + set_collector_ranges: HashMap::new(), watermarks: HashMap::new(), released_at: PrintingValueIndex::default(), price_usd: PrintingValueIndex::default(), @@ -7460,6 +7523,72 @@ fn streamed_walk_bounds_itself_by_the_sort_column_predicate() { } } +/// Round 32 of the printing-varying-depth ledger +/// (docs/issues/local-engine-gathered-scan-card-printing-varying-depth.md): +/// `cost::PlanFeatures::perm_walk_span`, the field `perm_steps`'s cost formula now multiplies by +/// instead of `n_cards` unconditionally. Reuses the same anti-correlated fixture as +/// `streamed_walk_bounds_itself_by_the_sort_column_predicate` above (8,500 non-matching cards +/// sorting ahead of 1,500 matching ones under `cmc asc`, so a bound that did nothing would be +/// caught by an order of magnitude), but checks the ACQUIRE-time feature directly rather than the +/// dispatch-time walk it now agrees with: before this round, `PlanFeatures` had no such field and +/// the cost model always priced the walk as if it covered the whole corpus regardless of what the +/// filter said about the sort column. +#[test] +fn acquire_perm_walk_span_matches_the_sort_column_bound() { + const PREFIX: usize = 8_500; + const MATCHING: usize = 1_500; + const N: usize = PREFIX + MATCHING; + const MATCH_CMC: u8 = 5; + + let mut vocab = VocabInterner::new(); + let mut cards = Vec::with_capacity(N); + for i in 0..N { + let mut c = stub_card((i + 1) as u128, TYPE_CREATURE, &[], &mut vocab); + // Same anti-correlation as the dispatch-level test: cmc 0 for the prefix, MATCH_CMC for the + // tail, so ascending cmc order puts every match last and descending puts every match first. + c.cmc = Some(if i < PREFIX { 0 } else { MATCH_CMC }); + c.edhrec_rank = Some(((i * 37) % N) as u32 + 1); // distinct, shuffled: no full-key tie blocks + cards.push(c); + } + let data = store_of(cards, &vec![1; N], vocab); + let bytes = rkyv::to_bytes::(&data).expect("serialize"); + let archived = rkyv::access::, Error>(&bytes).expect("access"); + let ctx = QueryCtx::from(archived); + + let filt = || FilterExpr::NumericCmp { + lhs: NumExpr::Field(NumField::Cmc), + op: CmpOp::Ge, + rhs: NumExpr::Const(f64::from(MATCH_CMC)), + }; + // Extracted from the UNSPLIT filter, exactly as `bind_and_split_filter` does. + let cmc_bound = sort_col_bound(&filt(), SortCol::Cmc); + assert_eq!( + cmc_bound, + SortBound { lo: Some(f64::from(MATCH_CMC)), hi: None }, + "a bare `cmc >= k` must bound the cmc column below and leave it unbounded above", + ); + + for descending in [false, true] { + let params = kernel_params(Mode::Card, SortCol::Cmc, descending, 10, 0).with_sort_bound(cmc_bound); + let (pe, mut filter) = + split_planes(filt(), &archived.indexes.planes, &archived.indexes.oracle_trigram.words, true); + let (feats, _prep, _plane_bits) = acquire_plan_features(&ctx, ¶ms, &mut filter, None, pe.as_ref()); + assert_eq!( + feats.perm_walk_span, MATCHING as u32, + "bounded acquire (descending={descending}) must narrow perm_walk_span to the matching \ + segment ({MATCHING}), not the whole corpus ({N})", + ); + } + + // Unbounded control: same filter, ordered by a column it says nothing about — must fall back to + // the whole corpus rather than accidentally reading zero or a stale value. + let params_unbounded = kernel_params(Mode::Card, SortCol::EdhrecRank, false, 10, 0); + let (pe, mut filter) = + split_planes(filt(), &archived.indexes.planes, &archived.indexes.oracle_trigram.words, true); + let (feats, _prep, _plane_bits) = acquire_plan_features(&ctx, ¶ms_unbounded, &mut filter, None, pe.as_ref()); + assert_eq!(feats.perm_walk_span, N as u32, "unbounded acquire must fall back to the whole corpus"); +} + // Group counts collapse duplicate illustrations within a card. #[test] fn artwork_group_counts_dedup_illustrations() { @@ -7754,7 +7883,7 @@ fn card_invariant_broadcast_compose_leaves() { let mut want = brute(f); want.sort_unstable(); assert_eq!(got, want, "compose_printing_bits disagrees with the residual path for {label}"); - let est_matches = super::compose_printing_estimate(f, &archived.indexes, &archived.offsets, n_printings).result; + let est_matches = super::compose_printing_estimate(f, &archived.indexes, &archived.offsets, n_printings).result.printing; assert!(est_matches >= want.len(), "compose_printing_estimate undercounts for {label}: {est_matches} < {}", want.len()); } @@ -7774,6 +7903,69 @@ fn card_invariant_broadcast_compose_leaves() { assert!(!super::is_broadcast_leaf_shape(&border), "border must not be treated as a card-invariant broadcast leaf"); } +// Regression for the `domain_hint` unit bug found investigating `GatheredScan[printing_compose]`'s +// worst-in-survey spread: `domain_hint` is a CARD count (its one consumer, `acquire_plan_features`'s +// `domain_cards`, feeds it straight into `eval_domain`), but its "2+ card-invariant planes intersect" +// arm used to scale the exact card popcount by `n_printings / n_cards` before storing it -- the same +// conversion `result`'s own (correctly PRINTING-space) tightening needs, reused unconverted for a +// card-space consumer. Confirmed live: `devotion:w c:u usd>5` predicted `eval_domain: 1705` against a +// realized `cards_visited` of 108 (15.8x over); fixed, `eval_domain: 553` (the true, unscaled 2-leaf +// intersection -- still not exact, since `usd>5` isn't folded into this tightening, but 3.08x tighter, +// matching `n_printings / n_cards` on that corpus exactly). +#[test] +fn domain_hint_is_card_space_not_printing_scaled() { + let mut vocab = VocabInterner::new(); + // card0: green colors + identity (matches `c:g id:g`). card1: red colors + identity (matches + // neither). card2: green+red colors + identity (matches `c:g id:g`) -- exact card intersection + // of the two plane-compilable leaves is {0, 2}, i.e. 2 cards. + let mut cards = vec![ + stub_card(1, TYPE_CREATURE, &[], &mut vocab), + stub_card(2, TYPE_CREATURE, &[], &mut vocab), + stub_card(3, TYPE_CREATURE, &[], &mut vocab), + ]; + cards[0].card_colors = super::color_to_bit("G"); + cards[0].card_color_identity = super::color_to_bit("G"); + cards[1].card_colors = super::color_to_bit("R"); + cards[1].card_color_identity = super::color_to_bit("R"); + cards[2].card_colors = super::color_to_bit("G") | super::color_to_bit("R"); + cards[2].card_color_identity = super::color_to_bit("G") | super::color_to_bit("R"); + let mut data = store_of(cards, &[2, 2, 2], vocab); // 3 cards, 2 printings each -- n_printings=6, n_cards=3 + let set_codes_by_pid = ["dmu", "lea", "dmu", "dmu", "lea", "neo"]; + for (p, code) in data.printings.iter_mut().zip(set_codes_by_pid) { + p.card_set_code = InlineStr::from_str(code); + } + let mut set_codes: TagIndex = HashMap::new(); + for (i, p) in data.printings.iter().enumerate() { + set_codes.entry(p.card_set_code.as_str().to_string()).or_default().push(i as u32); + } + data.indexes.set_codes = set_codes; + data.indexes.cmc = build_numeric_index(&data.cards, |c| c.cmc.map(|v| v as i16)); + let p2c = build_printing_to_card(&data.offsets); + data.indexes.value_totals = + build_all_value_totals(&data.cards, &data.printings, &p2c, &data.strings, &data.coll_vocab, usize::from(data.indexes.max_artwork_groups)); + data.indexes.arith_tuple = build_arith_tuple_index(&data.cards); + let bytes = rkyv::to_bytes::(&data).expect("serialize"); + let archived = rkyv::access::, Error>(&bytes).expect("access"); + let n_printings = archived.printings.len(); + + let green = FilterExpr::ColorCmp { field: ColorField::Colors, op: CmpOp::Ge, mask: super::color_to_bit("G") }; + let green_identity = FilterExpr::ColorCmp { field: ColorField::ColorIdentity, op: CmpOp::Ge, mask: super::color_to_bit("G") }; + let set_dmu = FilterExpr::TextExact { field: super::TextField::SetCode, op: CmpOp::Eq, value: "dmu".to_string() }; + // `set:dmu` isn't plane-compilable (no `compile_plane` arm), so it keeps this a genuine 3-child + // `And` reaching `compose_printing_estimate`'s own arm rather than getting split off into a bare + // plane residual -- exactly the shape `domain_hint`'s 2+-card-invariant-planes branch targets. + let filter = FilterExpr::And(vec![green, green_identity, set_dmu]); + let est = super::compose_printing_estimate(&filter, &archived.indexes, &archived.offsets, n_printings); + assert_eq!(est.result.card, Some(2), "est.result.card must be the exact 2-card intersection, not scaled by n_printings/n_cards"); + // The `c:g id:g` intersection's exact ARTWORK span: cards 0 and 2 each have 2 printings with + // `store_of`'s default all-distinct artwork groups, so 2 + 2 = 4 -- `n_printings/n_cards * 2` + // (the same average-ratio bug `card` guards against) would give 4 here too by coincidence (this + // fixture's ratio is exactly 2), so this alone wouldn't distinguish exact-sum from average-scaled; + // it exists to confirm the plumbing (`card_bits_span_total` over `indexes.artwork_base`) runs at + // all and produces a real, present value rather than silently staying `None`. + assert_eq!(est.result.artwork, Some(4), "est.result.artwork must be the exact artwork span of the same 2-card intersection"); +} + // #746: `set:`/`watermark:` postings leaves join the PrintingCompose leaf table. This is the // differential test the design doc calls for: for every filter shape (both `set:` polarities, // `watermark:` positive, and mixes with a year range) the exact `compose_printing_bits` bitmap must @@ -7887,7 +8079,7 @@ fn set_watermark_compose_leaves() { // The estimate feeds plan choice and must be a valid upper bound on the true match count // (AND takes the min-of-children intersection bound, OR the capped sum — never an // undercount, which would misprice the plan). For a bare leaf it's exact (postings length). - let est_matches = super::compose_printing_estimate(f, &archived.indexes, &archived.offsets, n_printings).result; + let est_matches = super::compose_printing_estimate(f, &archived.indexes, &archived.offsets, n_printings).result.printing; assert!(est_matches >= want.len(), "compose_printing_estimate undercounts for {label}: {est_matches} < {}", want.len()); if matches!(f, FilterExpr::TextExact { .. } | FilterExpr::Not(_)) { assert_eq!(est_matches, want.len(), "bare-leaf estimate must be exact for {label}"); @@ -7903,6 +8095,82 @@ fn set_watermark_compose_leaves() { assert!(!super::is_printing_composable(&unsupported, &archived.indexes), "-watermark inside an And keeps the whole And off the compose path"); } +/// `compose_printing_estimate`'s `And` arm has no tightening at all for `set:X` And'd with a +/// `collector_number_int` range today: `set` has no `compile_plane` arm and isn't in `ValueTotals`, +/// and `collector_number_int` isn't arith-tuple-eligible and has no `compile_plane` arm either, so +/// the plain min-fold picks whichever leaf's own (corpus-wide, not set-scoped) count happens to be +/// smaller. This is the Round 33 fix (`docs/issues/local-engine-gathered-scan-card-printing-varying- +/// depth.md`): a per-set `[min, max]` collector-number span, precomputed once at load time +/// (`build_set_collector_ranges`/`indexes.set_collector_ranges`), lets the estimate scale a density +/// (`count / (max - min + 1)`) by the query's own overlap with that span instead. +/// +/// Three sets share one corpus so the fold's OWN wrong number is reproduced deliberately, not just +/// the fix's right one: `con` (50 printings, `cn` 1..=50, contiguous) and `big` (100 printings, `cn` +/// 1..=100, contiguous) both contribute to `collector_number`'s CORPUS-WIDE range count, and `gap` +/// (20 printings, `cn` clustered 1..=5 then scattered 200..=999, i.e. non-contiguous) is the SLD- +/// shaped case Round 33's own doc flags as a real, un-closed residual. +#[test] +fn set_and_collector_number_range_density_tightening() { + let mut vocab = VocabInterner::new(); + let cards = vec![ + stub_card(1, TYPE_CREATURE, &[], &mut vocab), + stub_card(2, TYPE_CREATURE, &[], &mut vocab), + stub_card(3, TYPE_CREATURE, &[], &mut vocab), + ]; + // pids 0..50 = con, 50..150 = big, 150..170 = gap. + let mut data = store_of(cards, &[50, 100, 20], vocab); + let gap_cns: [u32; 20] = [1, 2, 3, 4, 5, 200, 250, 300, 350, 400, 450, 500, 550, 600, 650, 700, 750, 800, 900, 999]; + for (i, p) in data.printings.iter_mut().enumerate() { + let (code, cn) = if i < 50 { + ("con", (i + 1) as u32) + } else if i < 150 { + ("big", (i - 50 + 1) as u32) + } else { + ("gap", gap_cns[i - 150]) + }; + p.card_set_code = InlineStr::from_str(code); + p.collector_number_int = Some(cn as u16); + } + data.indexes.set_codes = { + let mut idx: TagIndex = HashMap::new(); + for (i, p) in data.printings.iter().enumerate() { + idx.entry(p.card_set_code.as_str().to_string()).or_default().push(i as u32); + } + idx + }; + data.indexes.set_collector_ranges = super::build_set_collector_ranges(&data.printings, |p| p.card_set_code.as_str(), |p| p.collector_number_int.map(u32::from)); + data.indexes.collector_number = build_printing_value_index(&data.printings, &data.cards, &data.offsets, |p| p.collector_number_int.map(u32::from)); + let bytes = rkyv::to_bytes::(&data).expect("serialize"); + let archived = rkyv::access::, Error>(&bytes).expect("access"); + let n_printings = archived.printings.len(); + + let set = |code: &str| FilterExpr::TextExact { field: super::TextField::SetCode, op: CmpOp::Eq, value: code.to_string() }; + let cn = |op: CmpOp, v: f64| FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::CollectorNumberInt), op, rhs: NumExpr::Const(v) }; + let est = |f: &FilterExpr| super::compose_printing_estimate(f, &archived.indexes, &archived.offsets, n_printings).result.printing; + + // Fused two-sided range (`cn>=10 cn<=19`, 3 literal children -> 2 `AndSource`s after fusion). + // `con` is contiguous (density 1.0), so the estimate is EXACT: 10 of con's own printings have + // cn in [10, 19]. Pre-fix, the min-fold picked `min(con's own 50, the corpus-wide 20 printings + // -- 10 from con, 10 from big -- with cn in [10,19])` = 20, 2x over the true 10. + let fused = FilterExpr::And(vec![set("con"), cn(CmpOp::Ge, 10.0), cn(CmpOp::Le, 19.0)]); + assert_eq!(est(&fused), 10, "set:con cn>=10 cn<=19 (fused): density estimate must be exact for a contiguous set"); + + // Bare one-sided range (`cn<=15`, no fusion needed -- a single `AndSource::Child`). Same + // contiguous set, same exactness. Pre-fix: min(con's own 50, corpus-wide cn<=15 -- 15 from con, + // 15 from big -- = 30) = 30, 2x over the true 15. + let bare = FilterExpr::And(vec![set("con"), cn(CmpOp::Le, 15.0)]); + assert_eq!(est(&bare), 15, "set:con cn<=15 (bare): density estimate must be exact for a contiguous set"); + + // Non-contiguous set (`gap`): true count for `cn<=100` is 5 (gap's own low cluster). Pre-fix, + // the min-fold picks `min(gap's own 20, corpus-wide cn<=100 -- 5 from gap, 100 from big -- = + // 105)` = 20, a 4x OVER-estimate. The density model (count=20, span=999, density~=0.02003) + // scales to `round(0.02003 * 100) = 2`, a 2.5x UNDER-estimate -- worse in the opposite + // direction, but a smaller |log-ratio| than the fold's 20 (|ln(2/5)|=0.916 vs |ln(20/5)|=1.386) + // -- the documented, accepted residual for a non-contiguous set (SLD-shaped), not a bug. + let gap_query = FilterExpr::And(vec![set("gap"), cn(CmpOp::Le, 100.0)]); + assert_eq!(est(&gap_query), 2, "set:gap cn<=100: density estimate for a non-contiguous set must match the documented (partial) improvement"); +} + /// Card-space collection containment fields (`type:`/`kw:`/`otag:`) and their printing-space siblings /// (`art:`/`is:`) as PrintingCompose leaves: `compose_printing_bits` must be bit-for-bit the residual /// path's truth for the positive `Ge` leaf, its negation, mixes with a range, and absent values — @@ -8027,7 +8295,7 @@ fn collection_compose_leaves() { want.sort_unstable(); assert_eq!(got, want, "compose_printing_bits disagrees with the residual path for {label}"); // The estimate feeds plan choice: a valid upper bound at minimum, and exact for a bare leaf. - let est_matches = super::compose_printing_estimate(f, &archived.indexes, &archived.offsets, n_printings).result; + let est_matches = super::compose_printing_estimate(f, &archived.indexes, &archived.offsets, n_printings).result.printing; assert!(est_matches >= want.len(), "compose_printing_estimate undercounts for {label}: {est_matches} < {}", want.len()); if matches!(f, FilterExpr::CollectionCmp { .. } | FilterExpr::Not(_)) { assert_eq!(est_matches, want.len(), "bare collection-leaf estimate must be exact for {label}"); @@ -9697,6 +9965,350 @@ fn border_other_bucket_closes_domain_for_tracked_negation() { assert!(!bitmap_contains(&bits2, yellow_card), "the same card must not satisfy border:black"); } +/// Regression fixture for the `residual_tier_ns100` under-charge found this session: an `And` of a +/// card-invariant arith range (`cmc`) with a printing-varying existential leaf (`border`), both +/// compiled into ONE plane under `unique=card` -- see +/// docs/issues/local-engine-gathered-scan-undercosted-arith-existential-and.md. Card 4 carries two +/// printings with DIFFERENT borders specifically so a card-level "some printing is black" fact is not +/// the same as "the first-checked printing is black" -- the shape `existential_plane_for`'s per-printing +/// walk exists for. +fn cmc_border_existential_fixture_store() -> CardData { + let mut vocab = VocabInterner::new(); + let mut interner = Interner::new(); + // (cmc, borders per printing) + let specs: &[(u8, &[&str])] = &[ + (0, &["white"]), // 0: outside the cmc range + (3, &["white"]), // 1: in range, wrong border + (3, &["black"]), // 2: in range AND black -- must match + (6, &["black"]), // 3: black but outside the cmc range + (2, &["white", "black"]), // 4: in range; only its SECOND printing is black + ]; + let cards: Vec = specs + .iter() + .enumerate() + .map(|(i, &(cmc, _))| { + let mut c = stub_card(i as u128 + 1, TYPE_CREATURE, &[], &mut vocab); + c.cmc = Some(cmc); + c + }) + .collect(); + let printing_counts: Vec = specs.iter().map(|(_, borders)| borders.len()).collect(); + let mut data = store_of(cards, &printing_counts, vocab); + let mut idx = 0; + for (_, borders) in specs { + for border in borders.iter() { + data.printings[idx].card_border_id = interner.intern((*border).to_string()); + idx += 1; + } + } + data.strings = interner.strings; + data.indexes.planes = build_bit_planes(&data.cards, &data.printings, &data.offsets, &data.strings); + // `PrintingCompose`'s applicability (`printing_compose_indexes_built`) declines cleanly unless all + // three of these are actually built -- an ordinary fixture store leaves them at their empty + // `Default`, which reports 0 and would otherwise silently reroute this fixture to the general + // `Candidates` path instead of the `PrintingCompose`-acquire branch this test exists to exercise. + data.indexes.border_printing = build_border_printing_planes(&data.printings, &data.strings); + data.indexes.rarity_printing = build_rarity_printing_planes(&data.printings); + data.indexes.arith_tuple = build_arith_tuple_index(&data.cards); + // Round 22 addition: `card_numeric_index` always returns `Some(&indexes.cmc)` regardless of + // whether it was actually built, so a bare `cmc` leaf's OWN single-child `compose_printing_ + // estimate` arm (`bare_numeric_field_count`) silently reads a wrong `Some(0)` from the empty + // default index instead of falling back to `arith_tuple_count` -- harmless for this fixture's + // ORIGINAL test (`compose_tier_charges_border_existential_and_arith_range`, which drives + // `acquire_plan_features`/`split_planes` and never reaches that per-child arm), but load-bearing + // for `compose_and_arm_tightens_lone_existential_leaf_with_no_card_invariant_partner` below, which + // calls `compose_printing_estimate` directly on the raw `And` and DOES fold in each child's own + // (correctly answered) estimate. + data.indexes.cmc = build_numeric_index(&data.cards, |c| c.cmc.map(|v| v as i16)); + data +} + +/// The bug: `cmc>=1 cmc<=5 border:black`/`unique=card` compiles BOTH children into one plane +/// (`split_planes`'s whole-filter shortcut), leaving `filter == True`. The `PrintingCompose`-acquire +/// branch of `acquire_plan_features` priced `residual_tier_ns100 == 0` ("nothing to verify") for EVERY +/// plane under `Mode::Card`, not just legality's — but `border` is printing-varying +/// (`needs_printing_verification` in planes.rs: "for rarity and border that is every leaf"), so the +/// executor's `existential_plane_for` still walks printings one by one +/// (`eval_plane_expr_for_printing`), real work the tier charge must not be zero for. Found live: +/// `GatheredScan` predicted 326,263ns against a measured 1.0-1.3ms on the real corpus +/// (docs/issues/local-engine-gathered-scan-undercosted-arith-existential-and.md). +#[test] +fn compose_tier_charges_border_existential_and_arith_range() { + let data = cmc_border_existential_fixture_store(); + let bytes = rkyv::to_bytes::(&data).expect("serialize"); + let archived = rkyv::access::, Error>(&bytes).expect("access"); + let ctx = QueryCtx::from(archived); + let bounds = &archived.indexes.planes; + let words = &archived.indexes.oracle_trigram.words; + // `SortCol::Rarity`/descending, matching the real query that found this (`orderby=rarity, + // direction=desc`) -- no permutation for that pair, so `PlanePopcountOrder` declines and this + // reaches the `PrintingCompose`-acquire branch the bug lives in, exactly like the real corpus. + let params = kernel_params(Mode::Card, SortCol::Rarity, true, 100, 0); + + let cmc_ge = FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::Cmc), op: CmpOp::Ge, rhs: NumExpr::Const(1.0) }; + let cmc_le = FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::Cmc), op: CmpOp::Le, rhs: NumExpr::Const(5.0) }; + let border_black = FilterExpr::TextExact { field: TextField::Border, op: CmpOp::Eq, value: "black".to_string() }; + + // AND(cmc range, border:black): both children compile into one plane, filter -> True. + let unsplit = FilterExpr::And(vec![cmc_ge.clone(), cmc_le.clone(), border_black]); + let (pe, residual) = split_planes(unsplit.clone(), bounds, words, true); + assert!(pe.is_some(), "cmc range + border:black must compile into a plane"); + assert!(matches!(residual, FilterExpr::True), "both children must be fully consumed, leaving no residual"); + let mut acq_filter = residual; + let (feats, prep, _bits) = acquire_plan_features(&ctx, ¶ms, &mut acq_filter, Some(&unsplit), pe.as_ref()); + assert_eq!(prep.count_source(), CountSource::PrintingCompose, "this fixture must reach the compose-acquire branch to exercise the fix"); + assert!( + feats.residual_tier_ns100 > 0, + "border is printing-varying (existential) even under Mode::Card -- the tier must not be zero \ + just because the OTHER conjunct (cmc) is card-invariant" + ); + + // Control: the bare cmc range alone genuinely has nothing to verify under Mode::Card and must stay + // at tier 0 -- this population is NOT the bug (see the tracking doc's Phase A Q1: a bare range + // alone routes correctly). + let bare_unsplit = FilterExpr::And(vec![cmc_ge, cmc_le]); + let (pe2, residual2) = split_planes(bare_unsplit.clone(), bounds, words, true); + assert!(matches!(residual2, FilterExpr::True)); + let mut acq_filter2 = residual2; + let (feats2, prep2, _bits2) = acquire_plan_features(&ctx, ¶ms, &mut acq_filter2, Some(&bare_unsplit), pe2.as_ref()); + assert_eq!(prep2.count_source(), CountSource::PrintingCompose); + assert_eq!(feats2.residual_tier_ns100, 0, "a bare card-invariant arith range has nothing to verify -- must stay free"); +} + +/// Round 22 regression, for the SPECIFIC `best_other` gate bug Round 21 root-caused (a different bug +/// from Round 15/16's tier-charge fix above, though it lives in the same `And` arm and shares this +/// fixture): `compose_printing_estimate`'s `best_other` computation used to require a card-invariant +/// PARTNER (`else if !card_invariant.is_empty()`) before running an existential leaf's own exact +/// popcount -- so `cmc>=1 cmc<=5 AND border:black` (an arith-tuple range ANDed with exactly ONE +/// existential leaf and nothing ELSE card-invariant) left `best_other`/`exact_domain_cards`/the +/// printing-space `result` tightening at their untightened defaults for the whole function, even +/// though `popcount_with_bits(Some(border_black))` (an empty `card_invariant` vec plus the one pushed +/// leaf) answers the true joint exactly. Root-caused with real `eval_domain`/`cards_visited` numbers in +/// docs/issues/local-engine-domain-cards-existential-arith-and.md. +/// +/// True card-space intersection of `cmc>=1 cmc<=5 AND border:black` on `cmc_border_existential_ +/// fixture_store`: card0 (cmc=0) is outside the range; card1 (cmc=3, white only) has no black +/// printing; card3 (cmc=6, black) is outside the range -- all three excluded. card2 (cmc=3, black) and +/// card4 (cmc=2, white+black) are both in range AND have a black printing -- exactly {card2, card4}, +/// printing span 1 (card2's one printing) + 2 (card4's two printings) = 3. +#[test] +fn compose_and_arm_tightens_lone_existential_leaf_with_no_card_invariant_partner() { + let data = cmc_border_existential_fixture_store(); + let bytes = rkyv::to_bytes::(&data).expect("serialize"); + let archived = rkyv::access::, Error>(&bytes).expect("access"); + let n_printings = archived.printings.len(); + + let cmc_ge = FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::Cmc), op: CmpOp::Ge, rhs: NumExpr::Const(1.0) }; + let cmc_le = FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::Cmc), op: CmpOp::Le, rhs: NumExpr::Const(5.0) }; + let border_black = FilterExpr::TextExact { field: TextField::Border, op: CmpOp::Eq, value: "black".to_string() }; + // No other card-invariant leaf at all -- `card_invariant` is empty, `existential` has exactly one + // element (`border_black`), which is precisely the shape the old `!card_invariant.is_empty()` + // guard refused to tighten. + let filter = FilterExpr::And(vec![cmc_ge, cmc_le, border_black]); + + let est = super::compose_printing_estimate(&filter, &archived.indexes, &archived.offsets, n_printings); + assert_eq!( + est.result.card, + Some(2), + "the AND's exact card-space intersection is {{card2, card4}} (2 cards) -- a lone existential \ + leaf with no OTHER card-invariant leaf must still get an exact joint via best_other, not fall \ + back to an untightened min/None" + ); + assert_eq!( + est.result.printing, 3, + "printing-space result must tighten to the same 2 cards' exact printing span (1 + 2), not a \ + looser per-child min" + ); + // This fixture has only 6 printings total -- nowhere near `PAIR_MIN_PRINTINGS` (1,024), so Round + // 24's `pair_range_sum` preferred path declines for every value here and the assertions above are + // exercising Round 22's ORIGINAL fallback, unchanged. See the two tests below for Round 24's own + // logic, exercised directly against a hand-built `PairTotals` at a scale a real fixture would need + // 1,024+ printings per value to reach. +} + +/// Round 24: `pair_range_sum`'s own summation and pruning-safety logic, against a hand-built +/// `PairTotals` (bypassing `build_pair_totals`/`PAIR_MIN_PRINTINGS` entirely -- a real fixture would +/// need 1,024+ printings per value to clear the floor, which a fast unit test cannot afford). Two +/// `cmc` values (1, 2) hold ids that cleared the floor; a third (3) was OBSERVED (`cmc_seen`) but +/// pruned from the id map, exactly the shape `pair_range_sum`'s own doc says must decline rather than +/// silently under-count. +#[test] +fn pair_range_sum_sums_disjoint_values_and_declines_on_a_pruned_one() { + let mut pt = PairTotals { n_ids: 3, ..Default::default() }; + pt.cmc.insert(1, 0); + pt.cmc.insert(2, 1); + pt.rarity.insert(5, 2); // one existential id (a rarity value) to pair against + pt.cmc_seen = vec![1, 2, 3]; + pt.pairs.insert(2, SpaceTotals { printings: 10, cards: 4, artworks: 6 }); // key(cmc_id=0, rarity_id=2) = 0*3+2 + pt.pairs.insert(5, SpaceTotals { printings: 20, cards: 8, artworks: 12 }); // key(cmc_id=1, rarity_id=2) = 1*3+2 + + let bytes = rkyv::to_bytes::(&pt).expect("serialize"); + let archived = rkyv::access::, Error>(&bytes).expect("access"); + + let cmc_ge1 = FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::Cmc), op: CmpOp::Ge, rhs: NumExpr::Const(1.0) }; + let cmc_le2 = FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::Cmc), op: CmpOp::Le, rhs: NumExpr::Const(2.0) }; + let bounds_1_2 = [&cmc_ge1, &cmc_le2]; + assert_eq!( + pair_range_sum(&bounds_1_2, NumField::Cmc, 2, archived), + Some((30, 12, 18)), + "both cmc=1 and cmc=2 cleared the floor -- the exact disjoint sum is (10+20, 4+8, 6+12)" + ); + + let cmc_le3 = FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::Cmc), op: CmpOp::Le, rhs: NumExpr::Const(3.0) }; + let bounds_1_3 = [&cmc_ge1, &cmc_le3]; + assert_eq!( + pair_range_sum(&bounds_1_3, NumField::Cmc, 2, archived), + None, + "cmc=3 was observed (cmc_seen) but pruned from the id map -- must decline, not silently treat it as zero" + ); +} + +/// Round 24: `pair_leaf_id`'s new `Cmc`/`Power`/`Toughness` `Eq` arms, against the same kind of +/// hand-built `PairTotals` -- resolves a bare-value leaf's id in either operand order, and declines +/// (falls back to the existing single-value `min` bound) for any op other than `Eq`, mirroring +/// rarity's own pre-existing restriction. +#[test] +fn pair_leaf_id_resolves_cmc_power_toughness_eq_and_declines_ranges() { + let mut pt = PairTotals { n_ids: 3, ..Default::default() }; + pt.cmc.insert(4, 7); + pt.power.insert(-1, 9); // negative power (e.g. Char-Rumbler) is a real value in this domain + pt.toughness.insert(2, 11); + + let bytes = rkyv::to_bytes::(&pt).expect("serialize"); + let archived = rkyv::access::, Error>(&bytes).expect("access"); + + let cmc_eq4 = FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::Cmc), op: CmpOp::Eq, rhs: NumExpr::Const(4.0) }; + assert_eq!(pair_leaf_id(&cmc_eq4, archived), Some(7)); + + let power_eq_neg1_flipped = FilterExpr::NumericCmp { lhs: NumExpr::Const(-1.0), op: CmpOp::Eq, rhs: NumExpr::Field(NumField::Power) }; + assert_eq!(pair_leaf_id(&power_eq_neg1_flipped, archived), Some(9), "the flipped Const-lhs/Field-rhs form must resolve identically"); + + let toughness_eq2 = FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::Toughness), op: CmpOp::Eq, rhs: NumExpr::Const(2.0) }; + assert_eq!(pair_leaf_id(&toughness_eq2, archived), Some(11)); + + let cmc_ge4 = FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::Cmc), op: CmpOp::Ge, rhs: NumExpr::Const(4.0) }; + assert_eq!(pair_leaf_id(&cmc_ge4, archived), None, "a RANGE (not Eq) must decline -- pair_range_sum is the range-capable counterpart"); +} + +/// Round 24: `single_arith_field` agrees only when every child constrains the SAME field -- the guard +/// that keeps `pair_range_sum` from being asked to sum over two different value axes at once (e.g. a +/// mixed `cmc>=1 power<=2`, which has no single per-value partition to answer from). +#[test] +fn single_arith_field_agrees_only_when_every_child_is_the_same_field() { + let cmc_ge = FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::Cmc), op: CmpOp::Ge, rhs: NumExpr::Const(1.0) }; + let cmc_le = FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::Cmc), op: CmpOp::Le, rhs: NumExpr::Const(5.0) }; + let power_ge = FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::Power), op: CmpOp::Ge, rhs: NumExpr::Const(1.0) }; + + assert!(matches!(single_arith_field(&[&cmc_ge, &cmc_le]), Some(NumField::Cmc)), "both children agree on cmc"); + assert!(single_arith_field(&[&cmc_ge, &power_ge]).is_none(), "a mixed cmc+power And has no single value axis to sum over"); + assert!(single_arith_field(&[]).is_none(), "no arith children means no field to report"); +} + +/// Round 16's companion to `cmc_border_existential_fixture_store`: the same shape (a card-invariant +/// `cmc` range ANDed with an existential leaf, both folding into one plane under `unique=card`), but +/// the existential leaf is LEGALITY, not border -- and carries TWO formats so the fixture can assert +/// both directions at once. Format A (shift 0) genuinely diverges: card 4 carries one printing legal in +/// A and one not, so `divergent_formats_of` marks A's bits set. Format B (shift 2) is legal on every +/// printing of every card -- never diverges, the ordinary case every OTHER format is in production +/// (docs/issues/local-engine-gathered-scan-undercosted-arith-existential-and.md: 31 of the corpus's 32 +/// formats are this shape, only `oldschool` is format A's shape). +fn cmc_legality_existential_fixture_store() -> CardData { + let mut vocab = VocabInterner::new(); + const LEGAL: u64 = 0b01; // LEGALITY_LEGAL + const NOT_LEGAL: u64 = 0b00; + // (cmc, per-printing format-A legality) -- format B is always LEGAL, set uniformly below. + let specs: &[(u8, &[u64])] = &[ + (0, &[LEGAL]), // 0: outside the cmc range + (3, &[NOT_LEGAL]), // 1: in range, wrong legality (A) + (3, &[LEGAL]), // 2: in range AND legal in A -- must match + (6, &[LEGAL]), // 3: legal in A but outside the cmc range + (2, &[NOT_LEGAL, LEGAL]), // 4: in range; only its SECOND printing is legal in A -- divergent + ]; + let cards: Vec = specs + .iter() + .enumerate() + .map(|(i, &(cmc, _))| { + let mut c = stub_card(i as u128 + 1, TYPE_CREATURE, &[], &mut vocab); + c.cmc = Some(cmc); + c + }) + .collect(); + let printing_counts: Vec = specs.iter().map(|(_, fa)| fa.len()).collect(); + let mut data = store_of(cards, &printing_counts, vocab); + let mut idx = 0; + for (_, fa_per_printing) in specs { + for &fa in fa_per_printing.iter() { + data.printings[idx].card_legalities = fa | (LEGAL << 2); + idx += 1; + } + } + data.indexes.planes = build_bit_planes(&data.cards, &data.printings, &data.offsets, &data.strings); + // Same three indexes `cmc_border_existential_fixture_store` builds, for the same reason: without + // them `printing_compose_indexes_built` declines and this fixture silently reroutes off the + // `PrintingCompose`-acquire branch the test exists to exercise. + data.indexes.border_printing = build_border_printing_planes(&data.printings, &data.strings); + data.indexes.rarity_printing = build_rarity_printing_planes(&data.printings); + data.indexes.arith_tuple = build_arith_tuple_index(&data.cards); + data +} + +/// Round 16's finding: Round 15's fix scoped the `Mode::Card` bypass off `border`/`rarity` via a +/// plane-INDEX-RANGE check (`plane_touches_rarity_or_border`), not off the general +/// `plane_expr_is_existential` this now uses instead -- so a plane that is existential ONLY via a +/// DIVERGENT legality leaf (no rarity, no border) still hit the unscoped first disjunct +/// (`Mode::Card && !touches_rarity_or_border`, true whenever the plane has no rarity/border leaf at +/// all) and read `residual_tier_ns100 == 0`, reproducing the exact same under-costing bug for a +/// different field. Found live on the real corpus: `f:oldschool` alone, `unique=card` -- `GatheredScan` +/// predicted 10,358ns against a measured 29,083ns (2.8x under-charge); `f:oldschool cmc>=1 cmc<=5` -- +/// predicted 18,519ns against a measured 58,625ns (3.16x under-charge). Both closed to 31,394ns/51,276ns +/// respectively after the fix (ratios 1.21/0.90). +#[test] +fn compose_tier_charges_divergent_legality_existential_and_arith_range() { + let data = cmc_legality_existential_fixture_store(); + let bytes = rkyv::to_bytes::(&data).expect("serialize"); + let archived = rkyv::access::, Error>(&bytes).expect("access"); + let ctx = QueryCtx::from(archived); + let bounds = &archived.indexes.planes; + let words = &archived.indexes.oracle_trigram.words; + let params = kernel_params(Mode::Card, SortCol::Rarity, true, 100, 0); + + let cmc_ge = FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::Cmc), op: CmpOp::Ge, rhs: NumExpr::Const(1.0) }; + let cmc_le = FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::Cmc), op: CmpOp::Le, rhs: NumExpr::Const(5.0) }; + let legal_in_a = FilterExpr::Legality { shift: Some(0), expected: 0b01 }; + let legal_in_b = FilterExpr::Legality { shift: Some(2), expected: 0b01 }; + + // AND(cmc range, legal-in-DIVERGENT-format-A): must charge a nonzero tier, same bug shape as + // Round 15's border finding, just via legality instead. + let divergent = FilterExpr::And(vec![cmc_ge.clone(), cmc_le.clone(), legal_in_a]); + let (pe, residual) = split_planes(divergent.clone(), bounds, words, true); + assert!(pe.is_some(), "cmc range + legal-in-A must compile into a plane"); + assert!(matches!(residual, FilterExpr::True), "both children must be fully consumed, leaving no residual"); + let mut acq_filter = residual; + let (feats, prep, _bits) = acquire_plan_features(&ctx, ¶ms, &mut acq_filter, Some(&divergent), pe.as_ref()); + assert_eq!(prep.count_source(), CountSource::PrintingCompose, "this fixture must reach the compose-acquire branch to exercise the fix"); + assert!( + feats.residual_tier_ns100 > 0, + "format A genuinely diverges in this fixture (card 4's two printings disagree) -- the tier must \ + not be zero just because the plane never touches rarity/border, nor just because mode is Card" + ); + + // Control: AND(cmc range, legal-in-NON-divergent-format-B) has genuinely nothing to verify -- + // format B never diverges, so the executor's per-printing walk never actually runs for it, and the + // legitimate #667 Mode::Card bypass must still apply. This is NOT the bug: it must stay free. + let non_divergent = FilterExpr::And(vec![cmc_ge, cmc_le, legal_in_b]); + let (pe2, residual2) = split_planes(non_divergent.clone(), bounds, words, true); + assert!(pe2.is_some(), "cmc range + legal-in-B must compile into a plane"); + assert!(matches!(residual2, FilterExpr::True)); + let mut acq_filter2 = residual2; + let (feats2, prep2, _bits2) = acquire_plan_features(&ctx, ¶ms, &mut acq_filter2, Some(&non_divergent), pe2.as_ref()); + assert_eq!(prep2.count_source(), CountSource::PrintingCompose); + assert_eq!( + feats2.residual_tier_ns100, 0, + "format B never diverges in this fixture -- a card-invariant legality format ANDed with a \ + card-invariant arith range must stay free, exactly like the bare-cmc-range control" + ); +} + /// 7 of 8 cards have a black printing (87.5%, past narrow_candidates_exact's /// keep-if-<=75%-of-domain broadness guard, `domain - domain/4` with integer /// division); the 8th has a borderless printing (12.5%). @@ -13327,3 +13939,198 @@ fn compose_perm_three_phase_order_only_fires_when_enabled_and_sparse() { ); } } + +/// Round 30 of the printing-varying-leaf depth ledger +/// (docs/issues/local-engine-gathered-scan-card-printing-varying-depth.md): `stream_scan_units` +/// must NOT simply inherit `scan_units` for a printing-varying leaf (no legality partner) once the +/// query's own `matches` sits at or below `STREAM_MIN_MATCHES` -- `run_query_streamed`'s small-total +/// branch pays a second, `push_card_matches`-driven redo pass over every matching card that +/// `GatheredScan`'s single pass never does, and `scan_units` (calibrated only against the single-pass +/// counter, `printings_examined`) has no way to see it. Before this round's fix, the bare `else` arm +/// of `printing_compose`'s `feats.stream_scan_units` assignment read `scan_units as u32` verbatim -- +/// reverting `STREAM_SMALL_TOTAL_REDO_BIAS`'s branch back to that would make this test fail on the +/// first assertion (`stream_scan_units > scan_units`), which is the point: this is the population +/// Round 1's own depth-proxy revision quietly broke for `StreamedSelect` (see the doc's "Round 30" +/// section for the mechanism and the real-corpus numbers this fixture only approximates). +#[test] +fn stream_scan_units_prices_the_small_total_redo_for_a_printing_varying_leaf() { + use rand::SeedableRng; + let mut rng = rand::rngs::SmallRng::seed_from_u64(20_260_830); + // Large enough that a narrow collector_number window still clears `MIN_ROWS`-style noise floors + // and reliably narrows via the index rather than declining to a full scan; narrow enough that the + // window's matches land comfortably under the default `STREAM_MIN_MATCHES` (1,024) so the + // small-total branch this round targets is the one that fires. + let data = fuzz_store_n(&mut rng, 4_000); + let bytes = rkyv::to_bytes::(&data).expect("serialize"); + let archived = rkyv::access::, Error>(&bytes).expect("access"); + let ctx = QueryCtx::from(archived); + let params = kernel_params(Mode::Card, SortCol::Name, false, 100, 0); + + // A fused two-sided bound on a printing-varying, non-legality field -- the exact shape + // (`f:pioneer cn>=30 cn<=39`) the diagnostic round that opened this ledger entry chased, minus + // the legality partner (out of scope for this fix; see the `filter_touches_legality` arm above). + let cn_range = FilterExpr::And(vec![ + FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::CollectorNumberInt), op: CmpOp::Ge, rhs: NumExpr::Const(100.0) }, + FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::CollectorNumberInt), op: CmpOp::Le, rhs: NumExpr::Const(110.0) }, + ]); + let (pe, filter) = split_planes(cn_range, &archived.indexes.planes, &archived.indexes.oracle_trigram.words, true); + let mut acq_filter = filter; + let (feats, prep, _bits) = acquire_plan_features(&ctx, ¶ms, &mut acq_filter, None, pe.as_ref()); + + assert_eq!( + prep.count_source(), + CountSource::PrintingCompose, + "this fixture must reach the compose acquire branch to exercise the fix -- a fused two-sided \ + range bound never qualifies for CardRangePopcount (see its own applicability doc)" + ); + assert!( + feats.matches > 0 && feats.matches <= *STREAM_MIN_MATCHES as u32, + "the window must land in the small-total population this fix targets, got matches={}", + feats.matches + ); + assert!( + feats.stream_scan_units > feats.scan_units, + "a printing-varying leaf whose matches sit at/under STREAM_MIN_MATCHES must charge P3 MORE \ + than the shared scan_units for the redo pass GatheredScan never pays: scan_units={} \ + stream_scan_units={}", + feats.scan_units, feats.stream_scan_units + ); + + // Control: the same field, but the WHOLE printing-varying-leaf else-arm's premise is `Mode::Card` + // only -- `Printing`/`Artwork` must fall back to the bare inheritance, unmodified by this round. + for (label, mode) in [("printing", Mode::Printing), ("artwork", Mode::Artwork)] { + let params2 = kernel_params(mode, SortCol::Name, false, 100, 0); + let cn_range2 = FilterExpr::And(vec![ + FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::CollectorNumberInt), op: CmpOp::Ge, rhs: NumExpr::Const(100.0) }, + FilterExpr::NumericCmp { lhs: NumExpr::Field(NumField::CollectorNumberInt), op: CmpOp::Le, rhs: NumExpr::Const(110.0) }, + ]); + let (pe2, filter2) = split_planes(cn_range2, &archived.indexes.planes, &archived.indexes.oracle_trigram.words, false); + let mut acq_filter2 = filter2; + let (feats2, prep2, _bits2) = acquire_plan_features(&ctx, ¶ms2, &mut acq_filter2, None, pe2.as_ref()); + if prep2.count_source() != CountSource::PrintingCompose { + continue; // this mode/shape didn't reach the branch under test; nothing to assert + } + assert_eq!( + feats2.stream_scan_units, feats2.scan_units, + "{label}: this round's redo correction is Mode::Card-only -- {label} must keep the bare \ + inheritance unchanged" + ); + } +} + +/// Round 31 of the printing-varying-leaf depth ledger +/// (docs/issues/local-engine-gathered-scan-card-printing-varying-depth.md): `PhaseStats::redo_examined` +/// must actually be populated from `push_card_matches`'s return value in `run_query_streamed`'s +/// `total <= STREAM_MIN_MATCHES` branch, not discarded as a bare statement the way it was before this +/// round (Round 30 had to fit `STREAM_SMALL_TOTAL_REDO_BIAS` against a wall-clock residual precisely +/// because no structural counter existed). Reverting the capture back to a bare statement would leave +/// this field at its `PhaseStats::default()` zero forever, which is exactly what the first assertion +/// below rules out. +/// +/// One corpus, two disjoint match groups, so both P3 exits are covered in one fixture: +/// - `cmc == SMALL_CMC` selects `SMALL_GROUP` cards -- under `STREAM_MIN_MATCHES`, so P3 takes the +/// small-total gather-and-quickselect branch this round's counter targets. +/// - `cmc == LARGE_CMC` selects `LARGE_GROUP` cards -- over `STREAM_MIN_MATCHES`, so P3 takes the +/// permutation walk instead, and `redo_examined` must read 0 there (that branch's own +/// `push_card_matches` call is deliberately left uninstrumented -- its cost already flows into +/// `ns_loop`/`ns_finish` via the walk's wall-clock timing, unlike the small-total branch's second +/// pass, which no existing counter could see at all). +#[test] +fn redo_examined_counts_only_the_small_total_redo_pass() { + const SMALL_GROUP: usize = 500; // well under STREAM_MIN_MATCHES (1,024) + const LARGE_GROUP: usize = 2_000; // well over STREAM_MIN_MATCHES + const N: usize = SMALL_GROUP + LARGE_GROUP; + const SMALL_CMC: u8 = 1; + const LARGE_CMC: u8 = 2; + const LIMIT: usize = 10; + + let mut vocab = VocabInterner::new(); + let mut cards = Vec::with_capacity(N); + for i in 0..N { + let mut c = stub_card((i + 1) as u128, TYPE_CREATURE, &[], &mut vocab); + c.cmc = Some(if i < SMALL_GROUP { SMALL_CMC } else { LARGE_CMC }); + cards.push(c); + } + // One printing per card: the redo pass's `examined` count is then bounded to exactly 1 per + // visited card under the default prefer's early-break arm, which is what the lower-bound + // assertion below rests on. + let data = store_of(cards, &vec![1; N], vocab); + let bytes = rkyv::to_bytes::(&data).expect("serialize"); + let archived = rkyv::access::, Error>(&bytes).expect("access"); + let ctx = QueryCtx::from(archived); + + let eq_filter = |cmc: u8| FilterExpr::NumericCmp { + lhs: NumExpr::Field(NumField::Cmc), op: CmpOp::Eq, rhs: NumExpr::Const(f64::from(cmc)), + }; + + // Small-total branch: matches must actually land at/under STREAM_MIN_MATCHES and the small-total + // exit (not the walk) must be the one that ran, checked directly via `perm_steps == 0` rather than + // assumed from the group size alone. + { + let params = kernel_params(Mode::Card, SortCol::Cmc, false, LIMIT, 0); + let (pe, filter) = + split_planes(eq_filter(SMALL_CMC), &archived.indexes.planes, &archived.indexes.oracle_trigram.words, true); + let mut streamed_filter = filter.clone(); + take_phase_stats(); + let streamed = run_query_with_plan(PhysicalPlan::StreamedSelect, &ctx, ¶ms, &mut streamed_filter, None, pe.as_ref()) + .expect("store_of builds the cmc permutation, so StreamedSelect is applicable"); + let stats = take_phase_stats(); + assert_eq!(streamed.0, SMALL_GROUP, "the small group's total must match its own group size exactly"); + assert!( + streamed.0 <= *STREAM_MIN_MATCHES, + "fixture must land in the small-total population this round's counter targets" + ); + assert_eq!(stats.perm_steps, 0, "small-total exit must not have taken the permutation walk"); + assert!( + stats.redo_examined > 0, + "the small-total branch's redo pass ran (matches_pushed={}) but redo_examined read 0 -- \ + push_card_matches's return value is being discarded again, exactly the Round 30 gap this \ + round closed", + stats.matches_pushed, + ); + // Every visited card contributes exactly one push and examines at least the one printing it + // chose (one printing per card here, so exactly one) -- so the sum can never fall short of the + // total matched, only meet or exceed it on a fixture with more printings per card. + assert!( + stats.redo_examined >= stats.matches_pushed, + "redo_examined ({}) must be at least matches_pushed ({}) -- each pushed match examined at \ + least the one printing it chose", + stats.redo_examined, stats.matches_pushed, + ); + + // GatheredScan never redoes anything -- confirms the field stays at its zero default on the + // OTHER materializing plan for the identical query, not just "some plan somewhere". + let mut gathered_filter = filter.clone(); + take_phase_stats(); + let gathered = run_query_with_plan(PhysicalPlan::GatheredScan, &ctx, ¶ms, &mut gathered_filter, None, pe.as_ref()) + .expect("GatheredScan is always applicable"); + let gathered_stats = take_phase_stats(); + assert_eq!(gathered.0, SMALL_GROUP, "GatheredScan must agree with StreamedSelect on the total"); + assert_eq!(gathered_stats.redo_examined, 0, "GatheredScan must never report a redo -- it pays one pass only"); + } + + // Walk branch: matches must land OVER STREAM_MIN_MATCHES and the walk (not the small-total exit) + // must be the one that ran, checked directly via `perm_steps > 0`. + { + let params = kernel_params(Mode::Card, SortCol::Cmc, false, LIMIT, 0); + let (pe, filter) = + split_planes(eq_filter(LARGE_CMC), &archived.indexes.planes, &archived.indexes.oracle_trigram.words, true); + let mut streamed_filter = filter.clone(); + take_phase_stats(); + let streamed = run_query_with_plan(PhysicalPlan::StreamedSelect, &ctx, ¶ms, &mut streamed_filter, None, pe.as_ref()) + .expect("store_of builds the cmc permutation, so StreamedSelect is applicable"); + let stats = take_phase_stats(); + assert_eq!(streamed.0, LARGE_GROUP, "the large group's total must match its own group size exactly"); + assert!( + streamed.0 > *STREAM_MIN_MATCHES, + "fixture must land OVER STREAM_MIN_MATCHES so the walk, not the small-total gather, runs" + ); + assert!(stats.perm_steps > 0, "walk branch must have actually stepped the permutation"); + assert_eq!( + stats.redo_examined, 0, + "the walk branch's own push_card_matches call is deliberately uninstrumented -- its cost \ + already flows into ns_loop/ns_finish via the walk's wall-clock timing, so this field must \ + stay at its zero default there", + ); + } +} diff --git a/docs/issues/00852-engine-compose-acquire-p3-p4-ranking.md b/docs/issues/00852-engine-compose-acquire-p3-p4-ranking.md index fddb63a0d..b9be184fb 100644 --- a/docs/issues/00852-engine-compose-acquire-p3-p4-ranking.md +++ b/docs/issues/00852-engine-compose-acquire-p3-p4-ranking.md @@ -1,11 +1,54 @@ # Rank `GatheredScan` vs `StreamedSelect` on the Compose Acquire -Status: open — the largest single routing error left in the engine. Filed as +Status: **this pair is resolved** — see "Re-verified" below. Filed as [#852](https://github.com/jbylund/sylvan_librarian/issues/852). Successor to [the loop-phase measurement record](done/local-engine-loop-phase-measurement.md), whose calibration work shipped in #833 / #834 / #836. -## Where it stands +## Re-verified after the printing-varying-depth loop (`local-engine-gathered-scan-card-printing-varying-depth.md`) + +That doc's Rounds 1/3/4/6/7 targeted exactly this doc's #1 remaining item — `eval_domain`/`domain_cards` +accuracy for an `And` of printing-varying range leaves in the narrowed regime — without realizing the +connection at the time. Re-measured with `bench_pairwise_ordering.py` against the accumulated result +(`costcell/trunk`, 10 rounds landed): + +| | historical (this doc) | re-verified | +| --- | --: | --: | +| pair ordered right (realistic) | 87% | **97%** | +| pair mean regret (realistic) | 4.29 µs | **0.82 µs** | +| pair gap meas/pred (realistic) | 0.98 | **0.99** | +| pair ordered right (uniform) | — | 95% | +| pair mean regret (uniform) | — | 1.74 µs | + +**This pair is closed as a priority** — 0.82 µs mean regret is now smaller than several pairs that were +never a concern. Item 1 on this doc's "Remaining, in order" list (the narrowed-regime `eval_domain` +error) is substantially resolved; items 2 and 3 are no longer worth pursuing on their own now that this +pair isn't the routing error the pursuit was justified by. + +**The re-verification surfaced a bigger, different problem**, not previously examined by this doc: both +`GatheredScan vs PrintingCompose` and `PrintingCompose vs StreamedSelect` are now the worst pairs in the +engine, concentrated in the `plane`-acquire branch specifically: + +| pair / acquire | ordered right | mean regret | gap meas/pred | +| --- | --: | --: | --: | +| `GatheredScan vs PrintingCompose` [plane], realistic | 87% | 19.09 µs | 0.85 | +| `GatheredScan vs PrintingCompose` [plane], uniform | 83% | 27.21 µs | 0.94 | +| `PrintingCompose vs StreamedSelect` [plane], realistic | 92% | 11.42 µs | 0.75 | +| `PrintingCompose vs StreamedSelect` [plane], uniform | 86% | 15.72 µs | 0.83 | + +Root cause (traced, not yet fixed): `acquire_plan_features`'s `Plane` branch returns +`mk_plan_feats(ctx, params, count, count, scan_units, 0)` directly with no further field overrides — +unlike every other acquire branch, which sets `PrintingCompose`-specific build-cost fields +(`broadcast_printings`, `scatter_printings`, `project_printings`, `popcount_words`, `compose_paging`) +after the shared call. Those fields' defaults in `mk_plan_feats` (0 / `ComposePaging::Gather`) are +correct for the plans that don't read them, but `PrintingCompose` — a genuine alternative plan whenever +the plane-covered predicate is also printing-composable — gets costed off inputs that describe nothing +real about what it would actually do if it won. Same class of bug as the historical +`compose_paging`-left-at-`Gather`-default issue this doc's "Both fixed" section already closed for a +different acquire branch, not yet applied to this one. Tracked onward in +[local-engine-plane-acquire-compose-costing.md](local-engine-plane-acquire-compose-costing.md). + +## Where it stands (historical, before re-verification above) The pair went **69% → 87% ordered right** and mean regret **35.96 µs → 4.29 µs** across that stack. What moved it last was `eval_domain`, and the mechanism is worth stating because it was misdiagnosed twice: diff --git a/docs/issues/done/local-engine-gathered-scan-undercosted-arith-existential-and.md b/docs/issues/done/local-engine-gathered-scan-undercosted-arith-existential-and.md new file mode 100644 index 000000000..13a2fd2f5 --- /dev/null +++ b/docs/issues/done/local-engine-gathered-scan-undercosted-arith-existential-and.md @@ -0,0 +1,1283 @@ +# GatheredScan Under-Costed for a cmc-Range AND a Card-Invariant Existential Leaf + +Found while looking for the highest-latency real queries in a 211k-query uniform sample +(`docs/issues/local-engine-gathered-scan-card-printing-varying-depth.md`'s benchmark corpus/protocol) and +checking whether the router picked the best plan on each. Not yet fixed — this is the starting point for +whoever picks it up. + +## The miss + +`cmc>=1 cmc<=5 border:black`, `unique=card`, `orderby=rarity`, `direction=desc`, `limit=175`, `offset=0`: +one of the 25 highest-latency real queries in the sample, and one of only two in that top-25 where routing +missed the best plan. + +``` +routed: GatheredScan 1,155,375 ns (measured) +best: PrintingCompose 581,708 ns (measured) +regret: 573,667 ns (~2x) +``` + +## Reproducing + +```python +from scripts import costbench +from api.parsing import parse_scryfall_query +engine = costbench.load_engine(pathlib.Path("benchmarks/bitplanes/corpus.jsonl"), pathlib.Path("/store")) +kw = dict(filters=parse_scryfall_query("cmc>=1 cmc<=5 border:black"), unique="card", orderby="rarity", + direction="desc", limit=175, offset=0, prefer="default") +acquire = engine.explain(**kw)["acquire"] +res = engine.explain_analyze(num_warmups=3, num_trials=15, **kw) +``` + +## Diagnosis: `PrintingCompose`'s estimate is fine here — `GatheredScan`'s is the one that's wrong + +Reconstructed both plans' `predicted_ns` term-by-term from the real `acquire` feature dump and `cost.rs`'s +constants, and both formulas reproduce the reported `predicted_ns` almost exactly — so the feature values +below are trustworthy, not an artifact of a different bug in the reconstruction: + +**`PrintingCompose`**: `broadcast_printings=181,706`, `project_printings=83,894`, `popcount_words=496`, +`compose_paging=OrderbyWalk` (`printings_walked=1,011`). +``` +build = 181,706*1.93 + 83,894*1.93 + 496*1.07 = 513,139 +page = 1,011*0.58 + 175*2.19 = 970 +total = 513,139 + 970 + 163.56 = 514,272 (reported: 514,272.07 — exact match) +``` +Real measured trials: 542,166 – 636,958 ns. **Ratio ~1.13-1.2x — reasonably well-calibrated.** `broadcast_printings` +alone is 68% of this total, and it's driven entirely by the bare `cmc` range: re-querying `cmc>=1 cmc<=5` +alone (no `border`) reproduces the identical `broadcast_printings=181,706`, while `border:black` alone gives +`broadcast_printings=0` — confirming `border` reads a precomputed plane (cheap) and `cmc`'s own card-invariant +broadcast is the real, correctly-priced cost driver here, not a bug in `PrintingCompose`'s own arm. + +**`GatheredScan`**: `eval_domain=24,734`, `scan_units=83,894`, `matches=24,543`, `residual_tier_ns100=0` +(i.e. "nothing to verify" — the `tier_ns > 0.0` gate in `cost.rs`'s `GatheredScan` arm never fires). +``` +loop = 24,734*3.88 = 95,968 +scan = 83,894*2.06 = 172,822 +push = 24,543*2.24 = 54,976 +collect = 175*9.79 = 1,713 +total = 95,968+172,822+54,976+1,713+169.6 = 325,649 (reported: 326,262.98 — matches within rounding) +``` +Real measured trials: 1,015,209 – 1,290,166 ns. **Ratio ~3.1-4.0x — this is the actual bug.** + +If `residual_tier_ns100` were nonzero instead of 0 (charging `GATHER_CARD_PASS_NS + GATHER_RESIDUAL_FLOOR_NS` +per candidate, the formula's own floor for "there is something to verify"): `24,734 * (3.00 + 18.89) = 541,427` +additional ns → a would-be total of **867,076**, closing most (not all) of the gap to the measured range. This +doesn't prove the mechanism, but it's the single largest lever in the formula and the most likely place to +look first. + +## Where to look + +- `card_engine/src/lib.rs`, the `PrintingCompose`-acquire branch of `acquire_plan_features` (search for where + `tier`/`residual_tier_ns100` gets decided — `verify_cost_tier_unproven`, `nothing_to_verify`, + `compose_leaf_nothing_to_verify`, `card_invariant_domain_exact` are the names that came up investigating + nearby rounds this session; none were traced against this specific shape). The question: for an `And` of an + arith-tuple range (`cmc`) and a card-invariant existential leaf (`border`), does whatever proves "nothing + left to verify" actually hold for `GatheredScan`'s own per-candidate pass, or is it borrowing a proof that's + only valid for a different plan/mechanism? +- `card_engine/src/cost.rs`: `GATHER_CARD_PASS_NS` (3.00), `GATHER_RESIDUAL_FLOOR_NS` (18.89), and the + `tier_ns > 0.0` gate in the `GatheredScan` arm of `plan_cost`. +- Cross-check against the exact-tightening machinery already built in `compose_printing_estimate` for + arith+existential combinations (`compose_printing_estimate`'s `And` arm, `best_other`, `arith_tuple_count`, + the ID-probe merge) — this may be a downstream consequence of one of those mechanisms correctly proving an + exact CARD COUNT while something else incorrectly reads that as "no residual work at all" for `GatheredScan` + specifically. + +## Open questions (not resolved here) + +- **Does the mis-route need the `AND` with `border:black`, or does bare `cmc>=1 cmc<=5` alone already + mis-route?** Only the *feature* values were isolated (both give `broadcast_printings=181,706`), not full + routing — worth checking before assuming the `And` combination itself is load-bearing. +- **Is this the same root cause as the other `printing_compose`-acquire miss in the same top-25** + (`f:commander year>2003`, unique=artwork, a much smaller ~43,300 ns/~5% miss in the other direction — + `GatheredScan` picked when `PrintingCompose` was actually 43µs better)? Not checked — could be the same + `tier` classification issue manifesting in both directions, or two unrelated mechanisms. +- **Real-traffic size of this population.** Not measured — a natural next check is + `bench_pairwise_ordering.py` sliced to this AST shape (arith-tuple range AND card-invariant existential + leaf, `printing_compose` acquire) to see whether this is a rare edge case or a real regret contributor + worth its own round. + +## Related + +- [local-engine-gathered-scan-card-printing-varying-depth.md](local-engine-gathered-scan-card-printing-varying-depth.md) — + the session-long effort this was found during; its Rounds 1-9 fixed `domain_cards`/`eval_domain` accuracy + for *printing-varying* range leaves, not `cmc` (card-invariant) — a different population from this one. +- [00852-engine-compose-acquire-p3-p4-ranking.md](00852-engine-compose-acquire-p3-p4-ranking.md) — the + `GatheredScan`/`StreamedSelect` pair, resolved; this doc is the `GatheredScan`/`PrintingCompose` pair, + still open. + +## Follow-up round: root cause found, fixed (correction to this doc's own title) + +Picked this doc up and answered the three open questions with real data before touching code, per this +round's brief. Correction up front: **the title's "card-invariant existential leaf" is a contradiction in +terms, and `border` is not one** — see Q2 below. `border` (and `rarity`) are printing-VARYING, which is +*why* they are existential. The bug is exactly that the router's tier logic treated them as if they were +card-invariant like `cmc`/`color`/`type`/`devotion`. + +### Q1 — does the mis-route need the `And` with `border:black`? + +Yes. Ran `explain_analyze` on bare `cmc>=1 cmc<=5` alone (same `unique=card`, several `orderby`/`limit`/ +`offset` combos) against the real corpus: + +``` +cmc>=1 cmc<=5 orderby=rarity desc limit=175 -> GatheredScan picked, median 184,208 ns + PrintingCompose median 542,292 ns (NOT picked) +``` + +`GatheredScan` really is ~3x faster here and the router picks it correctly. Adding `border:black` blows +`GatheredScan`'s REAL time up ~6x (184,208 -> 1,155,375-1,290,166 ns) while its *predicted* cost barely +moves (327,291 -> 326,263 — `PlanFeatures` even reads slightly cheaper). The `And` is load-bearing: this +is not a bare-range problem. + +### Q2 — where does `residual_tier_ns100` actually get set to 0, and is the classification wrong or is the bug elsewhere? + +Traced it exactly. `cmc>=1 cmc<=5 border:black` under `unique=card`: **both** `cmc>=1`/`cmc<=5` +(`compile_numeric_cmp`) and `border:black` (`compile_border_cmp`) compile into `PlaneExpr`s +(`planes.rs::compile_plane`), and `split_planes`'s whole-filter shortcut folds the entire `And` into ONE +plane, leaving the residual `filter == FilterExpr::True`. + +`acquire_plan_features`'s `PrintingCompose`-acquire branch then asks `plane_leaves_nothing_to_verify`: + +```rust +fn plane_leaves_nothing_to_verify(filter, mode, plane, indexes) -> bool { + matches!(filter, FilterExpr::True) + && plane.is_none_or(|expr| { + matches!(mode, Mode::Card) || !plane_expr_is_existential(expr, divergent_formats) + }) +} +``` + +For `Mode::Card` this returns `true` **unconditionally**, regardless of what the plane actually contains. +Its own doc justifies this only for *legality*: "the card has some legal printing" is exactly what +`unique=card` wants, so the #667 carveout lets `Mode::Card` skip re-verifying a divergent legality format +per printing. But `plane_expr_is_existential` is not legality-specific — `planes.rs:: +needs_printing_verification` says plainly: **"for rarity and border that is every leaf"** (unconditionally +existential, unlike legality's per-format `divergent_formats` gate). The blanket `matches!(mode, Mode::Card) +||` bypass does not distinguish "legality, existential only for one divergent format" from "border/rarity, +always existential" — it grants the SAME free pass to both. + +**So: the classification is a real bug, not a borrowed proof.** It is not "correct classification, bug +elsewhere" — `plane_leaves_nothing_to_verify`'s own Mode::Card carveout is unsound whenever the plane +touches rarity or border, and that unsoundness is exactly what leaks into `tier`/`residual_tier_ns100`. + +A second, independent instance of the same conceptual bug turned up while verifying the fix: even after +correcting the plane-side check, `t:swamp tou=5 border:black`/card still read `residual_tier_ns100 == 0`, +because the OTHER disjunct, `compose_leaf_nothing_to_verify(filter)`, fires whenever the *residual* is a +bare safe collection leaf (`t:swamp`) — correct on its own terms (subtypes really are card-invariant) but +blind to what the `plane` alongside it contains. The original code OR'd two whole-query claims +(`plane_leaves_nothing_to_verify(filter, mode, plane, ..) || compose_leaf_nothing_to_verify(filter)`) when +what was needed was an AND of two HALF claims (filter side AND plane side each independently have nothing +left to verify). + +### Q3 — what does the real executor do, and is the per-printing work genuinely necessary? + +Traced `exec_gathered_scan`/`push_card_matches` directly (with temporary `eprintln!` instrumentation, +since reverted). `prepare_candidates`'s `all_match_known` is (harmlessly) `true` in both the border and +no-border cases — but a SEPARATE mechanism, `existential_plane_for` (gated on `plane_expr_is_existential` +with NO Mode::Card carveout at all), independently returns `Some` whenever the plane touches an existential +leaf, and forces `push_card_matches` into a per-printing loop that calls `eval_plane_expr_for_printing` on +each printing until one satisfies the FULL plane (both the constant `cmc` bit and the per-printing `border` +bit) — because `unique=card` still must return an ACTUAL border:black printing as the result row, not +merely prove one exists somewhere in the card's span. Confirmed against the real counters: `printings_examined` +(27,142) exceeds `cards_visited` (26,905) by 237 — cards whose first-checked printing didn't happen to be +black and needed a second look — proving the per-printing walk is real, not a costing artifact. + +This work is genuinely necessary for correctness (unlike legality's carveout, which is a deliberate, +documented product decision that `unique=card` need not re-verify format legality per printing) — border +really can vary printing to printing for the same card, and the row returned has to actually match. The +cost model must charge for it; it cannot be modeled away. + +### Q4 — how big is this population in real traffic? + +Sampled `client.query_sampler.QuerySampler` in `realistic` mode: in 60s (31,111 queries, 7,928 +`printing_compose`-acquire), 2,424 had `residual_tier_ns100 == 0` with `GatheredScan` picked, of which **61 +(2.5%) touched `border`/`rarity`** — mostly common shapes like `r:rare t:plains`, `c:w r:mythic`, +`border:black t:shapeshifter`. This is not rare in the sense of "never happens" — rarity/border combined +with a type/color/keyword leaf is a completely ordinary real query. + +But most of those 61 are SMALL-domain queries (a type/subtype leaf narrows hard), where the added tier +charge is a few µs against a query that was already single-digit-µs to begin with — spot-checked six of +them directly and `GatheredScan` remained correctly picked (and still fastest measured) both before and +after the fix. The specific sub-case that produces a *large*, `2x`-latency-class regret — a BROAD +card-invariant range (`cmc`/`power`/`toughness`) that leaves a large candidate domain, ANDed with +`border`/`rarity` — is a narrower slice of that population. This matches the flat aggregate +`bench_pairwise_ordering.py`/`bench_regret_matrix.py` numbers below: real, worth fixing (it's free and +correctness-preserving), but not a population large enough to move whole-corpus aggregates on its own. + +### The fix + +`card_engine/src/lib.rs`: added `plane_touches_rarity_or_border` (walks a compiled `PlaneExpr`, true iff +any leaf's plane index is `>= PLANE_RARITY` — rarity and border are the last two plane families, +contiguous through `PLANE_COUNT`, so an index compare identifies them exactly with no new table to keep in +sync with `planes.rs`'s private `PLANE_BLOCKS`) and `cost_plane_nothing_to_verify` (the plane-only half of +the check, with the `Mode::Card` bypass scoped to exclude a plane touching rarity/border). The `tier` +computation now ANDs the filter half and the plane half independently: + +```rust +let filter_nothing_to_verify = matches!(filter, FilterExpr::True) || compose_leaf_nothing_to_verify(filter); +let nothing_to_verify = filter_nothing_to_verify && cost_plane_nothing_to_verify(mode, plane, indexes); +let tier = if nothing_to_verify { 0 } else { verify_cost_tier(composed) }; +``` + +Deliberately scoped to ONLY this call site (the `PrintingCompose`-acquire branch's `tier` decision, the +one term the tracking doc's diagnosis identified as the actual bug). `plane_leaves_nothing_to_verify` +itself — used by the EXECUTOR's `all_match_known` in `prepare_candidates` — is untouched: granting +Mode::Card's bypass to rarity/border there is harmless (it only skips a redundant, already-cheap +`card_pass` call; the real per-printing correctness work runs through the wholly separate +`existential_plane_for` mechanism regardless of what `all_match_known` says). The other call site of +`plane_leaves_nothing_to_verify` (the `eval_domain`/`scan_units` broad-reset guard a few lines up) is also +left alone — a different concern (domain-size estimation, not verify cost) that this round's diagnosis did +not implicate. + +**Pre-computation check**: the fix adds one small, bounded-size `PlaneExpr` tree walk (typically 1-5 +nodes) once per acquire — no new per-candidate, per-match, or per-printing work, and no new index probe. +Cost is independent of corpus size, match count, or candidate count. + +### Before / after (real corpus, original reproducer) + +``` +cmc>=1 cmc<=5 border:black, unique=card, orderby=rarity, direction=desc, limit=175, offset=0 + +BEFORE: GatheredScan predicted 326,263 ns picked=True median 1,155,375-1,290,166 ns + PrintingCompose predicted 514,272 ns picked=False median 542,166- 636,958 ns + +AFTER: PrintingCompose predicted 514,272 ns picked=True median 478,333- 511,166 ns + GatheredScan predicted 728,028 ns picked=False median 1,098,541-1,288,333 ns +``` + +Router now picks the actually-faster plan — a measured ~2.3-2.4x real latency win on this exact query. +`GatheredScan`'s revised `predicted_ns` (728,028) still under-charges the real 1.0-1.3ms (the tier's flat +`GATHER_RESIDUAL_FLOOR_NS`-based charge isn't calibrated for a per-printing existential walk specifically), +but the ARGMIN decision is what matters and it's now correct — no attempt was made to tighten the +absolute number further, since the primary metric for this round is ordering, not agreement. + +### Correctness gate + +`cargo test --manifest-path card_engine/Cargo.toml --release`: **168/168 passed** (167 pre-existing + one +new regression test, `compose_tier_charges_border_existential_and_arith_range`, added to `tests.rs` — a +minimal fixture reproducing the exact AST shape, asserting `residual_tier_ns100 > 0` for the `cmc`+`border` +`And` and `== 0` for the bare `cmc` range control). `cargo clippy --all-targets -- -D warnings`: clean. + +### Confirmation pass + +`bench_pairwise_ordering.py --seconds 60` (`GatheredScan` vs `PrintingCompose`, `printing_compose` acquire), +baseline vs fix, both uniform and realistic mode: + +``` +uniform: baseline 86% ordered right, 5.47µs mean regret -> fix 86%, 5.51µs (flat, within noise) +realistic: baseline 90% ordered right, 3.32µs mean regret -> fix 90%, 3.24µs (flat, within noise) +``` + +No aggregate movement either direction, consistent with Q4's population-size finding — the fix corrects a +real, narrow sub-population that doesn't dominate this pairwise slice's total regret. No regression. + +`bench_cost_model_agreement.py --seconds 60`, `GatheredScan`/`card`: baseline median 0.81 (25% within 25%) +-> fix median 0.81 (25% within 25%) — unchanged, still PASS. + +`bench_regret_matrix.py --seconds 120 --mode realistic`: baseline total regret 47.2ms over 52,384 queries +-> fix 45.2ms over 54,435 queries (~4% lower, more queries fit the same wall-clock budget because fewer +ran the now-corrected expensive misroute) — no regression, mild improvement, within this benchmark's +sample-to-sample noise band. + +`bench_query_latency_ab.py --sample 400 --mode realistic --seed 7`, plus a same-build canary at the same +seed: + +``` +canary (baseline vs baseline): B - A = -0.6µs 95% CI [-0.9, -0.4] (noise floor) +baseline vs fix: B - A = +0.1µs 95% CI [-0.2, +0.3] NO DETECTABLE DIFFERENCE +``` + +Both indistinguishable from the noise floor — expected at n=400 given the affected shape's rarity (Q4). +No regression on general realistic-mode latency. + +### Outcome + +**Fixed.** Real bug (not a rare/skip-it case, not a borrowed-proof-elsewhere case), cheap to fix (no +hot-path cost added), shipped with a passing correctness gate and no detected regression on any +confirmation metric. The Phase A Q4 population size (rarity/border combined with something else is a +common query shape) argues this was worth fixing on principle even though it's invisible in whole-corpus +aggregates; the specific large-regret sub-case (broad card-invariant range AND rarity/border) is real and +now routes correctly. + +## Round 16: the fix above was field-specific, and reproduced its own bug for a different field + +The round before this one shipped `plane_touches_rarity_or_border` -- a plane-INDEX-RANGE check +(`(*p as usize) >= PLANE_RARITY`) -- to scope the `Mode::Card` bypass off rarity/border. A review of that +fix raised the architectural objection this round exists to answer: the conditional should key off +BEHAVIOR (does a plane's existential semantics force per-printing re-verification even under +`Mode::Card`?), not off which specific FIELD is touched. The number of behavioral categories grows far +slower than the number of query attributes, and hardcoding a field-identity check (a plane-index range) +where a behavioral one already existed in `planes.rs` was the same shape of mistake the original bug was +built from, just one level up. + +### Is the gap real? Yes -- confirmed by reproduction, not just by reading the code + +`cost_plane_nothing_to_verify`'s shipped shape was `(Mode::Card && !plane_touches_rarity_or_border(expr)) +|| !plane_expr_is_existential(expr, divergent_formats)`. For a plane that is existential ONLY via a +DIVERGENT legality leaf (no rarity, no border anywhere in it), `plane_touches_rarity_or_border` returns +`false` -- so the first disjunct is `Mode::Card && true`, which is `true` under `Mode::Card` regardless of +what the second disjunct would say, short-circuiting the OR. This reproduces the exact bug shape Round 15 +fixed, scoped to legality instead of rarity/border. + +Reproduced on the real corpus (`oldschool` is the production corpus's one divergent format): + +``` +f:oldschool, unique=card, orderby=rarity desc, limit=175: + residual_tier_ns100 == 0 (bug fires) + GatheredScan predicted 10,358ns measured median 29,083ns ratio 0.36 (2.8x under-charge) + printings_examined=6,037 vs cards_visited=961 (6.3 printings/card -- the per-printing walk is real) + +f:oldschool cmc>=1 cmc<=5, unique=card, orderby=rarity desc, limit=175 (same AND shape as the original +cmc+border finding, legality instead of border): + residual_tier_ns100 == 0 (bug fires) + GatheredScan predicted 18,519ns measured median 58,625ns ratio 0.32 (3.16x under-charge) +``` + +Control (`otag:triggered-ability` alone, no plane at all): predicted 124,633ns vs measured 101,667ns, +`cards_visited == printings_examined` (no existential walk) -- confirms the gap is specific to the +existential-plane case, not a general costing artifact. + +### Mechanism: why legality's card-mode bypass is sound for a NON-divergent format, and why it was never sound for a divergent one + +Traced `existential_plane_for` (lib.rs) and `push_card_matches`'s `existential_plane` branch directly, +rather than trusting the doc comments that motivated Round 15's carveout. The answer turned out simpler +than "legality is special": **`existential_plane_for` grants NO per-family carveout at all.** It is: + +```rust +fn existential_plane_for(mode, plane, indexes) -> Option<...> { + match (mode, plane) { + (Mode::Card, Some(pe)) if plane_expr_is_existential(pe, divergent_formats) => Some((pe, planes)), + _ => None, + } +} +``` + +Whenever this returns `Some`, `push_card_matches` walks printings one by one +(`eval_plane_expr_for_printing`) to find an ACTUAL witnessing printing for row selection (#667: "the card +has some legal printing" is enough for the COUNT, but `unique=card` still must return a printing that +really satisfies the query) -- for rarity, border, OR a divergent legality format, identically. There is +no separate, cheaper mechanism for legality. Confirmed empirically: `f:oldschool` alone shows +`printings_examined` (6,037) far exceeding `cards_visited` (961) -- the walk is real, not a costing +artifact, for legality just as it was for border in Round 15. + +So "needs per-printing re-verification under Mode::Card" and "is this a printing-level property" are NOT +two concepts to reconcile -- they are the same fact, traced end to end. A property is printing-level +exactly when two printings of one card can disagree on it, and that is exactly when +`existential_plane_for` forces the row-selection walk. Rarity and border are STATICALLY printing-level +(the field structurally allows disagreement, unconditionally). Legality is the one property that is +DYNAMICALLY either bucket, resolved per format by `divergent_formats` (data-derived per store): card-level +for a format every printing happens to agree on (31 of 32 formats in the production corpus), printing-level +for one where they don't (`oldschool`). `needs_printing_verification`/`plane_expr_is_existential` +(`planes.rs`) already compute exactly this, per plane index, via the family-keyed `PLANE_BLOCKS` table -- +this was never a fact `cost_plane_nothing_to_verify` needed a NEW field-specific check to derive; it needed +to stop re-deriving a narrower, wrong version of a fact `planes.rs` already had exactly right. + +One more invariant closes the loop: `split_planes` (planes.rs) only ever folds an existential leaf into +`plane` under `unique_is_card` (its whole-filter and `And`-child guards are both `unique_is_card || +!plane_expr_is_existential`) -- so `plane_expr_is_existential(plane)` being true already implies `mode == +Mode::Card`. There is no live case where `mode` needs to appear in `cost_plane_nothing_to_verify` at all; +the `Mode::Card` term in both the buggy Round 15 shape and a naive "just drop the field check" fix would +be vestigial, not a correctness need. + +### The fix: delete the field-specific check, use the general one directly + +`card_engine/src/lib.rs`: deleted `plane_touches_rarity_or_border` entirely. `cost_plane_nothing_to_verify` +is now: + +```rust +fn cost_plane_nothing_to_verify(plane: Option<&PlaneExpr>, indexes: &Archived) -> bool { + plane.is_none_or(|expr| !plane_expr_is_existential(expr, u64::from(indexes.planes.divergent_formats))) +} +``` + +No `mode` parameter, no per-family branch, no new table: `plane_expr_is_existential` (planes.rs) already +is the general, field-agnostic predicate, keyed by family through `PLANE_BLOCKS`/`ExistentialLeaf`/ +`needs_printing_verification`, not by ad hoc field identity. A future printing-varying field is handled +correctly automatically by which `PLANE_BLOCKS` entry it lands in -- no new arm needed in this file at all. +`card_engine/src/planes.rs` gained doc-comment-only clarifications on `ExistentialLeaf` and +`needs_printing_verification` making this "same fact, not two concepts" framing explicit (no logic +changes there). + +### Checking for a duplicate, table-driven property classifier already existing elsewhere: not needed, but three OTHER card/printing classifiers already exist for different purposes + +Before extending `planes.rs`'s table, checked whether the crate already had a canonical card-vs-printing +classifier being duplicated. It does not need a NEW one -- `plane_expr_is_existential` already was one -- +but three OTHER classifiers exist, each scoped to a different purpose, each already documenting its own +deliberate disagreement with the canonical table on legality specifically: + +- **`estimator.rs::has_printing_varying_leaf`** (ANY-composition, for cardinality estimation): treats + `FilterExpr::Legality` as ALWAYS printing-varying, ignoring `divergent_formats` entirely. Its own doc + comment already flags this: "conservative... even though `printing_dependent` ranks it invariant for + its own (common-case) reason." Conservative in the estimator's own safe direction (overestimates + variance), not a silent-zero-cost bug. +- **`filter.rs::printing_dependent`/`leaf_compares_printing_field`** (verify-ORDER heuristic, ALL- + composition): treats `FilterExpr::Legality` as ALWAYS card-level, the OPPOSITE bias, also already + documented: "Divergent-legality cards defer to the printing, but they are a rare exception... rank by + the common card-level case." This only affects which child a verifier checks first, never correctness + -- a suboptimal order, not a wrong answer. +- **`lib.rs::is_broadcast_leaf_shape`/`is_broadcast_composable`**: NOT a duplicate -- `is_broadcast_composable` + and `broadcast_composable_card_bits` call `plane_expr_is_existential` directly as their own gate. Already + unified with the canonical table by construction. + +Neither of the first two is the same shape of bug as this round's finding: both are DOCUMENTED, ONE- +DIRECTION approximations for a heuristic or an estimate, not a place where real per-printing work gets +silently priced as free. Flagging as a candidate for a future doc (a single canonical property table the +first two could read from instead of carrying their own copy of the leaf list) -- not attempted here; +out of scope for this round, which is the router's cost-tier fix only. + +### Combinations verified (real corpus, `unique=card`, `orderby=rarity desc`, `limit=175`) + +| combination | predicted (before → after) | measured median | ratio (after) | +|---|---|---|---| +| rarity alone (`r:mythic`) | 24,087 (unchanged) | 20,834–21,083 | 1.14–1.16 (unaffected, already correct) | +| border alone (`border:black`) | 166,380 (unchanged) | 137,375–146,000 | 1.14–1.21 (unaffected, already correct) | +| non-divergent legality alone (`f:modern`) | tier stays 0 (unchanged) | n/a | correct both before/after | +| **divergent legality alone (`f:oldschool`)** | 10,358 → 31,394 | 25,917–29,083 | **1.08–1.21 (fixed, was 0.36)** | +| rarity + divergent legality (`f:oldschool r:mythic`) | 49,021 (unchanged) | 6,375–6,875 | unaffected (Round 15 already covered this: the plane touches rarity) | +| border + divergent legality (`f:oldschool border:black`) | 50,121 (unchanged) | 48,291–51,041 | 0.98–1.04 (unaffected, Round 15 already covered this) | +| **divergent legality + card-invariant range (`f:oldschool cmc>=1 cmc<=5`)** | 18,519 → 51,276 | 56,959–58,625 | **0.90–0.91 (fixed, was 0.32)** | + +The two combinations Round 15 already handled correctly (anything touching rarity/border, alone or +combined with legality) are unchanged by this round's fix -- `plane_expr_is_existential` agrees with +`plane_touches_rarity_or_border` whenever rarity/border is present; it only disagrees (correctly) when +the ONLY existential leaf is a divergent legality format. + +### Correctness gate + +`cargo test --manifest-path card_engine/Cargo.toml --release`: **172/172 passed** (168 pre-existing + +3 new property tests in `planes.rs` asserting the `PLANE_BLOCKS` family invariant holds for every plane +index -- rarity/border unconditionally existential, legality existential iff its own format's bits are +set in `divergent_formats`, everything else never existential -- for a matrix of representative +`divergent_formats` masks (0, `u64::MAX`, one arbitrary bit), plus 1 new concrete regression test, +`compose_tier_charges_divergent_legality_existential_and_arith_range`, mirroring Round 15's +`compose_tier_charges_border_existential_and_arith_range` fixture shape with legality instead of border +and asserting both directions: `cmc` range + DIVERGENT-format legality must charge a nonzero tier, `cmc` +range + NON-divergent-format legality must stay free). Round 15's own regression test still passes +unchanged. `cargo clippy --all-targets -- -D warnings`: clean. + +Pre-computation check: the fix is a net REDUCTION in per-acquire work versus Round 15's shape -- one +`PlaneExpr` tree walk (`plane_expr_is_existential`) instead of two (`plane_touches_rarity_or_border` plus +the `plane_expr_is_existential` fallback the OR could still reach). No new per-candidate, per-match, or +per-printing work; cost is still independent of corpus size, match count, or candidate count. + +### Confirmation pass + +`bench_pairwise_ordering.py --seconds 300`, baseline vs fix, both modes (printing_compose acquire slice, +the one this fix touches): + +``` +realistic: baseline 90% ordered right, 3.06µs mean regret -> fix 90%, 3.01µs (flat, within noise) +uniform: baseline 86% ordered right, 4.94µs mean regret -> fix 86%, 5.06µs (flat, within noise) +``` + +`bench_cost_model_agreement.py --seconds 300 --seed 0`, `GatheredScan`/`card`: baseline median 0.79 (25% +within 25%) -> fix median 0.79 (25% within 25%) -- unchanged. + +`bench_regret_matrix.py --seconds 120 --mode realistic --seed 0`: baseline total regret 41.4ms over +55,944 queries -> fix 41.6ms over 55,963 queries -- flat (+0.5%), within sample-to-sample noise. + +`bench_query_latency_ab.py --sample 800 --mode realistic --seed 7`, interleaved A/B/A, plus a same-build +canary at the same seed: + +``` +canary (baseline vs baseline): B - A = -0.8µs 95% CI [-0.9, -0.6] +baseline vs fix: B - A = -0.7µs 95% CI [-0.9, -0.5] INDISTINGUISHABLE FROM CANARY +``` + +No detectable difference from the fix on general realistic-mode latency, as expected given the affected +shape's rarity in the overall query mix (one divergent format in the production corpus). + +### Outcome + +**Fixed, and generalized.** The divergent-legality gap was real (2.8–3.2x under-charge, confirmed by +reproduction before touching code) and is now closed by deleting the field-specific check Round 15 added +and routing through the general, already-existing `plane_expr_is_existential` predicate instead -- which +also removes a `mode` parameter that turned out to be redundant by construction. Blast radius: `lib.rs` +(the fix), `planes.rs` (doc clarifications + 3 new property tests), `tests.rs` (+1 concrete regression +test). No hot-path cost added (net cheaper than Round 15's shape). No regression on any confirmation +metric. The three other card/printing classifiers found while checking for duplication (`estimator.rs`, +`filter.rs`) are flagged as a candidate for a future unification doc, not attempted here. + +## Round 17: the flat per-candidate charge is real, but a depth term doesn't fix it -- negative result + +Round 16 fixed the classification bug (`tier` correctly reads nonzero for an existential plane) but left +a note that the reproducer's `GatheredScan` prediction (728,028ns) still undershot the measured range +(1,015,209-1,290,166ns), and hypothesized the mechanism: `push_card_matches`'s `Mode::Card`/ +`Prefer::Default` arm early-exits (`(start..end).find(|&pid| satisfies(pid))`), so the number of +printings actually visited per candidate depends on the existential leaf's own selectivity, not a flat +per-candidate constant -- an "expected walk depth" problem, the same SHAPE Round 1 of the sibling +`local-engine-gathered-scan-card-printing-varying-depth.md` effort solved for printing-varying RANGE +leaves (price/date/collector_number). This round picked that up, built a real fix, and then discarded it +after the calibration data itself said no. Recorded here in full because the diagnosis along the way is +the useful part. + +### Re-confirming the reproducer, fresh + +Rebuilt an isolated release wheel from `costcell/trunk` (Round 16's state) and re-ran the exact +reproducer: + +``` +cmc>=1 cmc<=5 border:black, unique=card, orderby=rarity desc, limit=175, offset=0: +GatheredScan predicted 728,028ns measured 1,031,292-1,163,417ns (median 1,136,250) ratio ~1.4-1.6x +real counters: cards_visited=26,905 printings_examined=27,142 matches_pushed=26,905 +real depth (printings_examined / cards_visited) = 1.0088 +``` + +**The early-exit walk essentially never proceeds past the first printing for this exact query** -- +average depth 1.009, i.e. almost every candidate's very first checked printing already satisfies +`border:black`. This on its own already says a depth-scaled correction cannot explain this specific +query's gap: at depth ≈ 1 any sound depth model can only multiply the existing charge by ≈1, and the +gap is 1.4-1.6x. + +### Quantifying across a broader population: depth is real, but it's not what's wrong here + +Sampled 19 hand-picked existential-plane/`Mode::Card`/`Prefer::Default` queries first (varying which +border/rarity value, with and without an ANDed `cmc`/`pow`/`tou` range), then a much larger, non-cherry- +picked sample via `client.query_sampler.QuerySampler` (2,564 rows, `uniform` mode, seed 0; 4,431 rows, +`realistic` mode, seed 1; both filtered client-side to `unique=card` queries whose text touches +`border:`/`r[<>]=?`/`f:oldschool`), recording `GatheredScan`'s `predicted_ns`/`plan_self_ns` (measured) +and the real depth from `printings_examined`/`cards_visited`. + +Two clean, well-supported findings came out of the broad sample: + +**1. Real depth genuinely predicts under/over-costing, in aggregate:** + +``` +uniform (n=2,564): realistic (n=4,431): + depth [1.00,1.05) n=657 median ratio 0.95 depth [1.00,1.05) n=2,220 median ratio 0.53 + depth [1.05,1.50) n=201 median ratio 0.75 depth [1.05,1.50) n=816 median ratio 0.54 + depth [1.50,2.50) n=188 median ratio 1.01 depth [1.50,2.50) n=448 median ratio 0.70 + depth [2.50,4.00) n=181 median ratio 1.38 depth [2.50,4.00) n=289 median ratio 1.07 + depth [4.00, ∞) n=1,327 median ratio 2.81 depth [4.00, ∞) n=583 median ratio 2.17 +corr(log depth, log ratio) = 0.50 (uniform), 0.39 (realistic) +``` + +Not noise: monotonic in both modes, over thousands of rows, and the direction matches the hypothesis +(higher real depth ⇒ more under-costed). + +**2. But depth is a property of the LEAF VALUE, not of what's ANDed alongside it -- and for the flagship +reproducer's shape (a common existential value), that intrinsic depth is ≈1, so the "AND" isn't where +the gap comes from.** Confirmed directly: `cmc>=1 cmc<=5 r:mythic` and bare `r:mythic` (no `cmc` at all) +measure the SAME real depth (2.538 vs 2.528) -- the arith range restricts WHICH cards are candidates, +but does not change WHERE in a candidate's own print history the existential value tends to sit. Same +for `cmc>=1 cmc<=5 border:black` (depth 1.009) vs bare `border:black` (depth 1.009). So whatever is +wrong with the flagship reproducer's costing is NOT "the AND makes the walk deeper" -- it is something +else, present regardless of depth. + +Isolating the executor's own per-candidate loop cost (`ns_loop / cards_visited`, from `explain_analyze`'s +per-plan phase breakdown) against the model's flat per-candidate charge (`GATHER_LOOP_PER_CARD_NS + +GATHER_CARD_PASS_NS + tier.max(GATHER_RESIDUAL_FLOOR_NS)` = 25.77ns, constant for every row below since +`residual_tier_ns100` reads the same 400 for all of them) shows what that "something else" is: + +``` +query real depth real ns_loop/candidate model's flat charge +border:black (bare) 1.01 9.46 25.77 (over-charged) +cmc>=1..5 border:black 1.01 36.07 25.77 (under-charged) +cmc>=2..3 border:black 1.01 28.65 25.77 (~even) +pow>=1..3 border:black 1.01 30.66 25.77 (~even) +r:mythic (bare) 2.53 15.24 25.77 (over-charged) +cmc>=1..5 r:mythic 2.54 73.22 25.77 (under-charged) +``` + +At the SAME real depth (~1.0-1.01), evaluating the COMPOUND existential plane (the `cmc` bound AND the +`border`/`rarity` equality, both tested per printing by `eval_plane_expr_for_printing`) costs 3-4x more +per candidate than evaluating the BARE existential leaf alone. That is a real, distinct gap -- the +`GATHER_CARD_PASS_NS`/`GATHER_RESIDUAL_FLOOR_NS` constants were fitted against a "one `filter.card_pass` +call" cost shape (see their own docs in `cost.rs`), not against "evaluate a multi-leaf `PlaneExpr` +conjunction per printing" -- but it is a plane-EVALUATION-cost gap, not a depth gap, and it is out of +scope for "design an expected-depth estimate" (this round's brief item 2). Flagging it here rather than +chasing it, since the brief's escape hatch is specifically for exactly this outcome. + +### A depth fix was still built and tested, for the population where depth genuinely is the mechanism + +Even though depth doesn't explain the flagship reproducer, the broad sample's bucketed table above says +depth-driven under-costing is real SOMEWHERE in this population (the `depth ≥ 4` bucket reads median +ratio 2.2-2.8x). So a real attempt was made: added `PlanFeatures::existential_extra_units` (`cost.rs`), +set only in the `PrintingCompose`-acquire branch's tier decision (`lib.rs`) exactly when `tier != 0` +comes ENTIRELY from the plane (`filter_nothing_to_verify && !nothing_to_verify`, `Mode::Card`, +`Prefer::Default`) -- the precise condition under which `push_card_matches`/`card_match_count` run the +early-exit walk with no separate residual call. Charged in `cost.rs`'s `GatheredScan`/`StreamedSelect` +arms as `existential_extra_units * GATHER_EXISTENTIAL_DEPTH_NS`, additive on top of the existing flat +charge (which already assumes depth 1; `existential_extra_units` is only the printings EXPECTED beyond +that one). + +**Deliberately NOT read off the existing `scan_units` feature**, after finding it already carries an +unrelated, pre-existing gap for exactly this population: `scan_units` floors to `domain_cards` (depth 1 +assumed) whenever `card_invariant_domain_exact` holds, which reads `composed_card_invariant` from +`filter.rs::touches_printing_field` -- and that function's `Legality { .. } => false` arm (documented, +and already flagged as an accepted one-directional gap in Round 16's own "three other classifiers" +section above) treats EVERY legality leaf as card-invariant, including a DIVERGENT format. Confirmed +live: bare `f:oldschool` measured `scan_units == eval_domain` (961 == 961, depth 1 assumed) against a +REAL depth of 6.28 (`printings_examined`/`cards_visited` = 6,037/961). So `existential_extra_units` was +computed fresh, from the same already-in-scope scalars `scan_all` itself uses (`printing_matches`, +`domain_cards`, `printings_per_card`), independent of `card_invariant_domain_exact` -- no new per-query +scan, same pre-computation shape as Round 1's own feature. + +### Why it doesn't hold up: Round 1's order-statistics model itself underestimates depth for this leaf family + +Even computed fresh and confound-free, the numbers don't support shipping this. Two problems, found by +looking directly at which rows the fresh feature actually produces a nonzero value for: + +**The order-statistics model (uniform-random position among a card's printings) is itself wrong for +existential categorical leaves whose position correlates with print era.** `border:borderless` (bare, +sampled 24 times): `existential_extra_units` = 192 (implying `expected_depth` ≈ 1.51) against a REAL +depth of 6.09 (`printings_examined`/`cards_visited` = 21,185/3,478) -- a 4x underestimate, even with the +`card_invariant_domain_exact` confound removed. The likely reason: `border:borderless`-style values +correlate with a specific print era, and printings are stored in a fixed prefer-desc order, so a card's +few matching printings cluster at one END of its print history rather than landing at a uniformly random +position -- exactly the assumption Round 1's model makes and exactly where a continuous, less era- +correlated field like `price_usd` would not violate it as badly. This is a wrong SHAPE, not a wrong rate: +no single multiplicative constant on top of `expected_depth` can fix an estimate whose underlying +distributional assumption is violated in a data-dependent way. + +**The sample has almost no distinct queries to calibrate against.** Of 6,318 broadly-sampled rows (fresh +build, same two-mode sampling as above), only 148 read `existential_extra_units > 0` at all, and of +those, one query (`border:borderless`, repeated by the sampler) accounts for 24 rows and a second +(`r>=special`) for another 6 -- there are not enough DISTINCT queries in reasonable sampling time to fit +or validate a new constant responsibly, even setting the shape problem aside. + +**Held-out calibration, run anyway, confirms both problems combined into a fit that shouldn't ship.** +Split by a hash of the query string (even/odd), calibration half n=77, held-out half n=71: + +``` +calibration half: fitted rate = 0.036 (statistically indistinguishable from 0, n=77 dominated by ~2 + distinct repeated queries) +held-out half: total abs error, NO fix: 1,597,558 + total abs error, fitted rate: 1,620,196 (WORSE, not better) + median ratio, NO fix: 0.593 median ratio, fitted rate: 0.595 (no change) +``` + +Applying the fitted correction to the held-out half made total absolute error slightly WORSE, not +better -- a clean, unambiguous "this doesn't hold up" signal, not a marginal call. + +### Outcome: discarded, reverted + +**Negative result, code reverted.** `cost.rs`/`lib.rs`/`tests.rs` are back to `costcell/trunk` (Round +16's state) -- `git diff --stat costcell/trunk` reads empty. `cargo test --release`: 172/172 passed +(unchanged from Round 16). `cargo clippy --all-targets -- -D warnings`: clean (unchanged, no code to +lint). No bench re-runs against a reverted build -- there is nothing to confirm. + +What this round DID establish, worth keeping for whoever picks this up next: + +- The flagship reproducer's gap is NOT a depth problem (real depth ≈1.009) -- ruling out this round's + hypothesized mechanism for that SPECIFIC query, correcting the framing this doc opened with. Its actual + driver is `eval_plane_expr_for_printing` costing more per call for a COMPOUND plane (arith range AND + existential leaf) than the `GATHER_CARD_PASS_NS`/`GATHER_RESIDUAL_FLOOR_NS` constants (fitted on a + single-`card_pass`-call shape) assume -- a plane-evaluation-cost gap, unfixed, a candidate for a future + round scoped to THAT mechanism specifically (not depth). +- Depth genuinely does drive under/over-costing elsewhere in the existential-plane population (broad + sample, thousands of rows, monotonic, corr ~0.4-0.5) -- real, but Round 1's uniform-random-position + `expected_depth` formula underestimates it badly for existential leaves whose matching position + correlates with print era (`border:borderless` real depth 6.09 against the model's 1.51). A real fix + needs a different distributional assumption for this leaf family, not a coefficient on the existing + one -- plausibly a per-(field, value) "typical position within a card's print history" statistic + computed once at store-build time (alongside `BorderPrintingPlanes`/`RarityPrintingPlanes`), not + per-query. Not attempted here; flagged as the concrete next step. +- A separate, smaller, already-partially-known gap resurfaced concretely: `filter.rs::touches_printing_ + field`'s documented `Legality { .. } => false` (card-invariant, unconditionally) silently zeroes the + `card_invariant_domain_exact` depth-1 shortcut's honesty for a DIVERGENT format specifically (bare + `f:oldschool` reads `scan_units == eval_domain` against a real depth of 6.28). Round 16's doc already + flagged `filter.rs`'s classifier as a documented, accepted one-directional approximation for VERIFY + ORDERING; this round found a second, concrete consumer (`card_invariant_domain_exact`'s "no depth-1 + fast path is needed" test) where the same approximation leaks into a materially wrong SCAN_UNITS + estimate, not just a suboptimal ordering. Not fixed here (out of this round's narrow scope), but worth + its own line item if `local-engine-cost-model-cleanup-remaining.md` or a similar tracking doc gets + revisited. + +## Round 19: the compound-plane mechanism confirmed twice over, an additive fix built, and discarded -- +## it regresses the metric that actually matters + +Round 17 flagged, but did not chase, the real driver of the flagship reproducer's remaining gap: +`eval_plane_expr_for_printing` costs more per call for a COMPOUND plane (a `cmc`/`power`/`toughness` +range ANDed with an existential leaf) than the flat `GATHER_CARD_PASS_NS`/`GATHER_RESIDUAL_FLOOR_NS` +constants assume, because they were fit against a single-bare-leaf-check shape. This round quantified +that mechanism precisely with two independent measurements, built a feature and a fix, and discarded it +after a held-out check on the metric that actually drives routing said no -- a second negative result in +this doc, and a useful one: the mechanism is real, but a simple additive linear term makes absolute +routing accuracy worse, not better. + +### The compiled tree shape is not what this doc's own brief assumed + +Before measuring anything, checked the ACTUAL compiled `PlaneExpr` for `cmc>=1 cmc<=5 border:black` +(`planes.rs::compile_plane`/`compile_numeric_cmp`) rather than trusting this doc's own prior paraphrase +("compiles to `Or([Plane(p1)...Plane(p5)])`, up to 5 lookups"). It does not: `cmc>=1` and `cmc<=5` are +two SEPARATE `NumericCmp` leaves, each compiling to its OWN `Or` over `numeric_layout`'s 13 one-hot +interior planes (plus the shared "hi" bucket), and nothing in `compile_plane`/`compile_plane_children` +intersects the two `Or`s into one narrower one. The real shape is: + +``` +And([Or(≤14 planes, from cmc>=1's own bound), Or(≤14 planes, from cmc<=5's own bound), Plane(border)]) +``` + +20 total `Plane` leaves for the exact flagship reproducer (verified by a node-counting walk over the +real compiled tree, `tests.rs::plane_expr_shape`), 16-28 depending on range width (verified across four +widths: `cmc=3` alone → 16, `cmc∈[2,4]` → 18, `cmc∈[1,5]` → 20, `cmc∈[0,12]` (the full interior range) → +28). This matters for anyone reusing this doc's mechanism description later: the leaf count scales with +BOTH bounds' own `Or` width, not with the intersected range's width. + +### Measurement 1: a confound-free kernel micro-benchmark confirms the mechanism directly + +Added `tests.rs::plane_eval_compound_leaf_cost` (`#[ignore]`d, real corpus via a `benchmarks` symlink +into the primary checkout -- read-only, nothing under `benchmarks/` touched): compiles a handful of real +`PlaneExpr` trees against the real corpus's `BitPlanes`/`OracleWordIndex`, finds one real witnessing +`(cid, printing)` pair for each, then calls `eval_plane_expr_for_printing` on that SAME fixed pair +directly in a tight best-of-80 loop (200,000 calls/round) -- no candidate walk, no page selection, no +`explain_analyze` overhead, so the number is purely the function's own per-call cost: + +``` +border alone: 4.12 ns/call rarity alone: 2.17 ns/call +cmc[1,5] AND border: 30.30 ns/call cmc[1,5] AND rarity: 33.26 ns/call + +width sweep (cmc range AND border:black, fresh witness per width): + cmc[3,3] (16 leaves, 15 extra): 22.30 ns/call delta +18.18 ns (1.21 ns/extra-leaf) + cmc[2,4] (18 leaves, 17 extra): 30.31 ns/call delta +26.19 ns (1.54 ns/extra-leaf) + cmc[1,5] (20 leaves, 19 extra): 32.70 ns/call delta +28.58 ns (1.50 ns/extra-leaf) + cmc[0,12] (28 leaves, 27 extra): 33.10 ns/call delta +28.98 ns (1.07 ns/extra-leaf) +``` + +Confirms, directly and mechanistically: a bare existential leaf costs 2-4ns/call; ANDing a card-invariant +range partner costs 7-8x more (30-33ns/call), scaling with the range's own leaf count at roughly +1.0-1.5ns per extra `Plane` leaf (not perfectly linear -- `Or`'s `.any()` short-circuits at the first +`true` child, so the REAL evaluated count for a specific witness depends on where its bit falls, which a +static per-query feature can't know -- but clearly monotonic and the right order of magnitude). + +### Measurement 2: a matched-eval_domain paired-diff on the real corpus, independently, agrees + +Sampled 77 real queries (`border`/`rarity` bare and ANDed with `cmc`/`power`/`toughness` ranges of width +1, 3, 5, and 13, plus `f:oldschool`) via `explain_analyze` against a freshly-built store (the checked-in +`real.store` predates this round's `PlanFeatures` field and reads header-mismatch; rebuilt via +`costbench.load_engine` against `benchmarks/bitplanes/corpus.jsonl` instead). Found a real, genuine +confound while doing this, unrelated to this round's own mechanism: `eval_domain` reads IDENTICAL across +every range width for several MINORITY existential leaf values (`border:white/gold/borderless`, any bare +`rarity` value paired with a `power`/`toughness` range) -- an already-existing gap in the +`card_invariant_domain_exact`/estimated-domain fallback, not something this round introduced or fixes. + +Controlling for it directly: comparing two rows for the SAME base leaf with the IDENTICAL `eval_domain` +isolates the leaf-count effect with zero contribution from that confound. Restricted to `eval_domain >= +4,000` (excluding the smallest populations, where single-query/single-trial-median noise swings the +per-pair rate by 10-60 ns/leaf) leaves 79 matched pairs across `border:black`/`r:common`/`r:uncommon`/ +`r:rare`/`r:mythic`/`f:oldschool`. Split by a hash of `"{leaf}|{plan}|{lo_leaves}|{hi_leaves}"`: + +``` +calibration half (n=42): median rate 2.92 ns/leaf +held-out half (n=37): median rate 2.64 ns/leaf -- within 10% of the calibration half +held-out mean abs error on the DELTA prediction: rate=0 baseline 178,670 ns -> fitted rate 101,370 ns + (43% lower) +``` + +Two independent measurements (a confound-free kernel micro-benchmark and a confound-controlled real- +corpus paired-diff) agree the mechanism is real and land in the same order of magnitude (1.0-1.5 vs +2.6-2.9 ns/leaf -- the real numbers read higher, plausibly because the whole-query walk touches a +DIFFERENT `printing` struct and different bitmap words per distinct candidate card, unlike the +micro-benchmark's artificially-hot repeated-same-leaf loop). + +### The fix built, and why it fails the metric that actually matters + +Built the fix per the brief's design: `planes.rs::count_plane_leaves` (a new, small, structural tree +walk), `PlanFeatures::plane_extra_eval_leaves` (`count_plane_leaves(plane) - 1`, computed once per query +in the `PrintingCompose`-acquire branch exactly when `cost_plane_nothing_to_verify` says the plane is +existential -- `0` otherwise, so the already-calibrated bare-existential-leaf population is untouched), +and `GATHER_PLANE_LEAF_NS`/wiring into both `GatheredScan` and `StreamedSelect`'s arms (the same +`eval_plane_expr_for_printing` call backs both kernels). Regression tests added and passing (172 existing ++ `compose_prices_compound_plane_leaf_count_above_bare_existential`, asserting the compound reproducer +gets a nonzero charge and both bare-existential-leaf and bare-card-invariant-range controls stay at +exactly `0`). `cargo test --release`: 173/173. `cargo clippy --all-targets -- -D warnings`: clean. + +Then ran the MANDATORY held-out check on this doc's own stated primary metric -- held-out +predicted-vs-measured, not the paired-delta above -- and it fails: + +``` +GatheredScan, eval_domain >= 4,000, plane_extra_eval_leaves > 0, hash-split by query string: + rate=0.00 (baseline): held-out median |log(predicted/measured)| = 0.417 + rate=2.80 (the calibrated mechanistic rate): held-out median |log ratio| = 1.145 -- WORSE, not better +``` + +The flagship reproducer itself shows why: BEFORE this round, `cmc>=1 cmc<=5 border:black`/card predicted +728,028ns against measured ~1.10-1.54M ns (under-charged, ratio ~0.5-0.65x, the gap this doc opened +with). Adding the calibrated 2.8ns/leaf term moves it to 2,043,877ns against the SAME measured range -- +now OVER-charged by ~1.5x. The additive fix does not close the gap, it overshoots past it. + +Root cause, isolated by computing the "rate `pred0` would need" per absolute row rather than per matched +pair, within the SAME leaf family (`border:black`, widths 15/17/19/27 extra leaves, at their own native +`eval_domain`): the implied rate is **negative** (-1.17, i.e. already OVER-predicted) at 15 extra leaves, +crosses to positive around 17-19, and only reaches +1.2 at 27. The existing `GATHER_CARD_PASS_NS`/ +`GATHER_RESIDUAL_FLOOR_NS` floor (18.89ns, fit against a "one bare check" shape per its own doc) is +measurably NOT a clean bare-leaf baseline in practice -- it already reads as generous at low compound-leaf +counts and stingy at high ones, most likely because whatever traffic sample calibrated it in an earlier +round already contained a mix of compound-AND shapes, baking an uneven, population-dependent AVERAGE +leaf-count contribution into one flat constant. Layering a mechanistically-correct marginal rate on top +of that uneven baseline overshoots exactly where the baseline was already over-generous, and only +partly helps where it was under-generous -- a single linear additive term cannot fix a floor that is +itself already leaf-count-dependent in an uncontrolled way. A real fix would need to jointly recalibrate +the floor and the new term together over a much larger, controlled sample -- a bigger blast radius than +this round's scope (Round 15-17's `GATHER_RESIDUAL_FLOOR_NS`/`GATHER_CARD_PASS_NS` are validated, +shipped constants; re-deriving them here risks a regression across the WHOLE existing residual-tier +population, not just the compound-plane slice this round targets). + +### Outcome: discarded, reverted + +**Negative result, code reverted.** `cost.rs`/`lib.rs`/`planes.rs`/`tests.rs` are back to `costcell/trunk` +(Round 17's state) -- `git diff --stat costcell/trunk` reads empty. `cargo test --release`: 172/172 +passed (unchanged). `cargo clippy --all-targets -- -D warnings`: clean (unchanged, no code to lint). No +bench re-runs against a reverted build -- there is nothing to confirm. + +What this round DID establish, worth keeping for whoever picks this up next: + +- The compound-plane mechanism is real, confirmed by two independent measurements (a confound-free + kernel micro-benchmark and a confound-controlled real-corpus matched-domain paired-diff), landing in + the same 1-3 ns/extra-leaf order of magnitude. This is no longer a hypothesis. +- The compiled tree shape for a two-sided numeric range ANDed with an existential leaf is + `And([Or(≤14), Or(≤14), existential leaf])` -- NOT the narrower `Or(width)` this doc's own earlier + round paraphrased. Anyone reusing this mechanism description should use THIS round's section above as + the reference shape (traced directly against `planes.rs::compile_numeric_cmp`/`compile_plane_children`, + and confirmed by a real node-count over the compiled tree), not the earlier paraphrase. +- A simple additive `rate * eval_domain * plane_extra_eval_leaves` term, layered on top of the existing + `GATHER_RESIDUAL_FLOOR_NS`/`GATHER_CARD_PASS_NS` floor, does NOT survive held-out validation on + predicted-vs-measured (the metric routing actually uses) even though the SAME rate is well-supported by + a held-out check on the marginal/paired-delta metric. The floor itself appears to already absorb an + uneven, population-dependent share of the compound-leaf effect. A future attempt should jointly refit + the floor and a leaf-count term together (a proper weighted regression over a much larger real-traffic + sample of existential-plane `Mode::Card` queries, not an additive patch on the existing constant), or + investigate whether the floor's ORIGINAL calibration sample already contained enough compound-AND + queries to explain the unevenness directly. +- A separate, small, already out-of-scope confound was found (not fixed): `eval_domain` reads identical + across every range width for several minority existential leaf values (rare `border`/`rarity` values + paired with a `power`/`toughness` range) -- a gap in the `card_invariant_domain_exact`/estimated-domain + fallback, distinct from (and a further complication on top of) the `filter.rs::touches_printing_field` + gap Round 17 already flagged. Worth its own line item in a future domain-estimation cleanup pass. + +## Round 20: a joint refit was attempted, and blocked by the SAME confound Round 19 flagged in +## passing -- now shown to be structurally inseparable from the population this fix needs, not just a +## small aside + +Round 19's own brief for a future attempt was explicit: jointly refit `GATHER_RESIDUAL_FLOOR_NS`, +`GATHER_CARD_PASS_NS` and a new leaf-count rate TOGETHER, against a sample broad enough to cover every +population that shares those two floor constants, rather than anchoring the floor at its old value and +fitting only the new term (Round 19's shape, which passed a marginal/paired-delta check but failed the +metric that actually drives routing). This round built that joint fit. It failed too, for a reason Round +19's own "Outcome" section already named as a loose end but did not chase: **the `eval_domain` confound +it flagged in passing ("reads identical across every range width for several minority existential leaf +values") is not a small aside on the side of this population -- once measured broadly, it dominates the +error for every existential leaf value except the one the flagship reproducer happens to use, and it +turns out to be structurally coupled to the exact acquire branch that produces the leaf-count feature.** +That coupling, not just the floor's own uneven fit, is why a broad sample cannot validate this fix with +today's architecture. + +### What's different this time + +Rounds 17 and 19 each built a plausible mechanism, fit it, and found the FIT didn't survive a held-out +check on the right metric. This round got as far as building the fit Round 19 asked for (feature +plumbing reintroduced verbatim, a scoped 3-constant joint-fit script, ~260 systematically varied rows +across all three named populations) -- and found the DATA itself is not trustworthy enough to validate +any fit against, for a reason specific to how this feature is computed. That is a different failure mode: +not "the mechanism doesn't hold up" but "the population needed to test the mechanism is dominated by an +orthogonal bug living in the same code path," discovered by trying to build the broad sample the brief +demanded rather than by reasoning about it in the abstract. + +### The feature and the fit, rebuilt + +Reintroduced Round 19's exact plumbing (its diagnosis was sound; only the fit around it was the problem): +`planes.rs::count_plane_leaves` (a plain node-counting walk over a compiled `PlaneExpr`, mirroring +`plane_expr_is_existential`'s own recursion shape), `PlanFeatures::plane_extra_eval_leaves` (`lib.rs`, +`count_plane_leaves(plane) - 1`, set only in the `PrintingCompose`-acquire branch's tier decision exactly +when `filter_nothing_to_verify && !cost_plane_nothing_to_verify` -- the plane, not a real filter residual, +is the reason `tier != 0`), and both `GatheredScan`/`StreamedSelect` arms in `cost.rs` reading +`GATHER_CARD_PASS_NS + tier_ns.max(GATHER_RESIDUAL_FLOOR_NS) + plane_extra_eval_leaves * GATHER_PLANE_LEAF_NS` +(`STREAM_*` counterparts identical in shape). `cargo test --release`: 173/173 (172 pre-existing + Round +19's own regression test format, re-added). `cargo clippy --all-targets -- -D warnings`: clean. + +`scripts/fit_cost_model.py` was read first, per the brief's instruction, and NOT used as-is: it refits +every coefficient in an arm at once (all 7-8 of `GatheredScan`'s), which is the general tool this +session's discipline exists to avoid reaching for on a single-mechanism round -- fitting it here would +have moved `GATHER_LOOP_PER_CARD_NS`, `GATHER_SCAN_PER_ROW_NS`, `GATHER_PUSH_PER_MATCH_NS`, and every +other already-validated rate in the arm as a side effect of trying to fit three constants. Built a +standalone script instead (`fit_round20.py`, this session's scratchpad, not `scripts/`): it holds every +OTHER coefficient in the `GatheredScan`/`StreamedSelect` arms at its CURRENT shipped value, computes +`other_terms = measured - (those coefficients * their features)`, and fits only +`[GATHER_CARD_PASS_NS, GATHER_RESIDUAL_FLOOR_NS, GATHER_PLANE_LEAF_NS]` (and the `STREAM_*` triple) +against what's left, using the identical non-negative log-ratio IRLS `fit_cost_model.py` itself uses +(copied, not imported, so this script's narrower scope can't accidentally widen if `fit_cost_model.py` +changes later). + +Sample: 3 numeric fields (`cmc`/`power`/`toughness`) x 7 range widths (0-12, giving `plane_extra_eval_ +leaves` from 0 to 48) x 11 existential leaf values (`border:{black,white,borderless,gold,silver}`, +`r:{common,uncommon,rare,mythic,special}`, `f:oldschool`) plus 12 triple-AND rows for population A +(compound); the same 11 bare leaves alone for population B; 26 hand-picked real-residual queries +(`name:`/`artist:`/`flavor:`/`watermark:`/anchored and unanchored `o:` regexes) spanning `MASK_COMPARE`/ +`SET_LOOKUP`/`TEXT_SCAN`/`REGEX_MACHINERY` tiers for population C -- 259 rows total, `unique=card, +orderby=rarity direction=desc limit=175 offset=0 prefer=default` (the reproducer's own paging shape, to +avoid contaminating the isolated term with the page/perm-walk terms' own separately-validated noise). +Confirmed first that natural sampling cannot substitute for this hand-built grid: 20,000 `QuerySampler` +draws in `realistic` mode and 40,000 in `uniform` mode produced **zero** rows with `plane_extra_eval_ +leaves > 0` -- this population is real (the flagship reproducer is a top-25 real query) but too rare for +either sampler mode to hit in tens of thousands of tries, matching this doc's own Q4 finding. + +Split calibration/held-out by `hashlib`-stable hash of the query string (Python's built-in `hash()` is +per-process randomized for strings and was caught giving a DIFFERENT split, and different fitted +coefficients, on two consecutive runs of the identical script before this was noticed and fixed -- +recorded here so whoever reuses `fit_round20.py`'s shape does not repeat it). + +### The confound: `eval_domain` badly overestimates the true candidate count for every leaf value except the corpus-dominant one, and it is the SAME branch that sets `plane_extra_eval_leaves` + +Checking each row's realized `cards_visited` (from `explain_analyze`) against the `eval_domain` feature +`plan_cost` actually multiplies -- the same `counter_check` discipline `fit_cost_model.py` itself insists +on before trusting a fit ("a feature that mis-counts by 2.5x cannot be repaired by any rate, and the fit +will happily bury the error in whichever coefficient correlates with it") -- only **13 of 223** population +A rows land within 15% of `cards_visited`. The other 210 are not scattered noise; they are one-sided and +huge, and they sort cleanly by which existential leaf value is in the query, independent of the numeric +range's width (i.e. independent of leaf count, the very thing this round's term is about): + +| leaf (ANDed with `cmc>=1 cmc<=N`, `N` swept 1-13) | `cards_visited / eval_domain`, across widths | +|---|---| +| `border:black` (near-universal -- most cards have a black-border printing) | 0.68 - 1.22 (clean) | +| `border:white` | 0.20 - 0.71 | +| `border:borderless` | 0.17 - 0.85 | +| `border:gold` | 0.07 - 0.69 | +| `r:common` / `r:uncommon` | 0.26 - 0.85 | +| `r:rare` | 0.17 - 0.68 | +| `r:mythic` | 0.02 - 0.55 | +| `f:oldschool` | 0.08 - 0.55 | + +`eval_domain` for `cmc>=1 cmc<=5 r:mythic` reads 4,710 -- IDENTICAL to `matches` (also 4,710, the exact +result total) -- while the REAL `GatheredScan` loop only ever visits 1,842 cards (`cards_visited`). The +model believes the candidate list is the full exact match count; the real loop, going through a narrower +candidate set some other mechanism built, visits well under half of that. This is not new -- Round 19's +own "Outcome" section already named it in one sentence ("`eval_domain` reads identical across every range +width for several minority existential leaf values... a gap in the `card_invariant_domain_exact`/ +estimated-domain fallback") -- but Round 19 measured it on a handful of rows found while chasing a +different mechanism. Measured broadly and systematically here (all 11 leaf values x 7 widths x 3 fields), +it is not a minority-case aside: **`border:black` is the only leaf value tested where `eval_domain` is +trustworthy**, and `border:black` is exactly the value this doc's own flagship reproducer and every +earlier round's hand-picked verification queries happened to use -- which is why nobody had measured this +gap's true size before this round went looking for a genuinely diverse sample. + +**The coupling that blocks a fix, not just a caveat on the sample:** `plane_extra_eval_leaves` is only +ever nonzero on rows that reach the `PrintingCompose`-acquire branch's tier decision -- and that is +*exactly* the same branch whose `domain_cards`/`card_invariant_domain_exact` machinery produces the +broken `eval_domain`. Checked directly: the identical queries under `orderby=name` (which routes through +a completely different acquire path, `Prep::Candidates`, not `PrintingCompose`) show `eval_domain` within +15% of `cards_visited` on 206 of 223 rows -- clean -- but `plane_extra_eval_leaves` reads exactly **0 on +all 259 rows**, because that acquire path never sets it (this round's plumbing, like Round 19's, is +deliberately scoped to the one branch where the mechanism applies). So there is no substitute sample: the +one acquire path that reports leaf count is the one with the broken domain estimate, and the one with a +correct domain estimate never reports leaf count. A joint fit of `GATHER_CARD_PASS_NS`/ +`GATHER_RESIDUAL_FLOOR_NS`/`GATHER_PLANE_LEAF_NS` against population A's absolute predicted-vs-measured is +therefore fitting mostly to `eval_domain` noise, not to the per-printing evaluation cost this round is +about -- restricting to the 13 clean rows leaves only `border:black` (plus two other near-universal +values that happen to floor `tier` to 0) and 5 distinct leaf-count values, nowhere near the "varying which +existential leaf" diversity the brief's own step 2 requires. + +Population B has an independent version of the same problem, previously flagged by Round 17 +(`scan_units`'s uniform-random-position assumption is wrong for a leaf value whose matching printing +clusters by print era): **0 of 10** bare-existential-leaf rows land within 15% on the `scan_units`/ +`printings_examined` counter check. Population C (real residual-filter queries -- `name:`/`artist:`/ +`flavor:`/`watermark:`/regex, none of which touch a plane) is comparatively clean: 24 of 26 rows pass the +`eval_domain` check. + +### The fit itself, run anyway, for the record + +Run on the full (unfiltered) sample, since the brief asks for the numbers even where the outcome is +negative: + +``` +GatheredScan (116 calibration / 143 held-out rows, hash-of-query split): + CARD_PASS current 3.00 fitted 2.90 + FLOOR current 18.89 fitted 14.85 + PLANE_LEAF_NS current 0.00 fitted 0.92 + +held-out, by population (median predicted/measured, within-25%): + A (compound) before 0.789 (20%) -> after 1.141 (18%) -- flips under- to over-predicted, no gain + B (bare leaf) before 1.960 (17%) -> after 1.688 (17%) -- moves toward 1.0 but within-25% unchanged + C (residual) before 1.268 (50%) -> after 1.140 (57%) -- the one population that actually improves + pooled before 0.874 (22%) -> after 1.146 (22%) -- pooled within-25% unchanged +``` + +`StreamedSelect` could not be fit at all: it never enters contention under `orderby=rarity`/`usd` for +any of these queries (`explain_analyze` reports only `PrintingCompose`/`GatheredScan`), so the calibration +set had zero rows. It DOES enter contention under `orderby=name` -- but that is exactly the acquire path +where `plane_extra_eval_leaves` is always 0 (see above), so even a `StreamedSelect`-only sample could not +inform `STREAM_PLANE_LEAF_NS`. `STREAM_CARD_PASS_NS`/`STREAM_RESIDUAL_FLOOR_NS` were left untouched +entirely -- not even a same-value no-op refit was attempted, since there was no leaf-count-varying data to +jointly fit them against. + +Population A's within-25% agreement does not improve (20% -> 18% held-out) and population B's does not +move (17% -> 17%) -- consistent with the diagnosis above: most of both populations' error is the +`eval_domain`/`scan_units` confounds, not the CARD_PASS/FLOOR/leaf-rate terms this fit can move. Only +population C, which does not touch a plane at all and is not subject to either confound, shows a genuine +improvement (50% -> 57% within-25%) -- but C alone has no leaf-count variation (`plane_extra_eval_leaves` +is 0 for every C row by construction) and so cannot inform `GATHER_PLANE_LEAF_NS` either. No population +in this sample can jointly validate all three constants at once with today's feature set. + +### Re-verifying the flagship reproducer + +Unchanged from `costcell/trunk` (Round 16's state), since nothing shipped: `cmc>=1 cmc<=5 border:black`, +`unique=card`, `orderby=rarity desc`, `limit=175`, `offset=0` -- `GatheredScan` predicted 728,028ns against +a freshly re-measured 1,161,583ns (single representative trial; the broader range across repeated runs +this session was 1,077,625-1,452,625ns), ratio ~0.5-0.68, same under-prediction this doc has reported +since Round 16. Neither Round 17's depth term, Round 19's additive leaf term, nor this round's jointly- +refit version closes this gap for a reason that generalizes across all three attempts: `border:black` is +the one leaf value where `eval_domain` is NOT the dominant source of error, so the reproducer's own +remaining gap really is the per-printing-evaluation-cost mechanism Rounds 17/19/20 all correctly +identified -- but fitting a rate against a BROADER sample (as this round's brief required, precisely to +avoid overfitting to this one reproducer) immediately runs into the `eval_domain` confound on every OTHER +leaf value, which a fit cannot tell apart from the mechanism it's trying to measure. + +### Outcome: discarded, reverted + +**Negative result, code reverted.** `cost.rs`/`lib.rs`/`planes.rs`/`tests.rs` are back to `costcell/trunk` +(Round 16's state) -- `git diff --stat costcell/trunk` reads empty. `cargo test --release`: 173/173 passed +(unchanged). `cargo clippy --all-targets -- -D warnings`: clean (unchanged, no code to lint). No bench +re-runs against a reverted build -- there is nothing to confirm. + +What this round DID establish, worth keeping for whoever picks this up next: + +- Round 19's own hypothesis (jointly refit the floor and the leaf rate, rather than anchoring the floor) + was the right next experiment to run, and it still doesn't ship -- but not for the reason Round 19 + anticipated ("the floor already unevenly absorbs part of the compound-leaf effect"). The blocking + problem is upstream of the floor entirely: `eval_domain`, the feature every candidate-count term in + `GatheredScan`'s/`StreamedSelect`'s arms multiplies by, is itself wrong by up to 14x (0.02-1.22 measured + across the sample) for the `PrintingCompose`-acquire branch's arith-tuple-range-AND-existential-leaf + shape, for every leaf value except the corpus-dominant one this doc's own reproducer happens to use. +- That confound is not merely correlated with this round's population by coincidence -- it is produced by + the SAME acquire branch that computes `plane_extra_eval_leaves`, and the one alternative acquire path + that has a trustworthy `eval_domain` (`orderby=name`, `Prep::Candidates`) never computes leaf count at + all. No sample built from the current architecture can jointly offer both a clean `eval_domain` and + leaf-count variation, which is a stronger and more specific claim than Round 19's one-line flag. +- **The real next step is fixing `eval_domain`/`domain_cards` for this shape FIRST** -- an arith-tuple + numeric range ANDed with a non-card-invariant existential leaf under `Mode::Card` -- in whichever of + `est.result.card`/`arith_tuple_count`/`compose_printing_estimate`/`card_invariant_domain_exact` actually + computes it (not traced to a single line here; out of this round's `cost.rs`/`lib.rs`-tier-decision-only + blast radius, and a large enough independent question -- domain estimation, not per-candidate cost model + rates -- to deserve its own doc rather than a fourth attempt bolted onto this one). Once `eval_domain` + is trustworthy across leaf values, a joint refit of `GATHER_CARD_PASS_NS`/`GATHER_RESIDUAL_FLOOR_NS`/a + leaf-count rate becomes testable for real, using the same sample construction and fitting script this + round built (`fit_round20.py`, this session's scratchpad -- not checked in, but the design/queries are + fully specified above for whoever rebuilds it). +- Population B's `scan_units` confound (Round 17, era-correlated print position) is confirmed independently + here at a larger scale (0 of 10 bare-leaf queries pass a 15% counter check) -- still not fixed, still + flagged as needing a per-(field, value) store-build-time statistic rather than a per-query estimate. +- Population C (real residual-filter queries, no plane involved) is the one population where the existing + floor is reasonably close (held-out within-25% 50%, before any refit) and where refitting helps (57% + after) -- consistent with this being closer to the population the floor's ORIGINAL calibration (see + `GATHER_RESIDUAL_FLOOR_NS`'s own doc, `MASK_COMPARE`/`SET_LOOKUP`/`TEXT_SCAN` tiers) actually targeted. + +## Round 25: the Blocker Is Cleared, the Refit Was Rebuilt and Retested on Clean Data -- a Third, +## More Precise Negative Result, Discarded + +Rounds 22/24 (`docs/issues/local-engine-domain-cards-existential-arith-and.md`) fixed the `eval_domain` +confound Round 20 found blocking this refit. This round rebuilt Round 19/20's plumbing verbatim +(`planes.rs::count_plane_leaves`, `PlanFeatures::plane_extra_eval_leaves`, additive +`GATHER_PLANE_LEAF_NS`/`STREAM_PLANE_LEAF_NS` terms in `cost.rs`), re-ran the joint fit against fresh +data measured on the now-fixed engine, and the fit still does not survive held-out validation -- +**demonstrated for the first time on genuinely clean `eval_domain` data**, closing the open question +both Round 20 and Round 22 left explicit ("clearing the `eval_domain` blocker does not by itself imply +the rate refit will now succeed"). It does not. + +### Step 1: plumbing rebuilt, verified as a pure no-op + +`count_plane_leaves` (`planes.rs`) is a plain node-counting walk over the compiled `PlaneExpr` tree +(`Plane`/`Bits` leaves count 1, `Const` counts 0, mirroring `plane_expr_is_existential`'s own +recursion). `PlanFeatures::plane_extra_eval_leaves` (`count_plane_leaves(plane) - 1`) is set in +`acquire_plan_features`'s `PrintingCompose`-acquire tier decision exactly when the PLANE half (not the +filter) is the reason `tier != 0` -- the same condition Round 19/20 used. Both `GatheredScan`'s and +`StreamedSelect`'s arms in `cost.rs` gained `+ plane_extra_eval_leaves * GATHER_PLANE_LEAF_NS` (resp. +`STREAM_PLANE_LEAF_NS`) inside the existing `tier_ns > 0.0` bracket, both new constants shipped at +`0.0` (a pure no-op) pending the fit. A new regression test, +`compose_prices_compound_plane_leaf_count_above_bare_existential`, asserts the feature is nonzero for +the compound reproducer, equals `count_plane_leaves(plane) - 1` exactly, and stays `0` for both a bare +existential leaf alone and a bare card-invariant range alone. `cargo test --release`: 177/177 (176 +pre-existing + 1 new). `cargo test` (debug): 178/178. `cargo clippy --all-targets -- -D warnings`: +clean. Verified the compiled-shape claim directly too: `cmc>=1 cmc<=5 border:black` reads +`plane_extra_eval_leaves == 19` (20 total `Plane` leaves, matching Round 19's own node-count exactly). + +**Also fixed a real gap the rebuild surfaced**: `acquire_facts_to_pydict` (`lib.rs`, the Python-visible +feature dump `explain()` returns) never exposed `plane_extra_eval_leaves` at all -- Round 19/20 must +have added this in their own now-reverted work, since `round20_samples.jsonl` (a leftover artifact in +the scratchpad) already carried the key, but the field is genuinely absent from `costcell/trunk` today. +Added one line exposing it, alongside the other `PlanFeatures` fields already dumped there. + +### Step 2: `eval_domain` re-verified clean for population A, `scan_units` confirmed a SEPARATE, still-real confound for population B + +Measured fresh rather than trusting Round 22/24's own numbers unchecked (243 population-A rows, 3 +arith fields x 9 widths x 9 "clean" leaf values, `explain_analyze` against a freshly-built store): + +``` +population A: cards_visited / eval_domain within 15% = 98.4% median = 1.000 +population B: printings_examined / scan_units within 15% = 0.0% median = 3.950 +population C: printings_examined / scan_units within 15% = 29.6% median = 0.766 +``` + +`eval_domain` is confirmed clean for population A -- Rounds 22/24's fix holds broadly, not just on +their own 429-row sweep. **`scan_units` (a DIFFERENT feature, the printing-SPAN estimate +`GATHER_SCAN_PER_ROW_NS` multiplies, computed by `scan_all`/`card_invariant_domain_exact` -- untouched +by Rounds 15-24's `And`-arm work) is a real, severe, SEPARATE confound**, confirmed at a fresh, larger +sample: population B (bare existential leaf, no arith partner) under-predicts real printings scanned by +a median ~4x, exactly the era-correlated-print-position mechanism Round 17 first flagged and Round 20 +measured on a 10-row sample. Population C shows a milder version of the same gap (median 0.766). + +**Accounted for explicitly rather than left to pollute the fit**: the refit script +(`fit_round25.py`, this session's scratchpad) computes each calibration row's fit TARGET using the +REALIZED `printings_examined` counter in place of the `scan_units` feature -- the same counter-check +substitution `fit_cost_model.py` itself uses, so `scan_units`'s own (unrelated, unfixed) error cannot +be silently absorbed by the CARD_PASS/FLOOR/LEAF coefficients this round is fitting. HELD-OUT +agreement, reported separately below, still uses the REAL deployed formula (raw `scan_units`, exactly +what `cost.rs::plan_cost` computes) -- the substitution is for fitting honestly, not for reporting an +artificially rosier accuracy number. No population was excluded from calibration on this basis; the +substitution made exclusion unnecessary. + +### Step 3-5: the joint fit, run against clean data, per-population held-out validation + +Sample: 297 rows total (population A 243, B 27, C 27 -- same 9 "clean" leaf values Round 22/24 +validated, same `unique=card orderby=rarity direction=desc limit=175 offset=0 prefer=default` paging +shape as the flagship reproducer, population B varied over 3 page sizes instead for hash-split +diversity, population C the same real name:/artist:/flavor:/watermark:/o: shape as Round 20's). +Calibration/held-out split by a STABLE hash of the query string (`hashlib.sha256`, not Python's +built-in `hash()` -- Round 20's own script used the un-fixed builtin despite flagging the exact bug +in its own comment; this round's script fixes it for real, confirmed by re-running the fit twice and +getting IDENTICAL splits and coefficients both times, unlike the builtin-hash version, which gave +visibly different fitted coefficients across two runs before the fix). + +`StreamedSelect`: **0 distinct design rows from 0 calibration rows** -- confirms Round 20's own +finding again: `StreamedSelect` never enters contention on this acquire's own paging shape, so +`STREAM_PLANE_LEAF_NS` cannot be informed by this sample at all, same as before. + +`GatheredScan`, fitted coefficients (IRLS on the log ratio, ridge-anchored to shipped values, same +method as Round 20's script): + +``` +CARD_PASS current=3.000 fitted=2.708 +FLOOR current=18.890 fitted=7.310 +PLANE_LEAF_NS current=0.000 fitted=4.346 (order of magnitude agrees with Round 19's own two + independent measurements: 1.0-1.5 ns/leaf kernel + micro-benchmark, 2.6-2.9 ns/leaf real-corpus paired-diff) +``` + +HELD-OUT agreement (real deployed formula, raw `scan_units`), per population: + +``` + median pred/meas within-25% median |log ratio| +A before 0.333 17% 1.099 +A after 0.983 15% 0.838 +B before 1.745 17% 0.606 +B after 1.081 6% 0.626 +C before 1.464 9% 0.395 +C after 1.037 36% 0.391 +pooled before 0.600 16% 0.824 +pooled after 1.006 15% 0.736 +``` + +The MEDIAN and log-ratio numbers read like a real improvement across the board -- median moves to +within 2% of 1.0 for both A and pooled. **This is misleading on its own**, and checking why is what +makes this round's negative result more precise than Round 19's: + +### Why it still fails: the fit improves small-`eval_domain` leaves and badly overshoots large ones, and the flagship reproducer is exactly the latter + +Breaking population A's held-out rows down by which existential leaf value they use (the same +breakdown Round 19 used to first spot this shape): + +``` +leaf improved / total (held-out, GatheredScan, by |log ratio| before vs after) +border:black 0 / 12 +r:common 0 / 14 +r:rare 0 / 17 +r:uncommon 0 / 13 +border:borderless 15 / 17 +border:gold 13 / 15 +border:white 7 / 10 +f:oldschool 11 / 12 +r:mythic 12 / 13 +``` + +A clean split: the four leaves with the LARGEST bare selectivity in this corpus (`border:black` 98.9%, +`r:common`/`r:uncommon`/`r:rare` 32-35%) get WORSE on every single held-out row: their `eval_domain` is +tens of thousands, and the fitted `PLANE_LEAF_NS` rate, multiplied by both `eval_domain` and the +range's own leaf count, overshoots the true cost by 2-3x for exactly these rows. The five +minority-selectivity leaves (a few hundred to a few thousand `eval_domain`) improve substantially, +because the SAME flat rate is proportionally much smaller against their much smaller `eval_domain`. +Population A's TOTAL absolute ns error is **2.67x WORSE** after the fit (12,676,427ns -> 33,865,296ns +summed over 123 held-out rows), dominated by the large-`eval_domain` leaves' now-larger absolute +errors outweighing the small-`eval_domain` leaves' improvement. + +**The flagship reproducer is exactly the worst-case leaf** (`border:black`, the corpus's single most +selective existential value): re-measured directly (`eval_domain=24,734`, `plane_extra_eval_leaves=19`, +matching Round 19's own node-count exactly for this query's compiled shape): + +``` +cmc>=1 cmc<=5 border:black, unique=card, orderby=rarity desc, limit=175, offset=0: + measured range: 1,054,584 - 1,096,417 ns + before (shipped): 728,547 ns (under by ~1.4-1.5x -- the gap this whole doc opened with) + after (fitted): 2,477,459 ns (OVER by ~2.3x -- the fit makes the flagship query's own + prediction WORSE, not better) +``` + +This is the same shape of failure Round 19 found ("the additive fix does not close the gap, it +overshoots past it"), but the mechanism is now precise where Round 19's was not: Round 19 attributed +it to "the floor already unevenly absorbs part of the compound-leaf effect" from an unknown-shaped +population-dependent average baked into `GATHER_RESIDUAL_FLOOR_NS`. This round's clean data shows the +unevenness correlates cleanly and monotonically with `eval_domain`'s own SIZE (a 50-100x range across +this population's leaf values) -- not a vague "population mix" but a specific, checkable quantity. A +flat `ns/leaf` rate multiplied by `eval_domain` is structurally the wrong shape for a population whose +`eval_domain` spans two orders of magnitude: it can be right at one scale or the other, never both. + +Population B (bare leaf, `scan_units` confound already accounted for via the counter-check +substitution) shows a smaller but real absolute improvement (abs error ratio 0.480, roughly halved) -- +consistent with `plane_extra_eval_leaves` being `0` for every population-B row by construction (a +single leaf), so `PLANE_LEAF_NS` contributes nothing there and the improvement is coming entirely from +`FLOOR` moving down (18.89 -> 7.31), which happens to fit population B's own residual-tier cost better. +Population C shows abs error ratio 1.167 (essentially flat, mildly worse) -- smaller sample (11 +held-out rows) and no `plane_extra_eval_leaves` signal either, so this population mostly just absorbs +whatever `FLOOR`/`CARD_PASS` move does, which is a wash here. + +### Why `GATHER_CARD_PASS_NS`/`GATHER_RESIDUAL_FLOOR_NS` themselves cannot be safely moved by this fit alone, either + +Even setting `PLANE_LEAF_NS` aside, the fit also moves `FLOOR` from 18.89 to 7.31 and `CARD_PASS` from +3.00 to 2.71 -- outside this round's sample's own information: these two constants are shared across +the WHOLE residual-tier population (every `GatheredScan`/`StreamedSelect` query with a nonzero verify +tier, not just existential-plane queries), and Round 19 already flagged moving them from a narrow +3-population sample as a risk to the broader population that originally calibrated them. This round's +sample cannot see that broader population at all, so shipping the fitted `FLOOR`/`CARD_PASS` values +alongside a `0.0` `PLANE_LEAF_NS` (i.e., taking only "half" the fit) was considered and rejected for +the same reason -- it is not a smaller, safer version of this round's finding, it is an untested +change to constants this round has no evidence about outside this narrow slice. + +### Outcome: discarded, reverted + +**Negative result, code reverted.** `cost.rs`/`lib.rs`/`planes.rs`/`tests.rs` are back to +`costcell/trunk` -- `git diff --stat costcell/trunk` reads empty. `cargo test --release`: 176/176 +passed (unchanged). `cargo test` (debug): 177/177 (unchanged). `cargo clippy --all-targets -- -D +warnings`: clean (unchanged, no code to lint). No bench re-runs against a reverted build -- there is +nothing to confirm; the plumbing was validated as a pure no-op (176/177 before the rebuild, 177/178 +with it, identical predicted_ns everywhere with the new constants at `0.0`) before this round decided +not to ship the nonzero rate. + +What this round DID establish, worth keeping for whoever picks this up next: + +- **The `eval_domain` blocker Rounds 22/24 fixed is real and confirmed cleared** (98.4% within 15% on a + fresh, independent 243-row check) -- clearing it was necessary but, as Round 22 itself already + hedged, not sufficient. This round is the first to actually test that gap and confirm the refit still + fails on clean data, closing the open question. +- **A NEW, more precise characterization of why a flat additive leaf-rate term fails**: it is not (only) + "the floor absorbs an uneven population mix" (Round 19's framing) -- it is that `eval_domain` itself + spans 50-100x across this population's leaf values, and a rate multiplied by `eval_domain` cannot be + right at both ends. The four highest-selectivity leaves (`border:black`, `r:common/uncommon/rare` -- + not coincidentally, the corpus's most COMMON real-traffic shapes) get WORSE on 100% of held-out rows; + the five lower-selectivity leaves improve substantially. The flagship reproducer uses the single + worst leaf for this mechanism (`border:black`), which is why its own prediction moves from + under-charged (0.65x) to badly over-charged (2.3x) rather than landing closer to 1.0. +- **A genuinely NEW confound was found and handled cleanly**: `scan_units` (not `eval_domain`) is a + real, severe, still-unfixed error source for the bare-existential-leaf population specifically (median + 4x under-prediction) -- confirmed at a fresh, larger sample than Round 17/20's. This round's fit + script accounted for it via a counter-check substitution (fit against realized `printings_examined`, + report held-out against the real deployed `scan_units` feature) rather than letting it silently + corrupt the CARD_PASS/FLOOR/LEAF fit the way `eval_domain`'s confound corrupted Round 20's attempt -- + a concrete methodological answer to this round's own brief, reusable by whoever revisits this next. +- **A real next step, concretely scoped**: a SATURATING or bucketed leaf-rate term (e.g., a rate that + caps its total per-candidate contribution, or a small number of `eval_domain`-selectivity BANDS each + with their own calibrated rate) rather than a flat linear one, would not have this specific failure + mode -- large-`eval_domain` leaves would stop scaling the per-leaf charge past whatever cap or band + applies to them. Not attempted here (a materially different, larger mechanism than "rebuild Round + 19/20's plumbing and refit," and this round's blast radius/brief scoped the attempt to that + plumbing specifically) -- flagged as the concrete next step for whoever picks this arc up again. +- `scripts/fit_cost_model.py` was read again as reference only, not used, for the same reason Round 20 + gave (it refits every coefficient in an arm at once, too wide a blast radius for a 3-constant scoped + fit). `fit_round25.py` (this session's scratchpad, not checked in) rebuilds Round 20's design with the + `hashlib` split fix and the `scan_units` counter-check substitution described above. diff --git a/docs/issues/local-engine-card-printing-property-classifier-unification.md b/docs/issues/local-engine-card-printing-property-classifier-unification.md new file mode 100644 index 000000000..a109595de --- /dev/null +++ b/docs/issues/local-engine-card-printing-property-classifier-unification.md @@ -0,0 +1,286 @@ +# Card-vs-Printing Property Classifiers — Is One Canonical Table Achievable? + +## Context + +[docs/issues/done/local-engine-gathered-scan-undercosted-arith-existential-and.md](done/local-engine-gathered-scan-undercosted-arith-existential-and.md) +(Rounds 15-16) fixed the router's `cost_plane_nothing_to_verify` (`card_engine/src/lib.rs:9604`) by +deleting a field-identity check (`plane_touches_rarity_or_border`) and routing through +`planes.rs::plane_expr_is_existential` instead — a predicate keyed on *behavior* ("does this plane's +existential semantics force per-printing re-verification under `Mode::Card`") via the family-keyed +`PLANE_BLOCKS`/`ExistentialLeaf`/`needs_printing_verification` table. While confirming no other +router call site needed the same fix, Round 16 found three other classifiers in the crate that +independently reason about roughly the same card-vs-printing-level distinction, each for a different +purpose, each already documenting its own one-line stance on why it disagrees with the canonical +table on legality — but did not chase whether those three (four, including the canonical table +itself) could collapse into one shared table. This doc does that chase. + +## The four classifiers, side by side + +| # | Function | File:line | Operates over | Composition | Used for | +|---|---|---|---|---|---| +| A | `plane_expr_is_existential` / `needs_printing_verification` / `existential_leaf` / `PLANE_BLOCKS` | `planes.rs:1159,1171,1231,1119` | compiled `PlaneExpr` (post-`compile_plane`) | recursive And/Or/Not walk | Router cost-tier (`cost_plane_nothing_to_verify`, `lib.rs:9604`) and the executor's per-printing row-selection walk (`existential_plane_for`) — the SAME fact drives both, traced end-to-end in Round 16 | +| B | `has_printing_varying_leaf` | `estimator.rs:79` | raw `FilterExpr` | **ANY** (`.any`) | Standalone sound cardinality estimator's AND-lower-bound Bonferroni gate and NOT-branch selection (`estimate_rec`, `compose_and`, `estimator.rs:154-220`) | +| C | `printing_dependent` / `leaf_compares_printing_field` | `filter.rs:820,851` | raw `FilterExpr` | **ALL** (`.all`) | Verify-order heuristic — which And/Or child to evaluate first (`and_child_key`/`or_child_key`, `filter.rs:929`, `1269`) | +| D | `is_broadcast_leaf_shape` / `is_broadcast_composable` | `lib.rs:6781,6798` | raw `FilterExpr` | n/a (leaf-shape allow-list, not a tree walk) | Gates which leaf shapes `PrintingCompose`'s broadcast-card-bits-to-printings build arm supports (`is_printing_composable`, `lib.rs:6867`) | + +Classifier D turns out **not to be a fourth independent opinion** — see "Proposed unification" below; +it already reads table A directly. There is also a close cousin of C worth naming up front: +**`touches_printing_field`** (`filter.rs:840`) shares C's exact per-leaf table +(`leaf_compares_printing_field`) but composes with `.any` instead of `.all`, feeding the router's +"is the residual card-invariant" checks (`lib.rs:11873`, `12079`, `12568`). `printing_dependent` (ALL) +and `touches_printing_field` (ANY) are two adapters already reading **one** shared per-leaf table +inside `filter.rs` — this is the ANY/ALL-composition-wrapper pattern the user's framing asks for, +already built, just not yet extended to cover B or A. + +### Each classifier's own stated reasoning for special-casing + +- **A** (`planes.rs:1132-1158`, `needs_printing_verification`'s doc): "For rarity and border that is + every leaf — those are printing-varying by nature. For legality it is a per-FORMAT question... a + legality plane outside that mask is card-invariant." Legality is the one field whose classification + is *dynamic*, resolved per-store via `divergent_formats`. +- **B** (`estimator.rs:71-78`, `has_printing_varying_leaf`'s doc): "`Legality` is treated as varying + here (conservative): divergent reprints genuinely vary per-printing (#667), even though + `printing_dependent` ranks it invariant for its own (common-case) reason." +- **C** (`filter.rs:891-892`, the `Legality` arm's comment): "Divergent-legality cards defer to the + printing, but they are a rare exception (non-tournament reprints); rank by the common card-level + case." +- **D** (`lib.rs:6772-6780`, `is_broadcast_leaf_shape`'s doc): "Deliberately narrower than 'anything + `compile_plane` handles': rarity/legality/border also compile via `compile_plane`, but they are + EXISTENTIAL card-space facts (∃p: ...), and mixing them into a card-invariant broadcast here would + reintroduce the #667/#680 shared-witness bug." D's exclusion list is driven by calling A directly + (`is_broadcast_composable` calls `plane_expr_is_existential`), not by re-deriving the fact. + +## The cross-reference table + +Verdict key: **C**ard-level, **P**rinting-level, **Dyn** (legality: depends on `divergent_formats` +per format), **N/A** (classifier's domain doesn't include this field at all — see notes). + +| Field | A (planes.rs) | B (`has_printing_varying_leaf`) | C (`printing_dependent`) | D (`is_broadcast_leaf_shape`) | +|---|---|---|---|---| +| `cmc` / `power` / `toughness` | C (compiles via `compile_numeric_cmp`, never in `PLANE_BLOCKS`) | C | C | **P is included** (`NumericCmp` arm, `lib.rs:6784`) | +| `color` / `color_identity` / `produced_mana` | C | C | C | **Included** (`ColorCmp` arm) | +| `devotion` | C (compiles, never existential) | C | C | **Included** (`Devotion` arm) | +| `type` (`TypeCmp`) | C (compiles via `compile_plane`, never existential) | C | C | **N/A — not in D's arm list at all** (see below) | +| `rarity` (all values incl. "hi"/special/bonus bucket) | **P**, unconditional | P | C | N/A (D explicitly excludes; has its own native compose arm) | +| `border` (all values incl. "other" bucket) | **P**, unconditional | P | P | N/A (same as rarity) | +| `legality` (any format) | **Dyn** — P iff that format's bit is set in `divergent_formats`, else C | **P, always** (conservative, ignores mask) | **C, always** (common-case, ignores mask) | N/A (native compose arm, reads A directly via `status_plane_bases`) | +| `mana_cost` (`ManaCostCmp`) | N/A (never compiles to a plane) | C | C | N/A (not in D's arm list) | +| `rarity_int` as a NumericCmp *inequality* (not the plane path) | N/A for A (A only sees the compiled plane form; `rarity>=rare` goes through `compile_rarity_cmp`, still lands in `PLANE_BLOCKS`'s rarity blocks) | P | P | N/A | +| `collector_number` | N/A (never a plane) | P | P | N/A | +| `price` (usd/eur/tix) | N/A | P | P | N/A | +| `date` / `year` (`released_at`) | N/A | P | P | N/A | +| `set_code` | N/A | P (`TextExact{SetCode}`/`TextRegex{SetCode}` arm, `estimator.rs:100-103`) | P | N/A | +| `watermark` | N/A | P (same arm) | P | N/A | +| `border` as a bare `TextExact` (not the NumericCmp/rarity path) | N/A for the raw `FilterExpr` (only reachable once compiled) | P (`TextExact{Border}` arm) | P | N/A | +| `artist` | N/A | P | P | N/A | +| `flavor_text` | N/A | P | P | N/A | +| `oracle_text` / `name` (contains/exact) | N/A | C | C | N/A | +| collections: `subtypes`/`keywords`/`otag` | N/A | C | C | N/A | +| collections: `art_tags`/`is_tags`/`frame_data` | N/A | P | P | N/A | +| `loyalty` / `edhrec_rank` | N/A | C | C | N/A | +| `prefer_score` | N/A | P | P | N/A | + +### No further disagreements found + +Re-verified every row above directly against the two files' live source (`estimator.rs:79-121`, +`filter.rs:851-910`) rather than trusting a first-pass transcription. `prefer_score` +(`NumField::PreferScore`) is in B's `num_varying` list (`estimator.rs:90`, printing-varying) and in +C's `true` block (`filter.rs:864`, printing-varying) — both agree. `set_code`/`watermark` are in B's +`TextExact`/`TextRegex` arm (`estimator.rs:100-103`, printing-varying) and C's identical-looking arm +(`filter.rs:880-885`) — both agree; B and C's `TextExact`/`TextRegex`/`CollectionCmp`/`NumericCmp` +field lists are, in fact, byte-for-byte the same set of fields with the same verdict everywhere +except `Legality`. **Legality is the ONLY disagreement between B and C** — every other field in both +files' leaf tables already matches. This is worth stating plainly since it changes the shape of the +unification story: B and C are not two independently-drifting classifiers that happen to agree most +of the time — they are two copies of what is effectively already one table, differing by exactly one +documented, deliberate row. Unifying them mainly removes the duplication risk (two copies that must be +kept in sync by hand across two files) rather than resolving live disagreement. + +## The legality disagreement, resolved + +**B** (`has_printing_varying_leaf`) always classifies `Legality` as printing-varying: +> "`Legality` is treated as varying here (conservative): divergent reprints genuinely vary +> per-printing (#667), even though `printing_dependent` ranks it invariant for its own (common-case) +> reason." (`estimator.rs:76-78`) + +**C** (`leaf_compares_printing_field`, read by both `printing_dependent` and `touches_printing_field`) +always classifies `Legality` as card-level: +> "Divergent-legality cards defer to the printing, but they are a rare exception (non-tournament +> reprints); rank by the common card-level case." (`filter.rs:891-892`) + +**Verdict: deliberate, not a bug — confirmed by tracing consequences, not just reading the comments.** + +- **B's over-approximation stays sound.** `has_printing_varying_leaf` feeds two places in + `estimator.rs`: `compose_and`'s Bonferroni-lower-bound gate (`varying <= 1`, `estimator.rs:211-217`) + and `estimate_rec`'s `Not` branch selection (`estimator.rs:159-179`). In both, treating an + actually-card-invariant legality format as printing-varying can only make the bound **looser**, never + wrong: `compose_and`'s `varying > 1` branch forces `lo = 0`, which is trivially sound regardless of + whether the extra "varying" child really is; `estimate_rec`'s printing-varying `Not` branch uses + `finalize(0, ..., n, n)` — a hi of `n` (the loosest possible), never a hi that could be violated by + the true, tighter answer. The estimator's own hard invariant ("SOUNDNESS is the hard invariant; + tightness/cheapness are secondary", `estimator.rs:13`) is exactly what this preserves, and + `fuzz_row_identity_matches_reference` (`tests.rs:2712`) exercises `estimate_cardinality`'s bound + against thousands of random filter trees per run, including ones containing `Legality` leaves, + asserting `lo <= true_count <= hi` every time (`tests.rs:2610-2617`) — this would fail loudly if the + conservative direction were ever wrong, and it passes. +- **C's under-approximation only affects performance, never correctness.** `printing_dependent`/ + `touches_printing_field` only steer which child of an And/Or a verifier tries first + (`and_child_key`/`or_child_key`) and which residual is deemed "card-invariant" for a router + fast-path decision. Misclassifying a divergent-format legality leaf as card-level means it might get + tried first when a genuinely card-settling sibling would have been cheaper to check — a suboptimal + ORDER, never a wrong verification result, because whichever child runs still evaluates the real + three-valued `tri()` logic regardless of what order it ran in. +- **Both are correctly scoped to their OWN purpose's error-cost asymmetry.** B must never let a + looseness bias become an unsoundness — being wrong toward "more printing-varying than reality" is + free. C must never let a looseness bias become a WRONG ANSWER — being wrong toward "assume the common + case, order accordingly" is at most a slow path, and the actual value returned never depends on + `leaf_compares_printing_field`'s answer. These are opposite biases because the two functions have + opposite failure costs, not because one of them is right and the other wrong. + +This is the same conclusion Round 16 stated in passing ("both are DOCUMENTED, ONE-DIRECTION +approximations... not a place where real per-printing work gets silently priced as free") — this doc +traces the actual consequence chain for each (which downstream branch reads the value, and what bound/ +behavior it produces) rather than taking that framing on faith. + +## Proposed unification + +### D is not a fourth table — it already reads A + +`is_broadcast_leaf_shape` is a **leaf-shape allow-list**, not a card/printing classifier: it decides +which specific `FilterExpr` SHAPES `PrintingCompose`'s broadcast-build arm has been wired up to accept +(`ColorCmp`, `Devotion`, `NumericCmp` on `cmc`/`power`/`toughness`, and `Not` of those). The actual +card-vs-printing check is `is_broadcast_composable`'s direct call to `plane_expr_is_existential` +(table A) — confirmed by reading `lib.rs:6798-6801`. Where D and A's *field lists* look like they +disagree (`TypeCmp` is card-level per A/B/C, but absent from D's arm list entirely), that is **not** a +classification disagreement — it's an unbuilt compose arm. `TypeCmp` compiles to a non-existential +plane per A (confirmed: `compile_plane`'s `TypeCmp` arm, `planes.rs:1317-1322`, never touches +`PLANE_BLOCKS`), so `is_broadcast_composable(TypeCmp, ..)` would return `true` if it were ever called +— but `is_broadcast_leaf_shape` never routes a `TypeCmp` there, and `is_printing_composable` +(`lib.rs:6818`) has no other arm for `TypeCmp` either, so **`t:goblin`-shaped leaves cannot reach +`PrintingCompose`'s composed-bits path at all today**, card-invariant or not. This is a separate, +narrower gap (missing engineering, not a wrong classification) — worth its own future doc if the win +is real, but out of scope here; D itself needs no unification work, since it already delegates to A. + +### A single canonical table is achievable at the DATA level, not at the function-call level + +The four (three, net of D) classifiers don't operate over the same tree shape: A walks a compiled +`PlaneExpr` (post `compile_plane`, which has already thrown away original field identity in favor of +plane indices, folded De Morgan through `Not`, and only ever contains the ~9 plane-eligible field +families); B and C walk the raw `FilterExpr` (the full ~25-variant leaf universe, most of which never +reaches a plane at all — price, dates, artist, flavor, set/watermark, non-Ge collections). There is no +single function both a `PlaneExpr` walker and a `FilterExpr` walker could call without also merging +those two representations, which is a far larger change than this doc's scope. + +What **is** achievable: one canonical **data** table, keyed by logical field identity (the union of +`NumField`/`TextField`/`TextSearchField`/`CollField`/`ColorField` variants, plus `Legality`/`Devotion`/ +`ManaCostCmp` as their own rows), each row holding a `Locality` value: + +```rust +enum Locality { + CardLevel, + PrintingLevel, + /// Legality's shape: printing-level iff this format's divergent-formats bit is set. + /// Callers that need a single static answer (B's/C's per-purpose bias) must say which + /// they want explicitly — see below — never read a silent default. + DivergentByFormat, +} +``` + +`PLANE_BLOCKS` (table A) is already this shape for the 9 plane-eligible families — legality (dynamic, +exactly `DivergentByFormat`), rarity and border (both unconditionally `PrintingLevel`) — and would not +need to change at all; it would become the reference implementation the new field-level table is +checked against (or literally the source A's plane-index blocks are derived from, if someone wants to +also collapse the "which plane index" concern into the same table — not necessary for this +unification, since A's `BlockKind`-to-plane-index mapping is a separate, already-correct concern). + +B's and C's field lists (`estimator.rs:79-121`, `filter.rs:851-910`) would become **thin adapters** +over the new table: +- Both are already per-leaf, so the per-leaf lookup is a direct table read for every field except + `Legality`. +- For `Legality`, each caller passes an explicit policy at the call site — `Locality::PrintingLevel` + for B (matching its documented "conservative" stance) or `Locality::CardLevel` for C (matching its + documented "common-case" stance) — rather than the table silently picking one. This makes the + existing one-line comments in each file into an explicit, typed argument instead of a hardcoded + match arm, which is the concrete form of "it's OK to keep this as a COMMENT, but the branch itself + must not key on field identity": the branch becomes "is this field `PrintingLevel`, or `CardLevel`, + or (`DivergentByFormat` AND caller-policy-says-printing-level)" — never "is this field literally + `Legality`". +- `printing_dependent` (ALL) and `touches_printing_field` (ANY) already share one leaf table + (`leaf_compares_printing_field`) with two composition wrappers — this is the **existing precedent** + for the ANY/ALL adapter the user's framing calls for; extending it to also cover B (a second ANY + consumer, just with the opposite legality policy and a slightly different field-family list) is the + same shape of change, not a new pattern. +- A itself needs zero changes — its callers (the router, the executor) already consume the + behaviorally-correct fact directly and should keep doing so. + +### Suggested migration order (independently landable, by risk and testability) + +1. **`filter.rs`'s C first.** Lowest risk: `printing_dependent`/`touches_printing_field` only affect + verify/paging ORDER, never a returned answer (see "risks" below) — a misclassification here is + caught by nothing worse than a slower path, and the existing + `verify_order_and_defers_printing_dependent_children` / `verify_order_or_defers_printing_dependent_ + children` tests (`tests.rs:11084`, `11106`) already assert the ordering outcome for representative + shapes, giving a regression harness for free. Migrating C to read the shared table (with an explicit + `Locality::CardLevel` policy for legality, matching its current behavior) should be a pure refactor + with zero behavior change, verifiable by running those two tests plus a differential run of + `bench_pairwise_ordering.py`/`bench_query_latency_ab.py` to confirm no ordering regression. +2. **`estimator.rs`'s B second.** Higher stakes (a SOUNDNESS invariant, not just an ordering + heuristic) but well-covered: `fuzz_row_identity_matches_reference` already asserts + `estimate_cardinality`'s bound against thousands of random trees per run, across every seed and the + 6k-card corpus fixture, and would catch a soundness regression from a botched migration + immediately (the harness is EXISTING, not new work needed first). Migrate with the explicit + `Locality::PrintingLevel` policy for legality (matching current behavior) and re-run the fuzz suite + before and after as the gate. +3. **`planes.rs`'s A last, if at all — and maybe not.** A is already the reference table other rows + would be checked against, is already correctness-critical for the router AND the executor (traced + end-to-end in Round 16), and already has 3 dedicated property tests + (`tests.rs`-adjacent property tests in `planes.rs:1656-1731`) asserting the exact family invariant + per plane index across a `divergent_formats` mask matrix. There is little to gain from migrating A + itself to read a new external table — it operates on the wrong tree shape (`PlaneExpr`, not + `FilterExpr`) to share code with B/C directly, and it's already the thing being unified TO. Its only + role in this unification is as validation: once a field-level table exists, a property test can + assert every plane-eligible field's `PLANE_BLOCKS` classification agrees with the new table's + `Locality` (mirroring the plane-index matrix test's own shape), closing the loop rather than + touching A's logic. + +### Not attempted here + +No code changes were made — this is a design/investigation doc, per this round's brief. No new +`Locality` enum, table, or adapter was written or prototyped; the migration order above is a +recommendation for whoever picks this up next, not a plan this doc has started executing. + +## Risks and staging + +| Call site | Failure mode if a field's `Locality` is migrated wrong | Current test coverage (proxy for how safely a migration could be verified) | +|---|---|---| +| **A** (`planes.rs`, router cost-tier + executor row selection) | **Worst.** A wrong `CardLevel` verdict on an actually-printing-varying field silently zeroes the router's verify-cost tier (exactly the Round 15/16 bug shape) AND, independently, tells the executor's `existential_plane_for` to skip the per-printing row-selection walk — returning a row that does not actually satisfy the query for `unique=card` (a correctness bug, not just a mispriced plan). | Strong, direct: 3 dedicated property tests over the full `PLANE_BLOCKS` matrix (`planes.rs:1656-1731`) plus 2 concrete router-regression tests (`compose_tier_charges_border_existential_and_arith_range`, `compose_tier_charges_divergent_legality_existential_and_arith_range`) from Rounds 15-16. Not planned to move in this unification (see above) — listed for completeness. | +| **B** (`estimator.rs`, standalone cardinality estimator) | A wrong `PrintingLevel`→`CardLevel` flip could make a bound UNSOUND (the estimator's one hard invariant) — e.g. classifying a real printing-varying field as card-invariant could let `compose_and`'s Bonferroni path apply where it shouldn't, or let `is_total_two_valued`/the `Not` branch pick a bound formula that assumes no printing divergence. The opposite direction (over-classifying as printing-varying) stays sound, just looser — see "legality disagreement" above for why B's current bias is safe. | Strong but indirect: no dedicated unit test for `has_printing_varying_leaf` by name, but its only consumer's soundness is fuzzed exhaustively (`fuzz_row_identity_matches_reference`, 96 seeds × ~10-13 random trees each + a 6k-card corpus pass with 2,500+ row-identity checks) and asserts the exact invariant a migration bug would violate. Not currently wired into routing (`estimator.rs:1-5`: "NOT wired into query routing"), so a regression here has zero production blast radius today — the safest of the three to migrate for that reason alone. | +| **C** (`filter.rs`, verify/paging order) | A wrong classification only produces a suboptimal evaluation order — never a wrong row, never a wrong count, per the "legality disagreement" trace above (every And/Or child still gets correctly evaluated regardless of the order it's tried in). Worst case is a measurable latency regression on a narrow query shape, the same failure class the confirmation-pass benchmarks in the Round 15/16 doc already gate on. | Moderate, direct-but-narrow: 2 ordering tests (`verify_order_and_defers_printing_dependent_children`, `verify_order_or_defers_printing_dependent_children`) exercise the ordering OUTCOME for representative shapes, but neither is a per-field property-matrix test like A's — a migration should add one before shipping, mirroring A's `PLANE_BLOCKS` matrix test shape. | +| **D** (`lib.rs`, compose broadcast gate) | Not migrating (already reads A directly) — a hypothetical wrong `is_broadcast_leaf_shape` ADDITION (widening its arm list past what A would call safe) would be caught by the existing rarity/border negative-composability tests (`tests.rs:7825-7831`) plus `fuzz_row_identity_matches_reference`'s card-mode row-identity checks, which would surface a wrongly-broadcast existential leaf as a returned row that fails the trusted `ref_filter.matches` assertion. | Moderate: 1 direct negative test (rarity/border rejection) plus indirect coverage via the `compose_printing_bits`-vs-brute-reference comparison cases in the same test function and the crate-wide fuzz harness. | + +**Cross-cutting risk this round's brief called out**: the reason this is a design doc and not a patch +is that a single migration commit touching A+B+C+D would span four modules with genuinely different +safety contracts (a soundness invariant, a verify-ordering heuristic, a compose-build correctness +gate, and a router cost-tier) — exactly the four failure modes tabulated above. The staged order +above is built so each step lands with its own existing (or one small added) regression gate, rather +than one commit whose blast radius spans all four at once. + +## Explicitly out of scope / open questions + +- **The `TypeCmp` compose-broadcast gap** (D's arm list excludes `TypeCmp` even though it's + card-level per every classifier) — a real, separate opportunity (or non-opportunity; not measured) + to extend `is_broadcast_leaf_shape`/`is_printing_composable`, unrelated to this unification. Would + need its own win-rate measurement (same shape as + [local-engine-plane-scope-printing-compose-executor.md](local-engine-plane-scope-printing-compose-executor.md)'s + 0/3,209 finding for a different gap) before deciding it's worth building. +- **Whether a fifth classifier exists that this round's four-classifier scope missed.** Not + exhaustively searched (e.g. any admin/backfill/tagging code path in `api/` that reasons about + card-vs-printing scope in Python, outside `card_engine/` entirely) — out of scope; this doc is + scoped to the Rust engine's four (per Round 16's own enumeration). +- **Whether `is_total_two_valued`** (`estimator.rs:136`, a strictly narrower "safe to cleanly + complement" classifier used only in `estimate_rec`'s `Not` branch selection) belongs in the same + unification. It answers a related-but-different question (total two-valuedness, a superset + restriction of card-level-ness — only `True`/`ColorCmp`/`TypeCmp` qualify, well short of every + card-level field) and was not treated as a fifth peer here, since it isn't one of the four Round 16 + named; flagging it exists in case a future table's design wants to fold it in as a derived property + (`CardLevel` AND "never Null") rather than leaving it as its own hand-maintained list. diff --git a/docs/issues/local-engine-cost-model-cleanup-remaining.md b/docs/issues/local-engine-cost-model-cleanup-remaining.md new file mode 100644 index 000000000..20791c035 --- /dev/null +++ b/docs/issues/local-engine-cost-model-cleanup-remaining.md @@ -0,0 +1,121 @@ +# Cost/Feature Estimation: Remaining Candidates After the Card-Invariant Scan Depth Fix + +Companion to branch `engine-cost-model-cleanup` (cut from #1024). That branch has landed four commits +so far: two `compose_printing_estimate` precision bugs (unit-mismatched `domain_hint`, match-count-vs- +span confusion for printing-varying fields), the `SpaceEstimate` triple refactor +([local-engine-compose-estimate-space-triple.md](local-engine-compose-estimate-space-triple.md)'s +design), and — most recently — pricing card-invariant bare leaves (`cmc`/`power`/`toughness`/ +`color_identity`/`legality`/`border`/card-space collections) at their true one-printing scan depth +instead of `domain_cards * printings_per_card * COMPOSE_CANDIDATE_SPAN_BIAS`, paired-diff verified at +897 improved / 2 regressed (91.5M → 84.7M total absolute `scan_units` error, pooled across every +`unique=card` plan/acquire). + +This is the punch list of what's left, in priority order, from re-running +[`bench_cost_model_agreement.py`](../../scripts/bench_cost_model_agreement.py) against that fix. Each +item needs its own measure-then-implement pass and its own commit — see the four already-shipped +commits for the pattern (paired diff, magnitude-weighted, before touching a shared constant or gate). + +## Current state of the table + +By plan × `unique` (measured/predicted; >1 under-costed; `within 25%` is the agreement rate): + +``` +plan unique n median p10 p90 within 25% +GatheredScan card 14120 0.68 0.25 2.73 16% FAIL +GatheredScan printing 14009 0.81 0.25 2.61 25% +GatheredScan artwork 13871 0.83 0.49 2.66 22% +PlanePopcountOrder card 2214 0.76 0.54 0.94 33% FAIL +PrintingCompose printing 5530 0.76 0.23 1.22 30% FAIL +PrintingCompose card 4256 1.00 0.55 1.80 46% +``` + +By plan × acquire branch, worst cell not already covered above: + +``` +plan acquire branch n median p10 p90 within 25% +StreamedSelect card_range_popcount 668 0.76 0.40 1.76 13% FAIL +``` + +## 1. `GatheredScan`/card's remaining `scan_units` error (biggest lever) + +Extracted into its own doc — see +[local-engine-gathered-scan-card-printing-varying-depth.md](local-engine-gathered-scan-card-printing-varying-depth.md). +It's the biggest lever (worst median, worst agreement, highest frequency of any cell in the table) and +is now the subject of an ongoing iteration ledger, not a single-pass fix — see that doc for the current +best, the constraints (pre-computation requirement, the price-triple correlation risk), and the round- +by-round log. + +## 2. `StreamedSelect`/`card_range_popcount` (worst other cell, cheap to check) + +**Population**: only 668 rows, but the worst agreement rate anywhere (13% within 25%, median 0.76). +Not investigated at all this session — everything focused on `PrintingCompose`'s `GatheredScan`/ +`StreamedSelect` competition, not the range-acquired path. + +**Investigation plan**: `CardRangePopcount`'s own row (668, median 0.80, 51% within 25%) is +reasonably healthy, so the miscalibration is specific to how `StreamedSelect` is COSTED when a bare +range wins the acquire — i.e. `acquire_plan_features`'s `CardRangePopcount` branch's `eval_domain`/ +`scan_units`/`compose_scan_printings` feed into `StreamedSelect`'s OWN cost arm (`plan_cost`), not +into `CardRangePopcount`'s. Start by reading that branch's `mk_plan_feats` call +(`card_engine/src/lib.rs`, `PhysicalPlan::CardRangePopcount.applicable` arm) alongside +`cost::plan_cost`'s `StreamedSelect` arm, and check with a live `explain()` probe on a handful of +one-sided range queries (`usd>5`, `cn>100`) whether the feature or the coefficient is the mismatch — +same first move as every fix that shipped this session. + +**Risk**: low — small population, likely narrow root cause given `CardRangePopcount` itself is fine. + +## 3. `PrintingCompose`/printing mode (unexamined mode) + +**Population**: 5,530 rows, 30% within 25%, median 0.76 (FAIL). Every fix this session was scoped to +`Mode::Card` — printing mode's own `.result.printing`-consuming leaf arms and its `scan_all`/ +`nothing_to_verify` paths have not been separately audited the way `.card` was. + +**Investigation plan**: re-run the same shape-based bucketing +(`profile_scan_units_bulk.py`-style, but filtered to `unique=="printing"`) to find which AST shapes +dominate. Printing mode has no "first match wins" dedup semantics at all (`GatheredScan` under +printing mode returns every matching printing, not one per card — see `push_card_matches`'s +non-`Mode::Card` arms), so the card-invariant depth-1 fix does NOT apply here; whatever the dominant +error shape turns out to be, it needs its own mechanism, not a port of this session's fix. + +**Risk**: unknown until the shape breakdown runs — treat as a fresh investigation, not an extension. + +## 4. The shared `GatheredScan` p90 tail (card 2.73, printing 2.61, artwork 2.66) + +**Observation**: all three modes show a very similar p90 (~2.6-2.7x under-costed at the tail), despite +having different median behavior (0.68 / 0.81 / 0.83). Similar magnitudes across otherwise-different +populations is suggestive of one shared root cause — e.g. a specific acquire branch, paging decision, +or query shape that feeds `GatheredScan` identically regardless of `unique` — rather than three +separate long tails that happen to coincide. + +**Investigation plan**: before assuming this is real, check population overlap directly — pull the p90 +rows from each mode's sample and see whether they cluster on the same query shapes (a `Not`, a wide +`Or`, a specific acquire branch like `plane` or `printing_range_scan`). If they do, fixing that one +shape moves all three cells at once, which is worth knowing before scoping items 1-3 as `Mode::Card` +only. If they don't overlap, this is coincidence and each mode's tail is a separate, lower-priority +problem than its own median. + +**Risk**: this is a triage step, not a fix — low cost, and it changes how items 1 and 3 should be +scoped if the tails turn out to share a cause. + +## Explicitly considered and rejected: exact intersection for lone non-arith + arith leaves + +`cmc=2 c=g`-shaped queries (exactly one plane-compilable card-invariant leaf — color/border/legality/ +rarity — ANDed with one-or-more `cmc`/`power`/`toughness` leaves) have no path to an exact card count +today: `compose_printing_estimate`'s `best_other` intersection requires **2+** non-arith card-invariant +leaves before it even starts, and the arith-merge that would otherwise combine a lone non-arith leaf +with the arith side is itself gated on `best_other` already existing. Relaxing the `>= 2` threshold to +`== 1` (paired with at least one arith sibling) closes the gap logically and was implemented, but a +targeted acquire-time A/B (not just `scan_units` accuracy) found a **23.6x acquire-time regression** on +exactly the newly-admitted population (median 875ns → 20,646ns), against a flat control population +(2+ non-arith leaves, unaffected) that moved 10,625ns → 10,792ns — noise. Reverted; not committed. The +mechanism (an unconditional `eval_planes`/`popcount_with_bits` pass now paid by a much larger query +population than before) is the same class of failure as the historical 2.33x regression documented +inline at the arith-merge site (a different over-eager widening, reverted for the same reason). + +A cheaper, sampled/capped version (probe the first N ids and extrapolate) was also considered and +rejected for now: it would still pay a nonzero floor (the `compile_plane`/`eval_planes` setup cost, not +just the ID-probe loop that scales with match count), it would stop being *exact* — which defeats the +purpose, since `card_invariant_domain_exact`'s whole argument depends on zero false positives — and +there is no evidence yet that this specific gap's accuracy loss actually changes any routing decision. +Worth reconsidering only after item 1 ships and the pooled agreement table is re-measured; if this +shape still shows up as a meaningful contributor, a sampled version with an explicit error bound (not +disguised as exact) would be the right design, not a resurrection of the reverted attempt. diff --git a/docs/issues/local-engine-domain-cards-existential-arith-and.md b/docs/issues/local-engine-domain-cards-existential-arith-and.md new file mode 100644 index 000000000..c6f9d33ce --- /dev/null +++ b/docs/issues/local-engine-domain-cards-existential-arith-and.md @@ -0,0 +1,1065 @@ +# `domain_cards`/`eval_domain` Is Wrong for Arith-Range AND Existential-Leaf, and Now Has a Root Cause + +## Round 25: the Blocker Is Confirmed Cleared, the Joint Refit Was Retested on Clean Data, and It Still +## Fails — a Third, More Precise Negative Result + +Round 20 named the recommended next step explicitly: fix `eval_domain` first, then the +`GATHER_CARD_PASS_NS`/`GATHER_RESIDUAL_FLOOR_NS`/leaf-count-rate joint refit (Rounds 17/19 built the +mechanism, both found a naive fit didn't survive held-out validation) becomes testable for real. Rounds +22/24 did that fix. This round is the first to actually run the refit against the now-clean data — and +it still fails, for a reason more precise than either prior round could see with a corrupted +`eval_domain`: **a flat, additive per-extra-leaf rate, multiplied by `eval_domain`, cannot serve both +ends of this population's `eval_domain` range at once** — `border:black`'s `eval_domain` (24,734, near- +universal selectivity) is ~50-100x `border:gold`'s or `f:oldschool`'s (a few hundred), and any single +rate that improves the small-`eval_domain` leaves overshoots the large ones by the same multiple. See +the parent doc (`docs/issues/done/local-engine-gathered-scan-undercosted-arith-existential-and.md`)'s +Round 25 section for the full mechanism, held-out numbers, and outcome — this section records only what +belongs here: confirmation that this doc's own `eval_domain` fix is holding up broadly, cleanly +separated from that round's (negative) refit attempt. + +### `eval_domain` re-verified clean for population A, broadly, not just on the 429-row sweep + +Re-ran a fresh, independent check (3 arith fields x 9 widths x 9 "clean" leaf values, 243 rows, +`explain_analyze` against a freshly-built store) rather than trusting Round 22/24's own reported +figures unchecked, per this round's brief: + +``` +cards_visited / eval_domain, population A (n=243): within 15% = 98.4%, median = 1.000 +``` + +Matches Round 22's own 89.3%-broad-sweep finding (the gap here is narrower, expected: this sweep drops +the two already-documented degenerate leaves, `border:silver`/`r:special`, that Round 22 itself +excluded from its "clean" figure). **Confirms, independently, that Rounds 22/24's `PairTotals`/ +`pair_range_sum` combination did what the parent doc's "What this means for the queued joint-rate +refit" section claimed**: `eval_domain` is not the reason the refit fails this round. The full negative +result and its real mechanism are in the parent doc, not duplicated here — see the link above. + +### Population B's `scan_units` confound: confirmed real, confirmed severe, and a SEPARATE quantity +### from `eval_domain` + +This doc's own "Open questions" section never covered `scan_units`; the brief for this round asked to +check it. Measured directly (27 bare-existential-leaf rows, no arith partner, `unique=card`): + +``` +printings_examined / scan_units, population B (n=27): within 15% = 0.0%, median = 3.950 +``` + +Real and severe — `scan_units` (the feature `GATHER_SCAN_PER_ROW_NS` multiplies) under-predicts the +real printings scanned by a median ~4x for this population, confirming Round 17/20's own smaller-sample +finding (era-correlated print position violates the uniform-random-depth assumption `scan_all` makes) +at a larger, fresh sample. This is orthogonal to `eval_domain`/`domain_cards` (a candidate CARD count, +already fixed) — `scan_units` is a printing-SPAN estimate, a different mechanism, still unfixed, and +out of this round's `cost.rs`/`lib.rs`-tier-decision blast radius (it lives in `acquire_plan_features`'s +`scan_all`/`card_invariant_domain_exact` machinery, untouched by Rounds 15-24's `And`-arm work). The +refit script accounted for it explicitly (substituted the realized `printings_examined` counter for the +`scan_units` feature when computing calibration TARGETS, so this confound could not leak into the +CARD_PASS/FLOOR/LEAF fit the way `eval_domain`'s confound did in Round 20) rather than letting it +silently pollute the fit — see the parent doc for why that substitution still wasn't enough to make the +fit ship. + +### Outcome + +No code change in this doc's own blast radius (`lib.rs`'s `PairTotals`/`compose_printing_estimate`, +untouched this round, exactly as Round 24 left it). The refit attempt and its negative result belong to +`GATHER_CARD_PASS_NS`/`GATHER_RESIDUAL_FLOOR_NS`/`plane_extra_eval_leaves` (`cost.rs`/`lib.rs`'s tier +decision) and is written up in full in the parent doc's own Round 25 section — read that for the +mechanism, the numbers, and the recommended next step (a saturating/bounded leaf term or a per- +selectivity-band calibration, not a flat linear one). + +## Round 24: `PairTotals` Extended to `cmc`/`power`/`toughness` — Round 22's Tax Closed for the Common +## Widths, Shipped + +Round 23 found the right shape but didn't build it: `indexes.pair_totals` (`card_engine/src/lib.rs`) +already stores an EXACT per-value-pair total for `border`/`rarity`/`frame`/`legality`, and the same +disjoint-partition argument (a card has exactly one `cmc`/`power`/`toughness`) applies to those three +fields too — summing the per-value pair-total over every value a RANGE admits reproduces the true joint +exactly, no independence assumption. This round built that extension and shipped it. + +### What changed + +`card_engine/src/lib.rs`: +- **`PairTotals`**: three new dimension maps (`cmc: HashMap`, `power`/`toughness: + HashMap`, mirroring `rarity`'s existing shape) plus three `_seen: Vec`/`Vec` lists — + every DISTINCT value observed at all, before `PAIR_MIN_PRINTINGS` prunes the id maps. The `_seen` + lists exist for one reason: without them, a range sum cannot tell "no card has this value" (safe, + contributes zero) apart from "some card has this value but it was pruned" (unsafe — silently + treating it as zero would undercount). A new `get_all` on `ArchivedPairTotals` returns all three + spaces (printing/card/artwork) for one pair in one hash lookup, instead of `get`'s one-space-at-a-time. +- **`build_pair_totals`**: the three new dimensions ride the SAME single accumulation pass every other + dimension already uses — one more per-value count in pass 1, one more `ids.push` in pass 2. `cmc`/ + `power`/`toughness` are read from the CARD (`OracleCard`), not the printing — confirmed directly from + the struct layout that these fields are stored once per card, not per printing (unlike `border`/ + `rarity`, which vary by printing). +- **`pair_leaf_id`**: three new match arms (`Cmc`/`Power`/`Toughness`, `Eq` only, either operand order), + mirroring rarity's existing `Eq`-only restriction. Feeds the EXISTING `pair_bounded_min` call site for + free — a bare `cmc=1`-shaped leaf paired with any other pairable leaf (existential or not) is now + answered there without any new call site. +- **`single_arith_field`** (new): the single `NumField` every one of a set of arith children agrees on, + or `None` if they don't (a mixed `cmc>=1 power<=2`) or the set is empty. +- **`pair_range_sum`** (new): given bound leaves on ONE arith field and one existential leaf's own + `pair_leaf_id`, sums `pt.get_all` over every value in that field's `_seen` list that the bounds admit, + declining (`None`) the instant any admitted value lacks an id (pruned). Bounded by the field's own + distinct-value count (~14-21 in this corpus), not by how the query phrases the range. +- **`compose_printing_estimate`'s `And` arm**: `card_invariant`/`existential` now carry the original + `FilterExpr` alongside each compiled `PlaneExpr` (a new `CompiledLeaf` type alias), so `pair_range_sum` + can ask `pair_leaf_id` about the lone existential leaf's own value without re-deriving it. A new + `pair_range_answer` is computed FIRST, before Round 22's `best_other` loop, for exactly the shape Round + 22 fixed (`card_invariant.is_empty()`, exactly one existential leaf, arith children on one field); when + it answers, Round 22's `popcount_with_bits`/arith-ID-probe-merge machinery is skipped entirely for that + query. When it doesn't (a card-invariant leaf present, 2+ existential leaves, 2+ distinct arith fields, + or a pruned value), Round 22's fallback runs completely unmodified. + +### Exactness: 429/429 agreement where both paths apply + +Re-ran Round 22/23's own 3-field × 13-width × 11-leaf-value sweep (429 rows) against two isolated +release wheels — `costcell/trunk`@`68f2cd7f` (Round 22's fix, pre-this-round) and this branch — reading +`engine.query()`'s own total (ground truth) and `explain()`'s `eval_domain` on both: + +``` +true_intersection: 0/429 mismatches between the two wheels (query correctness unaffected, as expected — + this round only touches cost ESTIMATION, never the executed result set) +eval_domain: 0/429 rows differ between before/after (both wheels answer exactly wherever either + can — no case where the new path disagreed with Round 22's exact fallback) +``` + +### Coverage: how much of the taxed population gets the new cheap path + +Instrumented directly (temporary `eprintln!`, reverted before commit) to distinguish, per sweep row, +whether `pair_range_answer` fired, declined due to pruning, or the shape didn't match at all: + +``` +429 rows total + 33 (width=13 only): the And arm's existential logic isn't reached AT ALL for this width — a separate, + pre-existing fusion mechanism takes over once the range covers essentially the + whole corpus (same "collapses at width 13" behavior Round 22/23 already documented) + 72 (border:silver, r=special — the two ALREADY-DOCUMENTED degenerate leaves): the existential leaf + itself never reaches `existential.len()==1` (a separate, pre-existing quirk in how these two + specific values get classified upstream, unrelated to this round and out of its blast radius) +324 (the 9 "clean" leaf values × 3 fields × 12 widths): shape matches every time + 180 (55.6% of the 324): pair_range_answer fires — exact, cheap + 144 (44.4% of the 324): declines due to pruning, falls back to Round 22's exact (more expensive) path +``` + +By width (9 clean leaves × 3 fields = 27 rows/width): **widths 1-6: 100% hit (162/162). Width 7-8: 33% +hit (18/54) — only `cmc`, whose survivor set (below) extends one value further than `power`/ +`toughness`'s. Widths 9-12: 0% hit (0/108), all correctly decline and fall back.** This traces exactly +to `PAIR_MIN_PRINTINGS` (1,024) pruning individual values, confirmed directly against the real corpus: + +``` +cmc survivors (>=1,024 printings): 0,1,2,3,4,5,6,7,8 (9 values) +power survivors: 0,1,2,3,4,5,6 (7 values) +toughness survivors: 1,2,3,4,5,6 (6 values) +``` + +A range up to width 6 stays within every field's survivor set; width 7-8 only `cmc` still clears (its +survivor set reaches 8); width 9+ exceeds all three. **This is a real, honest coverage boundary, not a +bug** — the population Round 22's fix taxed most heavily (wide ranges, per that round's own finding that +"the tax grows from +10,250ns at width 1 to +22,875ns at width 12") is exactly where this round's cheap +path covers LEAST — but the narrow-to-moderate ranges most plausible in real queries (`cmc<=3`, +`power>=1 power<=4`, ...) are exactly where it covers MOST, and that's the population this round +prioritized finishing over chasing the last few widths for diminishing returns. + +### Acquire-time improvement: measured directly, same reproducers + +Same sweep, `explain_analyze` acquire-time medians (20 warmups, 100 trials), before = Round 22's fix, +after = this round: + +``` +n=429, median delta (after-before): -771ns mean: -11,907ns +p10/p50/p90/max: -35,208ns / -771ns / +583ns / +2,396ns (min: -99,188ns) + +by width: 1: -9,793ns 2: -12,958ns 3: -15,751ns 4: -18,584ns 5: -20,854ns 6: -21,333ns + 7: -167ns 8: -313ns 9: +21ns 10: +83ns 11: +42ns 12: +83ns + (widths 7-12's near-zero median is the pruning cutoff above — most rows there fall back to + the unchanged Round 22 path, correctly paying the SAME cost as before, not a regression) + +by leaf (median): border:black -45,438ns r=common -22,500ns r=uncommon -20,980ns r=rare -20,959ns + border:borderless -10,208ns border:white -9,124ns r=mythic -9,250ns + f:oldschool -8,896ns border:gold -8,709ns (border:silver/r=special: ~0, unaffected) +``` + +Named reproducers (before → after, `explain_analyze` median): + +``` +cmc=1 border:black: 50,146ns → 4,708ns (10.6x faster) +cmc=1 border:white: 13,917ns → 4,666ns (3.0x faster) +cmc=1 r=mythic: 14,500ns → 5,000ns (2.9x faster) +cmc>=1 cmc<=5 border:black: 92,875ns → 5,333ns (17.4x faster) +cmc>=1 cmc<=5 border:white: 28,146ns → 5,333ns (5.3x faster) +cmc>=1 cmc<=5 r=mythic: 29,292ns → 5,625ns (5.2x faster) +``` + +Every one of Round 22's own named reproducers is now answered by the new cheap path (width ≤5, well +inside every field's survivor set) — this closes essentially all of Round 22's OWN acquire-time tax on +its own flagship population, not just a marginal slice of it. + +### Store-build-time cost: measured directly, negligible + +5 reps each, full corpus reload (`benchmarks/bitplanes/corpus.jsonl`, 97,812 printings), same two wheels: + +``` +before median: 2.470s after median: 2.511s delta: +40ms (+1.6%) — inside this measurement's own + run-to-run spread (before ranged 2.380-2.540s across + its own 5 reps, a ~160ms band bigger than the delta) +archived store size: before 72,402,040 bytes after 72,435,480 bytes delta: +33,440 bytes (+0.046%) +``` + +The transient build-time `n×n` co-occurrence array DOES grow more than the aggregate number suggests in +isolation — 22 new ids (9 cmc + 7 power + 6 toughness survivors) added to a pre-existing ~42, growing +`n_ids` to ~64 (a ~1.5x increase, ~2.3x for the `n²` array specifically) — but that transient array is a +small fraction of total reload time (JSON parsing + 22 other indices dominate), so the aggregate cost +lands at +1.6%, within noise. Accepted: real, measured, and small. + +### Correctness gate + +`cargo test --manifest-path card_engine/Cargo.toml`: **177/177 passed** (174 pre-existing + 3 new: +`pair_range_sum_sums_disjoint_values_and_declines_on_a_pruned_one`, `pair_leaf_id_resolves_cmc_power_ +toughness_eq_and_declines_ranges`, `single_arith_field_agrees_only_when_every_child_is_the_same_field` — +all three exercise the new logic directly against a hand-built `PairTotals`, bypassing `PAIR_MIN_ +PRINTINGS` entirely since a real fixture would need 1,024+ printings per value to clear it). `cargo test +--release`: **176/176 passed** (173 pre-existing + the same 3 new). Rounds 15/16/22's own regression +tests (`compose_tier_charges_border_existential_and_arith_range`, `compose_and_arm_tightens_lone_ +existential_leaf_with_no_card_invariant_partner`) pass unchanged — that fixture is too small to clear +the floor, so it exercises Round 22's fallback path exactly as it did before this round, confirming the +new path declines cleanly rather than silently taking over. `cargo clippy --all-targets -- -D warnings`: +clean. + +### Confirmation pass + +`bench_pairwise_ordering.py --seconds 300`, both modes, before vs after — no material change: + +``` +realistic: GatheredScan vs PrintingCompose 89%→89% ordered right, regret 9.29µs→9.77µs (flat) + GatheredScan vs StreamedSelect 97%→97%, 0.80µs→0.81µs (flat) + PrintingCompose vs StreamedSelect 94%→94%, 6.40µs→6.56µs (flat) +uniform: GatheredScan vs PrintingCompose 87%→87%, 7.16µs→7.62µs (flat) + GatheredScan vs StreamedSelect 95%→95%, 1.62µs→1.59µs (flat) + PrintingCompose vs StreamedSelect 95%→95%, 4.13µs→4.41µs (flat) +``` + +`bench_cost_model_agreement.py --seconds 300 --seed 0`, full table: + +``` +by acquire: 12/17 cells inside [0.8, 1.25] → 12/17 (unchanged, no cell flips) +by unique: 10/12 cells inside [0.8, 1.25] → 10/12 (unchanged, no cell flips) +``` + +`bench_regret_matrix.py --seconds 120 --mode realistic --seed 0`: + +``` +before: 53,497 queries, total regret 42.9ms (mean 0.80µs) +after: 53,437 queries, total regret 41.1ms (mean 0.77µs) -- improved 4.2%, same misroute categories, + no new outlier category +``` + +`bench_query_latency_ab.py --sample 400 --mode realistic --seed 7`, before vs after, plus a same-build +canary (before run 1 vs before run 2, interleaved: before1 → after1 → before2): + +``` +canary (before1 vs before2): B - A = +1.3µs 95% CI [+0.9, +1.8] +before1 vs after1 (real): B - A = +0.5µs 95% CI [+0.0, +1.0] +``` + +The real diff is SMALLER than the canary's own same-build noise — no detectable aggregate regression. +Consistent with the taxed population being a narrow slice (~0.85-1.23% of `Mode::Card` queries per +Round 21's proxy) of the realistic-mode sample this benchmark draws from. + +### Scope decision: the multi-arith-field generalization (power×toughness, etc.) — not folded in + +A real generalization exists and was checked against real data before deciding: `cmc`/`power`/ +`toughness` are each single-valued per card, but so is their JOINT tuple — a card has exactly one +`(power, toughness)` pair, not a range of possible pairs — so the same disjoint-partition argument +extends to an `And` spanning TWO (or three) arith fields together, with or without an existential leaf. +Measured the actual cross-product size directly against `benchmarks/bitplanes/corpus.jsonl` (grouped by +printing, matching `PAIR_MIN_PRINTINGS`'s own counting unit): + +``` +(power, toughness): 122 distinct pairs, 13 clear the 1,024-printing floor +(cmc, power): 141 distinct pairs, 11 clear the floor +(cmc, toughness): 144 distinct pairs, 14 clear the floor +(cmc, power, toughness): 497 distinct triples, 10 clear the floor +``` + +Small in every case — technically cheap, confirming the idea is sound and not a combinatorial trap. +**Not folded into this round anyway**, because it needs genuinely new plumbing beyond an extension of +what's already built: a compacted joint-value key (its own map, analogous to `legality`'s `(shift<<2)| +status` compaction, but for however many field combinations are worth covering), that key's OWN +pruning-safety `_seen` list, and a cross-product-aware version of `pair_range_sum` that enumerates BOTH +ranges' surviving values together — comparable in size to everything this round already built, for a +population narrower still than the single-field shape (a real query ranging power AND toughness +together, with exactly one existential leaf and nothing else card-invariant, is less common than the +already-narrow single-field case this round covers). **Correctness is unaffected either way**: Round +22's existing `arith_tuple_ids`-based probe-merge already answers the 2+-arith-field case EXACTLY today, +completely unchanged by this round (confirmed by reading the code path directly — `single_arith_field` +returns `None` for a mixed-field `arith_children`, so `pair_range_answer` stays `None` and the query +falls through to Round 22's unmodified fallback). This is a real, well-scoped follow-up for a future +round — a speed opportunity left on the table, not a correctness gap — and the measurements above are +that round's head start. + +### What this means for the queued joint-rate refit + +Round 20's blocked joint refit (`GATHER_CARD_PASS_NS`/`GATHER_RESIDUAL_FLOOR_NS`/leaf-count rate) needs +`eval_domain` to be trustworthy ground truth. Round 22 already cleared that for 89.3% of the broad sweep +(up from 5.8%). This round doesn't change that coverage number (it was already exact via Round 22's +fallback everywhere this round's new path doesn't reach) — what it changes is the ACQUIRE-TIME COST of +getting there for width ≤6-8 (the common case), not which rows are exact. **The refit's ground truth is +exactly as clean as it was after Round 22; this round makes reaching that ground truth cheaper for the +population it covers, and leaves the rest on Round 22's already-exact (just pricier) fallback.** No new +correctness caveat for whoever runs that refit next. + +### Commit + +One commit on `costcell/24-pair-totals-arith`. `git diff --stat costcell/trunk`: `card_engine/src/ +lib.rs`, `card_engine/src/tests.rs`, this doc. + +## Round 23: Is Round 22's Tax Avoidable? A Cheap Bound Investigated and Rejected, a Better Exact +## Alternative Found Instead — Not Shipped + +Round 22 fixed correctness (`eval_domain` now exact for this population) at a real, measured acquire-time +cost (`popcount_with_bits` + the arith-ID-probe merge, `O(matching_ids)`, now running where it used to be +skipped). The question this round investigates: can a much cheaper `O(1)`/`O(log n)` estimate — e.g. +`min(exact_count(arith_leaf), exact_count(existential_leaf))`, each side's own already-cheap count — get +"good enough" routing without paying that tax? Investigation only; no code shipped. All numbers below are +from a real corpus (`benchmarks/bitplanes/corpus.jsonl`), two isolated release wheels (`costcell/trunk`@ +`cc17e031`, pre-Round-22, and this branch's base `188a0ee4`, post-Round-22 — both built via `maturin build +--release`, no `make engine`/`maturin develop`), and a Python port of `cost.rs`'s `plan_cost` verified to +reproduce `explain()`'s own `predicted_ns` to 0.0000 relative error across 1,287 plan-rows before being +trusted for anything. + +### 1. The taxed population, characterized broadly (not just the 6 reproducers) + +Re-ran Round 22's own 3-field (`cmc`/`power`/`toughness`) x 13-width x 11-leaf-value sweep (429 rows, +`cmc>=1 cmc<=W` for `W` in 1..13, same 11 existential values), this time capturing real `explain_analyze` +acquire-time on BOTH wheels for every row, not just the 6 named reproducers: + +``` +n=429, median delta (after-before): +17,605ns mean: +22,921ns +p10 / p90 / max: -83ns / +52,562ns / +95,230ns +median ratio (after/before): 4.61x p90: 10.74x max: 19.81x +``` + +By existential leaf (median delta, median ratio): + +| leaf | median delta | median ratio | +|---|--:|--:| +| `border:black` | +69,958ns | 15.53x | +| `r=common` | +36,812ns | 8.22x | +| `r=rare` | +35,646ns | 8.05x | +| `r=uncommon` | +33,958ns | 7.68x | +| `f:oldschool` | +15,584ns | 4.49x | +| `r=mythic` | +17,500ns | 4.45x | +| `border:borderless` | +18,625ns | 4.94x | +| `border:white` | +16,229ns | 4.43x | +| `border:gold` | +15,334ns | 3.83x | +| `border:silver`, `r=special` | ~0ns | ~1.0x (degenerate — see below) | + +`border:silver` never appears in this corpus at all (the real 5th border value is `yellow`; a labeling +miss carried over from Round 22's own sweep, not a new finding) and `r=special` is the separate, +already-documented `eval_domain`-flat-across-widths bug (Question 1's own doc, above) — both are +degenerate zero/near-zero-signal rows, correctly showing ~no delta. By width, the tax grows from ++10,250ns (width 1) to +22,875ns (width 12), then collapses to +500ns at width 13 (the range covers +essentially the whole corpus, so both before/after `eval_domain` saturate near `n_cards` and the two +converge). **`border:black`'s 15.53x/+69,958ns is the worst cell specifically because of its own 98.9% +selectivity**: the arith-ID-probe merge's cost scales with `O(matching_ids)`, and border:black keeps +almost every one of the arith leaf's own matching ids in play — the widest possible probe set. + +Real-traffic prevalence: Round 21/22's own regex-proxy estimate (0.85-1.23% of `Mode::Card` queries) is +the best available figure for this specific "lone existential + arith, nothing else card-invariant" +population; narrowing it further would need an AST-level classifier over `QuerySampler` output, out of +scope for this round's budget. + +### 2. Cheap `min()` vs exact: real gap, and the RATIO alone is misleading + +The cheap estimate (`min` of each side's own bare exact count — `bare_numeric_field_count`/ +`numeric_range_ids`, a genuine `O(log n)` two-`partition_point` lookup with no allocation, confirmed by +reading `numeric_range_count`'s body directly; and `value_totals.border`/`.legality` or +`rarity_cards.distinct_cards`, both O(1)/O(log n) too — so "cheap" is real, not hand-waved) was computed +for all 429 sweep rows and compared against the true joint intersection (`engine.query()`'s own `total` +for the AND filter — unambiguous ground truth, independent of any estimate): + +``` +zero-true-match rows excluded (39, degenerate): 390 remain +exact match (cheap == true): 0/390 (0.0%) +overestimate ratio (cheap/true): median 2.015x p90 3.939x max 27.579x +rows where cheap < true (would violate the upper-bound guarantee): 0/429 +``` + +Cheap is *never* an underestimate (expected — it's a `min` of two supersets, a mathematically valid upper +bound) and *never* exact either. But **the ratio alone overstates which rows matter** — exactly the +failure mode the user flagged: a 16x error on a true value of 200 is a ~3,000-card absolute miss; a 2x +error on a true value of 5,500 is a larger, ~5,500-card absolute miss that the cost model actually feels. +By leaf (median true intersection, median cheap value, median ABSOLUTE delta, median ratio, and how many +of that leaf's 39 rows flip routing — see §4): + +| leaf | med. true | med. cheap | med. abs. Δ | med. ratio | routing flips | +|---|--:|--:|--:|--:|--:| +| `r=rare` | 5,321 | 11,059 | **+5,709** | 2.078x | **23/39** | +| `r=uncommon` | 5,190 | 10,279 | **+4,886** | 1.981x | **23/39** | +| `r=common` | 5,897 | 10,694 | **+4,636** | 1.792x | **23/39** | +| `border:black` | 16,417 | 16,664 | +246 | 1.015x | 1/39 | +| `r=mythic` | 1,410 | 2,620 | +1,210 | 1.858x | 0/39 | +| `border:white` | 946 | 2,059 | +1,113 | 2.177x | 0/39 | +| `border:borderless` | 1,684 | 3,478 | +1,794 | 2.065x | 0/39 | +| `f:oldschool` | 382 | 961 | +579 | 2.516x | 0/39 | +| `border:gold` | 147 | 551 | +404 | 3.748x | 0/39 | +| `r=special` | 153 | 370 | +217 | 2.418x | 0/39 | + +`border:gold`'s 3.748x median ratio is the WORST ratio among borders, and it flips *nothing* (absolute +miss ~400 cards). `r=rare/common/uncommon`'s ~2x ratios are unremarkable next to `border:gold`'s or +`r=mythic`'s, yet they cause every routing flip but one (§4) — because their bare existential-side counts +are ~10,000-11,000 cards (32-35% corpus selectivity), so even a "modest" 2x ratio is a multi-thousand-card +absolute error, and the cost model's per-candidate rate (~4-13ns/card across the competing plans) turns +that into tens of microseconds. **Absolute magnitude, not ratio, is what predicts real damage — confirmed +directly in §4, not asserted.** + +**Anti-correlation check, done directly on this corpus's own fields**: the historical `id:br devotion:w` +case is a near-ZERO true intersection against two individually-large marginals (a hard contradiction — +`id:br` requires white in the identity color-wise never mind, the point is the near-zero intersection). +Nothing that extreme appears here (zero rows had `true_intersection == 0` except the degenerate +`border:silver` rows, where the leaf itself doesn't exist in the corpus). But a MILDER version of the same +effect is real and visible: `field=1 r=mythic` (a low arith value ANDed with mythic) is the worst-ratio +population (up to 27.579x, `toughness=1 r=mythic`, true=95 vs cheap=2,620) precisely because low-cmc/ +power/toughness cards skew away from mythic rarity in this corpus (a plausible real card-design +correlation, not noise) — so `min()`'s independence assumption is measurably wrong here, just not +catastrophically so in absolute terms (see the table above: this shows up as a big ratio but a modest +absolute delta, and correctly causes zero routing flips). + +### 3. A better alternative than `min()`: an EXACT disjoint-bucket sum, not a bound + +`cmc`/`power`/`toughness` are **single-valued per card** (a card has exactly one cmc) — this corpus has +only 17/19/21 distinct integer values for the three fields respectively. That means partitioning cards by +their exact arith value is a genuine, exhaustive, non-overlapping partition of the card space: summing a +PER-VALUE exact joint count (arith value `v` -> count of cards with that `v` AND satisfying the existential +value) over every `v` in `[lo, hi]` reproduces the TRUE joint intersection exactly — no independence +assumption, no anti-correlation risk, because it's a sum over disjoint cells, not a product or a `min`. +This does NOT generalize to multi-valued fields (card types, formats/legality — a card can have several), +only to the single-valued arith-tuple family, exactly as scoped. + +**Validated directly against the real corpus**, not just argued: built the (arith value -> existential +value -> distinct card count) table from `benchmarks/bitplanes/corpus.jsonl` (grouping printings by +`oracle_id`, one row per card) for all three arith fields against border/rarity/`f:oldschool`, then checked +`sum(table[field][v] for v in [lo,hi])` against every one of the 429 sweep rows' real `true_intersection`: + +``` +checked=429 exact_matches=429 (100.0%) +``` + +Every single row, exact, including the degenerate `border:silver` (0) and `r=special` rows. Table size: +138 (cmc) + 145 (power) + 152 (toughness) non-zero (value, existential-value) cells = 435 cells total, +each a 3-space count — a few KB, not a new large structure. + +**An existing, already-shipped precedent for exactly this pattern was found**: `indexes.pair_totals` / +`pair_leaf_id` / `pair_bounded_min` (`card_engine/src/lib.rs`, ~2705-2900 and 8530-8574) is the SAME +disjoint-pair-sum idea, already built and already wired into the `And` arm — `pair_bounded_min(v, indexes, +folded.result.printing)` runs unconditionally at line 7736, *before* any of Round 22's existential-arith +machinery even starts. It currently recognizes exactly four dense, low-cardinality dimensions — `border` +(`TextExact` `Eq`), `frame_data` (`CollectionCmp` `Ge`), `legality` (per format/status), and `rarity` +(`NumericCmp{RarityInt}`, `Eq` only) — and **nothing for `cmc`/`power`/`toughness`**: `pair_leaf_id`'s +match falls through to `_ => None` for every `NumericCmp` on those three fields, at any operator. + +This is the SAME root-cause shape Round 21 found for `value_totals.border` — an existing exact answer, +unreached for this specific AST shape. Confirmed directly: `parse_scryfall_query('cmc=1 border:white')` +produces a genuine single `=` binary-operator node for `cmc`, not an implicit two-leaf range — exactly the +shape `pair_leaf_id` already handles for rarity today. **Extending `pair_leaf_id`'s match (3 new arms: +`Cmc`/`Power`/`Toughness` `NumericCmp` `Eq`) plus `build_pair_totals`'s per-printing id-collection pass (3 +more lookups per printing, reusing the same `PAIR_MIN_PRINTINGS`-gated selectivity floor already there) +would make the WIDTH-1 slice of this population (`cmc=1 border:white`-shaped, 33/429 = 7.7% of the sweep, +the cheapest-to-tax slice at +10,250ns median but still real) exact via the EXISTING `pair_bounded_min` +call site, with zero new call sites and no new top-level struct** — the smallest possible next step here. + +**This does not cover the ranged case.** `pair_leaf_id` is deliberately `Eq`-only (mirroring rarity's own +restriction — "any other op is a range over several values, which no per-value entry answers"), and the +`And` arm passes `pair_bounded_min` the ORIGINAL, unfused children (`v`), not `fuse_and_range_children`'s +output — so `cmc>=1 cmc<=5` (two `Ge`/`Le` leaves) would still not match, even with the extension above. +The ranged case is the bulk of both the row count (12/13 widths) and essentially all of the absolute +acquire-time tax (width-1 rows average +10,250ns vs the +19,000-23,000ns width 4-12 average). Closing it +needs a range-capable mechanism, and there are two honest ways to build one: + +- **(a) Extend `pair_totals` itself**, adding cmc/power/toughness as 3 more dense dimensions to the + existing `O(n²)` id x id co-occurrence matrix `build_pair_totals` already builds, plus a new helper that + sums `pt.get(x, y)` over however many arith values in `[lo, hi]` clear the pruning floor (bounded, ≤21 + lookups). Reuses the existing struct/build pass/selectivity floor entirely, but grows the co-occurrence + matrix by ~45 more candidate ids (17+19+21, before pruning) against today's ~15-30 — a real, if + transient (index-build-time only) memory cost, and it stores a lot of PAIR cells (cmc×power, cmc×frame, + ...) the router never actually queries for this shape, since only arith×existential pairs matter here. +- **(b) A new, purpose-built cumulative table**, attached wherever the arith index's own per-value data + lives, mirroring `RangeCardCounts`'s existing `below`/`at_or_above`/`at` prefix-sum design (already + shipped, for a different set of range dimensions — price/collector-number/release-date) but keyed + additionally by existential value. Sized like the validation table above (~57 arith values x ~11 + existential values, well under 20KB), answers a RANGE in true `O(1)` (two `partition_point` calls plus a + prefix difference) rather than `O(bucket count)`, and doesn't touch the unrelated existing `pair_totals` + matrix at all — strictly better cost shape, at the price of being new code (new struct, new build pass, + new query-time helper) rather than an extension of something that already ships. + +Both (a) and (b) are **index-build-time work, not free** — this is squarely the "pre-computation over +hot-path computation" tradeoff the round's brief named, just located one level earlier (build time) than +either Round 22's fix or the `min()` idea (both pure query-time mechanisms). Neither was implemented this +round; both are validated-by-construction (the 429/429 result above already IS what (b) would compute) and +ready for a dedicated follow-up round to actually wire in. + +### 4. Does the cheap estimate change routing? Yes, often, and mostly for the worse + +Restricted the comparison to the plans `explain()` itself reports as real candidates for this acquire +branch — confirmed directly that `StreamedSelect` is NEVER offered here (396/429 rows offer exactly +`{GatheredScan, PrintingCompose}`, 33/429 offer `{GatheredScan}` alone; scoring a plan the router would +never actually consider would be a fabricated comparison). For each row, recomputed both plans' predicted +cost under the exact `eval_domain`/`scan_units` (matches `explain()`'s own numbers, verified to 0.0000 +relative error) and under the cheap `eval_domain` with `scan_units` recomputed via the SAME `scan_all` +density fallback the pre-Round-22 code path took (this shape's `composed_card_invariant` is always false, +so the `card_invariant_domain_exact`/`exact_domain_won` fast paths are unavailable either way, before or +after — confirmed from the code, not assumed): + +``` +argmin flips: 70/429 (16.3%) +69/70 flip GatheredScan (exact) -> PrintingCompose (cheap); 1/70 the reverse +``` + +Flips concentrate ENTIRELY on the three leaves with the largest absolute bare counts, not the leaves with +the worst ratios: `r=rare`/`r=common`/`r=uncommon` account for 69 of the 70 flips (23/39 each), plus one +`border:black` row. `r=mythic` (comparable ratio, ~6x smaller absolute count) flips zero times. + +**Real measured regret**, not just a predicted-cost artifact: ran `explain_analyze` (15 warmups, 60 +trials) on the actual post-Round-22 wheel for all 70 flipped queries, reading REAL per-plan trial medians +for both `GatheredScan` and `PrintingCompose` (both genuinely execute — this is not a simulation): + +``` +n=70 +55/70 (79%): cheap's pick really is slower — median regret when worse: +51,625ns, summed: +3,318,023ns +15/70 (21%): cheap's pick happens to be faster — median improvement: -25,708ns, summed: -404,456ns +net summed regret over just these 70 rows: +2,913,566ns (+2.91ms) +``` + +The 15/70 "improvements" are not the cheap estimate doing something right — they're the EXACT model's own +`PrintingCompose` cost formula being mis-calibrated for unrelated reasons on wider `r=rare` ranges (a +pre-existing cost-model imprecision this round didn't introduce and isn't trying to fix), which the cheap +estimate's overestimate happens to route around by accident. That is not a case for shipping it: a +routing choice that's "right for the wrong reason" 21% of the time against "measurably wrong" 79% of the +time, concentrated on exactly the rarity values (common/uncommon/rare) most likely in real traffic — not +the corpus's rare border/legality corners — is a net loser. + +### 5. Recommendation + +**Do not ship the `min()` cheap estimate.** It disagrees with the real candidate set's argmin on 16.3% of +this sweep, the disagreement is real (not a modeling artifact — measured directly via `explain_analyze` on +both real candidate plans), 79% of those disagreements cost real wall time (median +51.6us, net +2.9ms +over just 70 rows), and the disagreements concentrate on the leaf values (common/uncommon/rare) most +plausible in real traffic, not corpus-specific corners. Round 22's exact fix should stay unconditional. + +**But the right next step is not "accept the tax" either — it's the disjoint-bucket-sum family (§3), +which this round found to be EXACT (validated 429/429) at a cost this round did not fully characterize at +query time but which is bounded by construction** (≤21 lookups for the general form, `O(1)` for the +`RangeCardCounts`-style cumulative form) **and is strictly better than `min()` everywhere it applies**: no +anti-correlation risk, no accuracy/speed trade to make. Two concrete next steps, in order of size: + +1. **Smallest, immediate**: extend `pair_leaf_id`/`build_pair_totals` to recognize `cmc`/`power`/ + `toughness` `Eq` values (3 new match arms, reusing 100% of existing machinery) — closes the width-1 + slice (`cmc=1 X`-shaped, 7.7% of this sweep) via the ALREADY-WIRED `pair_bounded_min` call site, for + free at query time. A good, low-risk follow-up on its own. +2. **Complete, bigger**: build the `RangeCardCounts`-style cumulative per-(arith field, existential value) + table sketched in §3(b) — closes the ranged case too (the bulk of both the row count and the acquire- + time tax), at the cost of new index-build-time code, not a query-time trick. This is where a dedicated + follow-up round should aim; this round's validation (429/429 exact against real data) is that round's + head start, not something it needs to re-derive. + +### Artifacts + +Exploratory scripts (scratchpad only, not committed): `sweep_r23.py` (429-row sweep, both wheels), +`analyze_r23.py` (acquire-delta/cheap-vs-exact/routing-impact analysis, includes the verified `cost.rs` +Python port), `measure_flip_regret.py` (real per-plan trial measurement for the 70 flipped rows), +`validate_bucket_sum.py` (disjoint-bucket-sum validation against real corpus data). No engine-code edits +were made or reverted this round — every number above came from the two wheels as they already exist +(`cc17e031`, `188a0ee4`) plus new Python analysis, so `git diff --stat costcell/trunk` for this round shows +only this doc. + + + +Round 21's sketch (below) turned into a fix. `compose_printing_estimate`'s `And` arm (`card_engine/src/lib.rs`, +~line 7845): the `else if !card_invariant.is_empty()` guard on `best_other`'s existential-leaf loop is now a +plain `else` — a lone existential leaf (no card-invariant partner at all) runs the SAME +`popcount_with_bits(Some(e))` loop a card-invariant-paired existential leaf already used, unmodified. +`popcount_with_bits` already handled an empty `card_invariant` vec correctly (it was never incapable of +answering this shape, just never *called* for it) — see Round 21's own diagnosis above, confirmed unchanged +on `costcell/trunk`@`cc17e031` before this round touched anything. + +### Ingredients 2 and 3 from Round 21's sketch: investigated, not shipped + +**Ingredient 2** (prefer a precomputed exact count over a fresh `eval_planes`+`popcount` for the scalar): +does not cleanly apply to the population this bug actually affects. The BITS from `popcount_with_bits` are +still needed regardless, for the arith-ID-probe merge a few lines below `best_other` — and for the exact +flagship shape (an arith-tuple leaf ANDed with the existential leaf), that merge is what produces the TRUE +joint intersection; the scalar from `popcount_with_bits` alone (before the merge) is only the existential +leaf's own bare count, not yet the answer this round needs. So the eval_planes call itself cannot be +skipped for the population that matters, and substituting the scalar alone (leaving bits untouched) would +only help a narrower, already-rare shape (zero arith children, `card_invariant` empty, one existential +leaf) that ingredient 3 would have covered for free anyway. Not shipped. + +**Ingredient 3** (fold `folded.result.card` in as a `.min()` floor at the final struct construction): +investigated and explicitly NOT shipped, because the SAME idea was already tried once before, for the +retired `domain_hint` field, and reverted — see the existing code comment directly above the final struct +construction (`card_engine/src/lib.rs`, ~line 7941 as of this round): *"a single BROAD leaf's own card +count... is not a safe upper bound on the whole `And` unless `narrow_rec` would actually USE that leaf to +narrow."* The concern is not mathematical (a single leaf's own count IS always ≥ the true joint) — it is +that `.card`/`domain_cards` is consumed downstream as an estimate of what the real EXECUTION PLAN will +visit, and `narrow_rec` may decline a broad leaf entirely (the documented `broad_ok: false` precedent, +`border:black` at 87%). Folding in a broad leaf's own count without the breadth guard the retired +`domain_hint` used to carry risks resurrecting that exact, already-fixed bug. Given the primary fix +(the gate) is independently sufficient and validated (below), this round left ingredient 3 out rather than +risk it — a future round could revisit it WITH an equivalent breadth guard, but that is new scope, not a +"free" addition. + +### Reproducers: before / after (real corpus, isolated release wheels, `costcell/trunk`@`cc17e031` vs this round) + +`unique=card`, `orderby=rarity`, `direction=desc`, `limit=175`, `offset=0`, `prefer=default`: + +| query | before `eval_domain` | before `cards_visited`/`eval_domain` | after `eval_domain` | after ratio | +|---|--:|--:|--:|--:| +| `cmc=1 border:black` | 4,893 | 0.624 | 3,052 | **1.000** | +| `cmc=1 border:white` | 2,756 | 0.113 | 311 | **1.000** | +| `cmc=1 r=mythic` | 4,710 | 0.021 | 101 | **1.000** | +| `cmc>=1 cmc<=5 border:white` | 2,756 | 0.641 | 1,766 | **1.000** | +| `cmc>=1 cmc<=5 r=mythic` | 4,710 | 0.391 | 1,842 | **1.000** | +| `cmc>=1 cmc<=5 border:black` | 24,734 | 1.088 | 24,734 | 1.088 (unchanged — already tightened via `arith_tuple_count`, a separate mechanism; the gate fix additionally now runs the arith-ID-probe merge here too, but border:black's near-100% selectivity means it doesn't move the number) | + +Every reproducer Round 21 named — `cmc=1 border:black`, `cmc=1 border:white`, and a rarity case +(`cmc=1 r=mythic` / `cmc>=1 cmc<=5 r=mythic`) — moves to an EXACT `eval_domain` (ratio 1.000). The +two-sided flagship shape (`cmc>=1 cmc<=5 border:black`) was already in a defensible band via a different, +unaffected mechanism and stays there. + +### Broader sweep: is the confound Round 20 hit now cleared, broadly? + +Round 20's own blocker was precise: of 223 systematically-varied population-A rows (3 arith fields × 7 +widths × 11 existential leaf values), only 13 (5.8%) landed within 15% of real `cards_visited` — +`border:black` was "the only leaf value tested where `eval_domain` is trustworthy." Re-ran the identical +sweep (3 fields × 13 widths × 11 leaf values, 393 successful rows) against this round's fixed wheel: + +``` +total rows: 393, within 15% of cards_visited: 351 (89.3%) -- was 13/223 (5.8%) before this round + +border:black n=39 median 1.000 100% within 15% except width-13 outlier (82.1% — border:black's + own near-universal count) +border:white/borderless/gold, r=common/uncommon/rare/mythic, f:oldschool: + n=39 each, median 1.000, 100% within 15% -- every one of these was badly wrong + (0.02-0.85) before this round +r=special n=39 median 0.699 10.3% within 15% -- STILL broken, but a DIFFERENT, already-known + bug (see below), not this round's gate +``` + +The gate fix clears the confound for every leaf value tested except `r=special` (0.012 bare selectivity, +325 of 31,724 cards). Traced directly: `r=special`'s `eval_domain` reads a FLAT 219 across `cmc<=1` through +`cmc<=9` (only changing once the range widens enough to include ALL 325 matching cards) — this is the +SAME, separately-documented gap Round 17/19/20 already flagged in passing ("`eval_domain` reads identical +across every range width for several minority existential leaf values... a gap in the +`card_invariant_domain_exact`/estimated-domain fallback"), not a symptom of the `best_other` gate this +round fixed. Out of this round's blast radius (a different mechanism, `acquire_plan_features`'s domain +fallback, not `compose_printing_estimate`'s `And` arm) — flagged here as still open, not chased. + +### Pre-computation check: acquire-time cost, measured directly + +The gate fix makes the SAME already-existing `popcount_with_bits`/arith-ID-probe-merge machinery run for +MORE query shapes than before (previously gated off whenever `card_invariant` was empty) — this is real, +measurable added cost for the newly-covered population specifically, not a new mechanism. Measured +directly (20 warmups, 200 trials, isolated release wheels, `explain_analyze`'s own `acquire_ns`): + +| query | before (median) | after (median) | delta | +|---|--:|--:|--:| +| `cmc=1 border:black` | 709 ns | 42,083 ns | +41,374 ns | +| `cmc=1 border:white` | 709 ns | 6,125 ns | +5,416 ns | +| `cmc=1 r=mythic` | 1,000 ns | 6,583 ns | +5,583 ns | +| `cmc>=1 cmc<=5 border:white` | 4,833 ns | 26,500 ns | +21,667 ns | +| `cmc>=1 cmc<=5 r=mythic` | 5,125 ns | 28,166 ns | +23,041 ns | +| `cmc>=1 cmc<=5 border:black` | 4,959 ns | 92,791 ns | +87,832 ns | + +This is NOT from `popcount_with_bits`'s `eval_planes` call itself (a fixed ~496-word bitmap AND, the same +small cost the mechanism has always paid when it ran) — traced to the arith-ID-probe merge a few lines +below `best_other`, now reached for this population for the first time. That merge's cost scales with the +arith-tuple leaf's own selectivity (`bare_numeric_field_ids`/`arith_tuple_ids` materializes one `Vec` +of matching card ids, then filters/sums over it) — `cmc>=1 cmc<=5` (77% of all cards) costs far more than +`cmc=1` alone (10% of all cards), matching the table above. This merge is NOT optional: it is what produces +the TRUE joint intersection (e.g. `cmc=1 ∩ border:white` = 311 cards, not border:white's own 2,059-card +bare count) — without it, the fix would only partially tighten `eval_domain`, not close it to exact. + +Confirmed the added cost is SCOPED to the newly-covered population, not a general regression: four +unrelated queries that already had a card-invariant partner (so `best_other` already fired before this +round) show no measurable acquire-time change: `c:w cmc<=3` 667ns→666ns, `f:modern c:u` 7,250ns→6,833ns, +`t:elf` 500ns→542ns, `devotion:w c:u usd>5` 7,500ns→7,833ns (15 warmups/100 trials each; all within noise). + +This is a real, bounded-but-non-trivial trade for a narrow, previously-mis-costed population — accepted +because (a) it reuses existing, already-designed-for-this-purpose machinery rather than adding anything +new, (b) it is invisible in whole-corpus aggregates (below), and (c) the population it fixes was previously +driving `eval_domain` off by up to 47x (`cmc=1 r=mythic`, 0.021 ratio), which is the more consequential +error for routing. + +### Correctness gate + +`cargo test --manifest-path card_engine/Cargo.toml --release`: **173/173 passed** (172 pre-existing + 1 new +regression test, `compose_and_arm_tightens_lone_existential_leaf_with_no_card_invariant_partner` in +`tests.rs`, reusing the existing `cmc_border_existential_fixture_store` fixture — asserts the AND's exact +card intersection (`Some(2)`) and printing span (`3`) directly via `compose_printing_estimate`, for the +specific "arith range AND one existential leaf, no other card-invariant leaf" shape. Confirmed the test +actually catches the regression by temporarily reverting the gate: fails with `left: None, right: Some(2)` +against the old `!card_invariant.is_empty()` guard, as expected). Every existing assertion unchanged. +`cargo test --manifest-path card_engine/Cargo.toml` (debug, with debug-assert tripwires): 173/173 passed. +`cargo clippy --manifest-path card_engine/Cargo.toml --all-targets -- -D warnings`: clean. + +### Broader regression check (mandatory — this touches the same `And` arm Rounds 1-9 validated for a +### different population) + +`bench_cost_model_agreement.py --seconds 300 --seed 0`, full table, baseline vs fix: + +``` +overall: 11/17 cells inside [0.8, 1.25] -> 12/17 cells inside [0.8, 1.25] (improved, not regressed) +GatheredScan/card: 0.84 (26% within 25%) -> 0.86 (27% within 25%) (moved toward 1.0) +GatheredScan/printing_compose: 1.18 (24%) -> 1.22 (24%) (flat, within noise) +``` + +No cell flips from PASS to FAIL. One cell (`PrintingCompose`/`plane`) moves from 17% to 30% within-25%, +both still below the 80% pass bar — not a regression, a small improvement. Total sampled queries in the +same 300s window: 101,108 → 97,253 (−3.8%), consistent with the measured acquire-time cost above diluted +across the WHOLE uniform sample (most queries in the sample never touch this population at all). + +`bench_pairwise_ordering.py --seconds 300`, `GatheredScan` vs `PrintingCompose`/`StreamedSelect`, both +modes, baseline vs fix: + +``` +realistic: GatheredScan vs PrintingCompose 88% ordered right both, regret 11.90µs->11.89µs (flat), + gap meas/pred 0.91 -> 1.01 (moved to near-exact) + GatheredScan vs StreamedSelect 97% both, 0.86µs->0.87µs (flat), 1.04->1.04 (unchanged) +uniform: GatheredScan vs PrintingCompose 88% both, 7.27µs->7.03µs (flat), 1.03->1.06 (flat) + GatheredScan vs StreamedSelect 95% both, 1.88µs->1.73µs (flat), 1.07->1.07 (unchanged) +``` + +`bench_regret_matrix.py --seconds 120 --mode realistic --seed 0`: baseline total regret 38.8ms over 50,549 +queries (mean 0.77µs) -> fix 42.9ms over 52,944 queries (mean 0.81µs), +5.2% mean, comparable to prior +rounds' own sample-to-sample noise band (Round 15/16 reported similar ±0.5-4% swings from re-sampling +alone). One new single-query outlier (max regret 1,927.6µs in a `StreamedSelect -> PrintingCompose` +misroute category that existed in baseline too, just with a smaller max there, 95.3µs) — likely a +different rare query landed in the differently-sized random walk (same seed, but per-query timing +differences shift how many queries QuerySampler draws in the same wall-clock budget); no NEW misroute +category appeared, and the "picked -> best" breakdown's set of categories is identical before/after. + +`bench_query_latency_ab.py --sample 400 --mode realistic --seed 7`, baseline vs fix, plus a same-build +canary at the same seed (sequential runs, not literally interleaved sub-second — see caveat below): + +``` +canary (base run 1 vs base run 2): B - A = +0.4µs 95% CI [+0.2, +0.5] +baseline vs fix: B - A = +1.0µs 95% CI [+0.6, +1.7] +``` + +The fix's interval does not fully overlap the canary's, but the two are close (0.6µs apart) at a sample +size the script's own docs flag as noisy for cross-process comparisons at these defaults (n=400). Given +the affected population's rarity (Round 21: ~0.85-1.23% of `Mode::Card` queries by a rough regex proxy, +an over-count relative to the exact AST shape), a small, real, borderline-detectable aggregate effect this +size is consistent with the acquire-time cost measured directly above, not a red flag on its own. + +### Step 5 (joint rate refit): blocker demonstrably cleared, refit itself deferred + +Round 20's own recommended next step was explicit: fix `eval_domain` first, then the `GATHER_CARD_PASS_NS`/ +`GATHER_RESIDUAL_FLOOR_NS`/leaf-count-rate joint refit becomes testable for real. The broad sweep above +confirms the blocker IS cleared for essentially the whole population Round 20 swept (89.3% of rows within +15% of `cards_visited`, up from 5.8%) — a real, load-bearing precondition for that future refit, not a +minor caveat. + +The refit itself was NOT attempted in this round. Building it properly means reintroducing Round 19/20's +`count_plane_leaves`/`plane_extra_eval_leaves` plumbing as a clearly-separate addition, a standalone fit +script mirroring `fit_round20.py`'s design, and the SAME three-population (compound/bare/residual) +held-out validation discipline Round 20 used — each of those was itself a full round's worth of work in +Rounds 19/20, and both of THOSE rounds' negative results (the additive term overshoots the flagship +reproducer even with a correct mechanism, Round 19; the floor's original calibration already unevenly +absorbs part of the compound-leaf effect, Round 20) were about the RATE FIT itself, not about `eval_domain` +— clearing the `eval_domain` blocker does not by itself imply the rate refit will now succeed. Rebuilding +that machinery and re-running the fit deserves its own dedicated round rather than being compressed into +this one's remaining scope. Per this round's own brief: "if the correctness fix (1-4) works but the refit +(5) doesn't [get attempted], ship the correctness fix alone" — done, with the next round now unblocked to +attempt the refit directly against trustworthy `eval_domain` data. + +### Commit + +One commit on `costcell/22-existential-and-fix` (gate fix + regression test + fixture addition; no +`cost.rs` changes, since step 5 was not attempted). `git diff --stat costcell/trunk`: `card_engine/src/lib.rs`, +`card_engine/src/tests.rs`, this doc. + + + +Round 20 of the `GatheredScan`-compound-plane effort +([done/local-engine-gathered-scan-undercosted-arith-existential-and.md](done/local-engine-gathered-scan-undercosted-arith-existential-and.md)) +found that three rounds of rate-fitting against `cmc>=1 cmc<=5 border:black`-shaped queries all failed +for the same reason: `eval_domain`/`domain_cards` (the acquire-time card-domain estimate every +`GatheredScan`/`StreamedSelect` per-candidate term multiplies) is itself wrong by up to 14x for this +population, and every rate fit was measured against a corrupted ground truth. This doc is that round's +named next step — fix the domain estimate first — and finds the exact code responsible, not just its +symptom. It is the third appearance of the same general problem: combining two range/existential leaves +into one accurate card-domain count was also the blocker in +[#852](00852-engine-compose-acquire-p3-p4-ranking.md)'s item 1, and an independence-product family of +fixes for a related (but distinct) shape was proven a dead end in +[local-engine-gathered-scan-card-printing-varying-depth.md](local-engine-gathered-scan-card-printing-varying-depth.md)'s +Round 2. + +No fix ships in this doc. Everything below was verified against the real corpus (`benchmarks/bitplanes/corpus.jsonl`) +via `engine.explain()`/`explain_analyze()` on an isolated release wheel built from `costcell/trunk`@`bb5798b2` +(Round 20's own commit), plus temporary `eprintln!` instrumentation in `compose_printing_estimate`/ +`acquire_plan_features` (reverted before this commit — `git diff --stat costcell/trunk` shows only this file). + +## Question 1: what type of queries are affected + +**The population is precisely: `Mode::Card`, `PrintingCompose` acquire, an `And` whose children are some +mix of arith-tuple leaves (`cmc`/`power`/`toughness`) and exactly one printing-varying existential leaf +(`border`/`rarity`/a divergent-format `legality`), with no OTHER card-invariant leaf (color, non-divergent +legality, devotion, ...) present.** Every dimension below was swept directly. + +### Fields and range width + +All three arith-tuple fields (`cmc`, `power`, `toughness`) show the identical pattern — this is not +`cmc`-specific. Width matters, but not in the way "a single value is exact" would suggest: it interacts +with which side of the `And` a plain per-child `min` picks (see Question 2), not with narrowness alone. +Swept widths 1 (`cmc=V`), 3, 5, and the full interior range (13) against 11 existential leaf values; every +width shows the same qualitative failure for every leaf except `border:black`. + +### Existential leaf families and values — the "near-universal" hypothesis, tested directly + +Bare-leaf selectivity in this corpus (`Mode::Card`, fraction of `n_cards=31,724`): + +| leaf | frac | +|---|--:| +| `border:black` | 0.989 | +| `border:borderless` | 0.110 | +| `border:white` | 0.065 | +| `border:gold` | 0.017 | +| `border:silver` | 0.000 | +| `r=rare` | 0.349 | +| `r=common` | 0.337 | +| `r=uncommon` | 0.324 | +| `r=mythic` | 0.083 | +| `r=special` | 0.012 | +| `f:oldschool` (the corpus's one divergent format) | 0.030 | + +No rarity value is near-universal (max 35%), and the corpus's only *divergent* legality format +(`oldschool`, the only format where "existential" even applies — a non-divergent format like `modern` or +`commander` is card-invariant and never reaches this code path at all) is itself a minority value (3%). +**Border is the only family in this corpus with a near-universal value at all**, so the brief's "find a +near-universal value in a different family" cannot be answered with a second clean field+value pair from +this corpus's own data — but the mechanism itself (Question 2) was confirmed directly by construction +instead: adding a near-vacuous 99.8%-selective leaf from a *different* family (`f:commander`, non-divergent +legality) to a broken query flips the same code path on and fixes it, which is a stronger and more direct +test than a second natural near-universal value would have been. Selectivity of the *discarded* side, not +field identity, is confirmed to be the true driver — see Question 2. + +Ratio (`GatheredScan.cards_visited / eval_domain`) across all 11 leaf values x 4 widths x 3 fields (33 +rows, `cmc>=1 cmc<=5`-shaped and narrower/wider): + +- `border:black`: 0.624 (width 1) to 1.268 (full range) — the only leaf that stays in a defensible band. +- Every other value (`border:white/borderless/gold`, `r=common/uncommon/rare/mythic/special`, + `f:oldschool`): 0.02 to 0.95, monotonically worse (lower) as the leaf's own selectivity drops, and + **always an over-estimate** (`eval_domain` too big) except for the degenerate `border:silver` case + (0 corpus matches at all — `eval_domain=0` while `cards_visited>0`, a separate, minor edge case in the + `range_too_broad_to_narrow`/zero-match interaction, not chased further here). + +### Mode + +Identical failure in `Mode::Printing` and `Mode::Artwork` — `eval_domain` for `cmc>=1 cmc<=5 border:white` +reads 2,756 in all three modes (`unique=card/printing/artwork`), and `Mode::Printing`/`Mode::Artwork` share +the exact same `domain_cards` computation `Mode::Card` reads (`acquire_plan_features`'s shared +`(eval_domain, scan_units)` tuple, keyed off `domain_cards` regardless of `mode`). Not card-mode-specific. + +### Acquire branch + +Confirmed acquire-branch-specific, matching Round 20's finding exactly: the same filter under +`orderby=name` (`Prep::Candidates`/`count_source=plane`, not `PrintingCompose`) reads `eval_domain` exactly +equal to `cards_visited` for all three test leaves (black/white/mythic), ratio 1.0 every time. The bug is +entirely a `compose_printing_estimate`/`acquire_plan_features`'s `PrintingCompose`-branch phenomenon, and +does not touch the `plane`/`Prep::Candidates` acquire's own (correct) domain computation. + +### Real-traffic representation + +A regex-based proxy over `QuerySampler` (40,000 draws each, `uniform`/`realistic`) matching queries that +mention both an existential-family leaf and an arith-tuple comparison anywhere in the text: 1.7-2.4% of all +sampled queries, ~0.85-1.23% in `Mode::Card` specifically. This over-counts the actual bug population, +because (per Question 2) adding *any other* card-invariant leaf — color, a non-divergent format, devotion — +cures it; a realistic query combining `f:modern c:w cmc<=3 border:black`-style leaves would not hit this +bug at all. Round 20's own, more rigorous measurement of the closely-related `plane_extra_eval_leaves` +population (60,000 combined draws, both modes) found **zero** naturally-sampled rows — this doc's +population is a superset of that one (it doesn't require the leaf-count feature, just the domain +corruption), but is still a narrow, hand-constructible shape rather than one `QuerySampler` reliably hits. +Confirms and refines Round 20's finding rather than contradicting it: real but rare, and rare specifically +because most real queries carry an incidental card-invariant leaf that happens to fix the bug as a side +effect, not because the AST shape itself is exotic. + +## Question 2: the mechanism — confirmed root cause, not just a symptom + +### The bug, precisely + +`compose_printing_estimate`'s `And` arm (`card_engine/src/lib.rs:7680`) computes an exact joint card +count only through `best_other` (line 7840): + +```rust +let mut best_other: Option<(usize, Vec)> = None; +if existential.is_empty() { + if card_invariant.len() >= 2 { best_other = Some(popcount_with_bits(None)); } +} else if !card_invariant.is_empty() { + for e in &existential { + let candidate = popcount_with_bits(Some(e)); + if best_other.as_ref().is_none_or(|(c, _)| candidate.0 < *c) { best_other = Some(candidate); } + } +} +``` + +`card_invariant`/`existential` are populated at line 7801 by filtering OUT every arith-tuple-eligible +child (`cmc`/`power`/`toughness` are excluded from both) and partitioning the rest by +`plane_expr_is_existential`. **For the flagship reproducer's own minimal shape — one or more arith leaves +plus exactly one existential leaf and nothing else — `card_invariant` is empty and `existential` has one +element, so *neither* branch of the `if`/`else if` fires: the `else if !card_invariant.is_empty()` guard +requires a card-invariant partner that a lone existential leaf does not need.** `best_other` stays `None` +for the rest of the function, so `exact_domain_cards` (and everything downstream: `est.result.card`, +`domain_cards`'s `is_and` tightening, `card_invariant_domain_exact`) never gets an exact answer — even +though `popcount_with_bits(Some(e))` (line 7818) works fine with an empty `card_invariant` vec; it is +never *called* for this shape, not incapable of answering it. + +Confirmed directly with a temporary `eprintln!` at line 7852 (right after the `if`/`else if` block, +reverted before commit): for every one of `cmc=1 border:white`, `cmc=1 border:white f:commander`, +`cmc=1 border:black`, `cmc>=1 cmc<=5 border:black`, `cmc>=1 cmc<=5 border:white`, `cmc>=1 cmc<=5 r=mythic` +— `card_invariant.len()==0`, `existential.len()==1`, `best_other.is_some()==false`, in every case with no +OTHER card-invariant leaf in the query. + +### Falsifiable test, run directly: does adding a card-invariant partner fix it? + +Yes, cleanly, and the fix does not need the partner to be selective — a near-vacuous one works just as +well as a real one, which is exactly what the "gating bug, not an accuracy bug" diagnosis predicts: + +| query | `card_invariant.len()` | `best_other` | `eval_domain` | `cards_visited` | ratio | +|---|--:|:--:|--:|--:|--:| +| `cmc=1 border:white` | 0 | false | 2,756 | 311 | 0.113 | +| `cmc=1 border:white f:commander` (99.8% selective, near-vacuous) | 1 | true | 309 | 311 | **1.006** | +| `cmc=1 border:white f:modern` (70.8% selective, a real constraint) | 1 | true | 127 | 127 | **1.000** | +| `cmc=1 border:black` | 0 | false | 4,893 | 3,052 | 0.624 | +| `cmc=1 border:black f:commander` | 1 | true | 3,044 | 3,068 | **1.008** | + +The last row is the sharper point: even `border:black` — the leaf every prior round called "clean" — is +*not* actually well-estimated at width 1 (ratio 0.624) once you isolate the shape from the wider-range +case. It only reads "clean" for the `cmc>=1 cmc<=5`-shaped flagship reproducer specifically, and that +cleanliness comes from a second, *unrelated* coincidence (below), not from `best_other` firing — `best_other` +is confirmed `false` there too. Once ANY card-invariant partner is present, `best_other` fires and the +estimate becomes essentially exact for every leaf tested, `black` included. + +### Why `border:black` looks clean anyway, for the specific `cmc>=1 cmc<=5` shape + +`acquire_plan_features`'s `domain_cards_before_card` (line 12147) does not even read the `best_other` +path's output when it fires — it reads `est.candidate.printing`/`est.result.printing` instead: + +```rust +let domain_cards_before_card = if est.candidate.printing == est.result.printing { + est_cards +} else { + calibrated_balls_into_bins(est.candidate.printing, n_cards as usize) +}; +``` + +For a 2-sided range (`cmc>=1 cmc<=5`, two arith children), the *printing-space* value `result` gets +tightened by a **separate**, already-working mechanism (`arith_tuple_count`, an exact `#743` index scan +over 2+ arith children — unaffected by the `best_other` bug, since it never touches `card_invariant`/ +`existential` at all) — but `candidate` never receives that tightening (`candidate` is deliberately the +untightened per-child `min`, "what narrow_rec actually leaves the alternatives to walk" per the function's +own doc). Confirmed via `eprintln!`: `cmc>=1 cmc<=5 border:black` has `est.candidate.printing=85,411` vs +`est.result.printing=83,894` (NOT equal), so `domain_cards_before_card` takes the `calibrated_balls_into_bins` +branch on the **untightened** 85,411, not the tightened 83,894. That untightened number is itself just +`min(cmc>=1's own printing count, cmc<=5's own printing count, border:black's own printing count)` — and +it happens to land close to the truth here purely because **whichever side the plain per-child `min` +discards is, for `border:black` specifically, close to 100% selective, so discarding it costs almost +nothing.** For every other leaf tested, the discarded side is a real minority constraint, and discarding +it is exactly the over-estimate measured in Question 1. This is the same "selectivity of the discarded +side" mechanism as the `best_other` gate, arrived at through a completely different code path — two +independent coincidences, not one robust mechanism, which is why `border:black` alone (width 1, no second +arith child) is *not* clean (ratio 0.624 above) even though the wider-range reproducer is. + +### A third, free, already-computed ingredient the fold already carries and discards + +Checked (via a second `eprintln!`, also reverted) whether the per-child fold that builds `folded` (line +7714, `children_estimates.iter().fold(...)`, `SpaceEstimate::min` at line 7417) already carries a +useful `.card` value before `best_other` ever runs. It does, for **border** specifically: `border`'s own +leaf arm in `compose_printing_estimate` calls `exact_result_total(filter, indexes, Mode::Card)` (which +hits `vt.border`, a precomputed exact 3-space per-value table — O(1)/O(log n), no bitmap) and returns it +via `ComposeEstimate::leaf_spaces`, so `folded.result.card` already holds `min` of every child's own exact +card count wherever one exists. Confirmed directly: `cmc>=1 cmc<=5 border:white` → `folded.result.card = +Some(2,059)`, exactly `border:white`'s own bare card-match count. **But this value is thrown away +regardless of whether `best_other` fires** — the final struct literal at line 7948, +`ComposeEstimate { result: result_space, exact_domain, ..folded }`, always sets `.card` from +`result_space` (built from `exact_domain_cards`, `best_other`'s output, `None` here), never falls back to +`folded.result.card` when that's `None`. This is free (no new probe, already computed today) and would be +a strict tightening (an individual child's own exact count is always ≥ the true joint intersection, so +`.min()`-ing it in can only help) — but it caps out at "the tightest single child's own marginal count," +not the true joint intersection, so it is a partial complement to fixing `best_other`, not a substitute +for it. **Rarity does not currently have this same free ingredient**: its own leaf arm +(`compose_printing_estimate`, `NumericCmp{RarityInt}`) deliberately uses `ComposeEstimate::leaf` (not +`leaf_spaces`), leaving `.card`/`.artwork` at `None` — its own comment cites a documented, pre-existing +bug in `RangeCardCounts::distinct_cards` for **broad** comparisons (`r<=mythic` read 31,722 against a true +31,724). Whether that bug also affects a narrow `Eq` value like `r=mythic` specifically was not +re-verified here — flagged as open below, not assumed either way. + +### Hypothesis, stated falsifiably, and the verdict + +**Hypothesis**: `domain_cards`/`eval_domain` for this population is not "estimated," it is a plain `min` +over each individual child's own marginal count (via one of two independent code paths — `best_other`'s +gate, or `domain_cards_before_card`'s untightened `candidate` fallback), which silently discards whichever +side of the `And` the `min` doesn't pick — and the estimate reads as "accurate" if and only if the +discarded side happens to be near-100% selective (so discarding it costs little), regardless of which +family or field that side belongs to. + +**Test**: constructed queries where the card-invariant partner is deliberately near-vacuous (`f:commander`, +99.8%) versus genuinely selective (`f:modern`, 70.8%) alongside a badly-broken leaf (`border:white`). +**Confirmed**: both restore `best_other` and make `eval_domain` exact (ratio 1.006 and 1.000 +respectively) — the fix works whether or not the added partner narrows anything, because it is a *gating* +fix (does an exact joint popcount run at all), not an *accuracy* fix for an existing estimate. Also +confirmed the corollary: `border:black` itself is NOT reliably clean absent the second-arith-child +coincidence (ratio 0.624 at width 1, becoming 1.008 once a card-invariant partner is added) — refuting the +version of the hypothesis that would say "border:black is intrinsically well-modeled." It isn't; it's +lucky, twice, in slightly different ways depending on range width. + +## What a fix would need to do (a sketch, not a design) + +Two complementary ingredients, both confirmed as real by the data above, likely both wanted together: + +1. **Drop the `!card_invariant.is_empty()` requirement in `best_other`'s `else if` branch** (line 7845), + so a lone existential leaf (no card-invariant partner) still gets its own exact popcount via + `popcount_with_bits(Some(e))` with an empty `card_invariant` vec — this is the direct fix for the + *gating* bug, confirmed to work by the `f:commander`/`f:modern` natural experiment above (which works + *because* it flips this exact gate, not because those queries are special). + +2. **Prefer an existing precomputed exact count over a fresh `eval_planes`+`popcount`, where one exists**, + rather than materializing a bitmap purely to get a scalar. `exact_result_total` already has `vt.border` + (a direct O(1) 3-space lookup, confirmed used by `border`'s own leaf arm already) and `vt.legality` + (`legality_totals_key`-keyed, same shape, confirmed to exist for legality too). Whether rarity has an + equally safe equivalent for a narrow `Eq` value specifically (as opposed to the documented-broad-range + bug in `RangeCardCounts::distinct_cards`) is unresolved — worth checking directly before relying on it, + not assumed by this doc. The BITS (needed separately for the arith-ID-probe merge a few lines below + `best_other`) may or may not need a fresh `eval_planes` call at all if a single leaf's compiled + `PlaneExpr` already resolves to a direct slice of `indexes.planes.words[...]` — not confirmed here, + worth a look before assuming the cheap-count path and the bits path have to be the same call. + +3. **Stop discarding `folded.result.card`/`folded.candidate.card` at the final struct construction** + (line 7948) when `exact_domain_cards` is `None` — a free, already-computed, strictly-safe `.min()` + floor (an individual child's own exact count is always ≥ the true joint), covering border today and + any other leaf whose own arm already populates `.card`, with zero new per-query cost. This is a partial + complement to (1)/(2), not a substitute — it only ever tightens to "the best single child's own count," + never to the true joint intersection two-or-more constraints would give. + +Whichever combination ships, the next round should re-run the exact `fit_round20.py` joint-refit protocol +Round 20 built (design fully specified in that doc, not checked in) — once `eval_domain` is trustworthy +across leaf values, the `GATHER_CARD_PASS_NS`/`GATHER_RESIDUAL_FLOOR_NS`/leaf-count-rate joint fit Rounds +19-20 couldn't validate becomes testable for real, on the same sample construction already built for that +purpose. + +## Open questions / what's still uncertain + +- **Does rarity's known `RangeCardCounts::distinct_cards` bug (documented for broad ranges) also affect a + narrow `Eq` value?** Not re-verified here; `compose_printing_estimate`'s rarity leaf arm blanket-disables + `.card` for the whole `NumericCmp{RarityInt}` family regardless of comparison operator, so this doc + cannot tell whether that blanket is itself over-broad. +- **Legality's own `best_other` behavior for a divergent format was not separately re-verified past + Round 15/16's existing fix** (`plane_expr_is_existential`) — this doc's data is entirely border/rarity; + `f:oldschool` was swept in Question 1's ratio table but not independently traced through `best_other` the + way border/rarity were. +- **Whether the arith-ID-probe merge's bits can come from a stored slice instead of a fresh `eval_planes` + call** (ingredient 2's second half) is a real-cost question for whoever implements the fix, not answered + by this investigation round. +- **The real-traffic frequency estimate (Question 1) is a rough regex proxy**, not an AST-level + classification of "empty card_invariant" — likely an over-count relative to the exact bug population, + for the reason stated (an incidental card-invariant leaf elsewhere in a real query cures it as a side + effect). A precise count would need to instrument the actual gate, not query text. diff --git a/docs/issues/local-engine-gathered-scan-card-printing-varying-depth.md b/docs/issues/local-engine-gathered-scan-card-printing-varying-depth.md new file mode 100644 index 000000000..1a6b940f5 --- /dev/null +++ b/docs/issues/local-engine-gathered-scan-card-printing-varying-depth.md @@ -0,0 +1,1902 @@ +# GatheredScan/card: Printing-Varying Leaf Scan Depth + +Extracted from item 1 of +[local-engine-cost-model-cleanup-remaining.md](local-engine-cost-model-cleanup-remaining.md) once it +became an ongoing iteration ledger rather than a single-pass fix. Base branch for all work here is +`engine-cost-model-cleanup`, never `main`. + +## Problem + +**Population**: `GatheredScan`/`unique=card` is the worst-agreement, highest-frequency cell in the +whole cost model — by frequency alone it dominates routing regret more than any other cell. + +**What we already know**: the shape-level breakdown (bucketing `printing_compose`/card-mode queries by +AST shape) found `and-2` and `and-3` — specifically pairs/triples over printing-varying fields +(`price_eur`, `price_usd`, `price_tix`, `collector_number_int`, `released_at`, and mixed pairs like +`card_color_identity + price_tix`) — carrying the bulk of the remaining magnitude-weighted error +(tens of millions of units each, at investigation time). These don't qualify for the prior session's +card-invariant depth-1 fix: a printing-varying field has no "first printing settles it" guarantee (a +card matching `price_usd<5` can have OTHER printings that don't), so they still fall through the flat +`domain_cards * printings_per_card * COMPOSE_CANDIDATE_SPAN_BIAS` fallback in the `scan_all` closure +(`card_engine/src/lib.rs:11616`; constant at `:11041`, currently 2.1) — and that formula prices every +card as if it needed its *average* reprint history walked, regardless of how selective the predicate +actually is at the printing level. + +Two starting-point ideas, either or both may end up in the ledger below: + +1. **Match-density depth proxy.** The query's own printing-level match density (`matches / + domain_cards` — average number of matching printings per matching card) is a much better proxy for + expected scan depth than the corpus-wide `printings_per_card` average. A per-card-first-match + expectation, using order statistics on the position of the first match among a card's printings, is + the natural model: `expected_depth ≈ (avg_printings_per_card + 1) / (avg_matches_per_matching_card + + 1)`, capped at the card's own span. `COMPOSE_CANDIDATE_SPAN_BIAS` was fit against the OLD flat-average + shape and should be re-derived (likely much closer to `1.0`) once the depth term itself carries real + selectivity information, not stacked on top of the new term unchanged. +2. **Per-leaf independence-product combination.** A generalization of (1) to multiple printing-varying + leaves at once: combine each leaf's own printing-level selectivity via an independence product + (with a fudge factor) rather than a single aggregate depth term — see Constraints below for why this + needs an explicit correlation guard before it can be trusted. + +## Constraints + +- **Pre-computation over hot-path computation, hard requirement.** This repo has a specific, measured + precedent for what goes wrong otherwise: relaxing `compose_printing_estimate`'s `best_other` + intersection threshold from `>=2` to `==1` closed a logical gap but caused a **23.6x acquire-time + regression** (875ns → 20,646ns median) on the newly-admitted population, because it added an + unconditional `eval_planes`/`popcount_with_bits` pass paid by every query in that population + regardless of whether the tightening ever changed the routing decision. Reverted; see + [local-engine-cost-model-cleanup-remaining.md](local-engine-cost-model-cleanup-remaining.md)'s + "Explicitly considered and rejected" section for the full account — link, don't restate it. Any new + idea here must trace every new number to an existing precomputed index/table/constant, not a new + per-query scan whose cost grows with match/printing/candidate count. +- **Price-triple correlation risk.** `price_usd`, `price_eur`, and `price_tix` are near-identical market + values expressed in different currencies/units — they are NOT independent. An independence-product + combination across this triple (or any pair of them) will badly underestimate the true joint count. + Any independence-style idea must be explicitly tested against this triple before being trusted, not + just against `collector_number_int`/`released_at`-shaped queries. (Power/toughness correlation is + already handled exactly elsewhere via `arith_tuple_count` — not a risk in this population, no need to + re-verify it here.) +- **Out of scope, hard**: `card_engine/src/estimator.rs` (its `estimate_cardinality` is live at + `lib.rs:11146` behind the `STREAM_MIN_MATCHES` gate — editing it can move a shipped routing decision, + and its `compose_and` independence estimator is unwired PR1 of #702, validated for soundness only). + Items 2-4 of the parent punch-list doc. `Mode::Printing`/`Mode::Artwork`. Anything outside `lib.rs`, + `cost.rs`, `tests.rs`, and this doc. + +## Current best + +As of Round 0 (baseline, `engine-cost-model-cleanup` @ `97dc30c8`), nothing from this doc has shipped +yet — the fix is still the flat fallback described above. Baseline measured against an isolated release +build (`maturin build --release`, extracted wheel, `PYTHONPATH`-pinned — never `maturin develop` into +the shared `.venv`, which silently redirects every other session's `import card_engine`): + +``` +GatheredScan card n=35,074 median 0.67 p10 0.25 p90 2.75 16% within 25% FAIL +``` +(`.venv/bin/python scripts/bench_cost_model_agreement.py --seconds 300 --seed 0`, run from a +`costcell/00-baseline` worktree branched off `engine-cost-model-cleanup`.) + +As of Round 1 (match-density depth proxy, `costcell/01-depth-proxy`), the flat fallback is replaced by +`domain_cards * expected_depth * COMPOSE_CANDIDATE_SPAN_BIAS` where `expected_depth = (printings_per_card ++ 1) / (density + 1)`, `density = printing_matches / domain_cards`, and `COMPOSE_CANDIDATE_SPAN_BIAS` is +refit to `0.7`. Same protocol: + +``` +GatheredScan card n=33,944 median 0.72 p10 0.25 p90 3.20 17% within 25% FAIL +``` + +Still FAIL by the [0.8, 1.25] median bar — see Round 1 below for why the whole-cell number barely +moves despite a real, controlled improvement in the feature itself. + +**As of Round 9 (`costcell/trunk` @ `58eebfdc`), the cell has crossed from FAIL to PASS** — the first +time since this doc opened. Independently re-measured (fresh isolated build, same protocol, not just +the shipping round's own self-reported numbers): + +``` +GatheredScan card n=35,132 median 0.81 p10 0.44 p90 2.99 26% within 25% PASS +``` + +Nine rounds landed six kept, held-out-validated fixes (Rounds 1, 3, 4, 6, 7, 9) and one clean +mathematical rejection (Round 2), spanning two previously-separate root causes: the printing-varying +range-leaf family (`compose_printing_estimate`/`scan_all`'s feature estimation, `lib.rs`) and, as of +Round 9, `GatheredScan`'s own cost FORMULA (`cost.rs`) under-charging zero-match `candidates`-acquire +queries by ~4x. The 26%-within-25% figure is still well short of the 90%-within-10% aspiration this +doc opened with — Round 8's diagnostic identified two more concrete, unaddressed mechanisms +(card-mode's unconditional `matches = count` ignoring residual selectivity, and an `Or`/negation +population invisible to this benchmark's flat-conjunction sampling) as the next candidates. + +As of Round 3 (`COMPOSE_RANGE_AND_CLUSTER_BIAS`, `costcell/03-cluster-bias`), the `est_cards` fallback +for an `And` of 2+ different-index printing-varying range leaves (the ~37% subset Round 1 identified +as ceiling-capped, and Round 2 proved no independence-product combination can fix) uses its own +clustering-bias constant, `1.1`, instead of `COMPOSE_CARD_ESTIMATE_BIAS`'s `1.78`. Held-out paired-diff +(1,500 and2/and3 RANGE_FAMILIES queries, `unique=card`, hash-of-query split): 433 improved / 117 +regressed / 192 tied, total absolute `scan_units` error 8.60M → 8.02M on the held-out half. Same +`GatheredScan`/`card` FAIL as before on the single-run agreement gate — see Round 3 below for why that +is expected and not a sign the fix did nothing. + +As of Round 4 (`COMPOSE_RANGE_AND_BROAD_SCAN_SCALE`, `costcell/04-broad-guard`), the LATER +`range_too_broad_to_narrow` guard's full-corpus reset (see Round 3's mid-investigation finding above) +scales `scan_units` alone down to `0.7 * n_printings` whenever `is_cross_index_range_and` holds -- +`eval_domain` is left at the full `n_cards`, confirmed exact (0 total absolute error against the real +`cards_visited` counter) for this population, not merely assumed. Held-out paired-diff (372/1,500 +guard-fired rows, hash-of-query split): 166 improved / 15 regressed / 0 tied, total absolute +`scan_units` error 5.40M → 1.90M on the held-out half. Same `GatheredScan`/`card` agreement-gate FAIL +as before (16%, unchanged) — this guard-fired subset is a small slice of that pooled cell, same +reasoning as Round 3. + +As of Round 6 (`COMPOSE_BARE_RANGE_BROAD_SCALE`, `costcell/06-bare-range`), the `CardRangePopcount` +arm's OWN `range_too_broad_to_narrow` reset -- a bare single range leaf under `unique=card`, a +completely separate acquire branch from Rounds 3/4's `PrintingCompose` target, and Round 5's +diagnostic finding of the single largest bucket in the whole pooled cell (53.7% of error) -- scales +`scan_units` alone down to `0.43 * n_printings`. `eval_domain` is left untouched: 96.6% of rows read +exactly 1.0 (mean 0.975; the small tail is real, driven by price-field null-exclusion, not chased). +Held-out paired-diff (3,500 guard-fired rows, hash-of-query split): 1,704 improved / 31 regressed, +total absolute `scan_units` error 93.3M → 16.0M on the held-out half. Same `GatheredScan`/`card` +agreement-gate result as Rounds 3/4 (15-17%, essentially unchanged) -- this is the largest single +lever fixed so far by pooled-error share, and it still barely moves the headline number, confirming +that gate's grain is simply too coarse to see any single arm's fix, not that this fix is inert. + +As of Round 7 (`COMPOSE_SAME_RANGE_BROAD_SCAN_SCALE`, `costcell/07-candidates-range`), `PrintingCompose`'s +OWN `range_too_broad_to_narrow` reset -- the rest of Round 5/6's "single:range" bucket that Round 6's +`CardRangePopcount` fix never reaches, because a bare range fails `card_range_popcount_applicable` +whenever no sort permutation exists for the query's orderby/direction, and a fused two-sided bound +(`eur>=0.23 eur<=0.45`) never reaches `CardRangePopcount` at all -- scales `scan_units` alone down to +`0.52 * n_printings`, gated on a new `is_same_index_range_only` (bare leaf or same-field `And`, as +opposed to `is_cross_index_range_and`'s different-index `And`). `eval_domain` is left untouched: +exact at 1.000 mean/median. Held-out paired-diff (13,053 guard-fired rows, hash-of-query split): 6,422 +improved / 33 regressed, total absolute `scan_units` error 304.8M → 57.6M on the held-out half. Same +`GatheredScan`/`card` agreement-gate result as every prior round (17-18%, essentially unchanged) -- +this population turned out to be even larger by row count than Round 6's, and still barely moves the +headline number, same grain argument as Rounds 3/4/6. + +As of Round 9 (`GATHER_FIXED_COST_ZERO_MATCH_NS`, `costcell/09-zero-match`), `PhysicalPlan::GatheredScan`'s +cost arm (`cost.rs`, not `lib.rs` — the first fix in this doc that lives in the cost FORMULA rather +than feature estimation) charges `42.0` instead of `GATHER_FIXED_COST_NS` (`169.6`) whenever `matches +== 0`, gated the same way the arm's own `tier_ns > 0.0` neighbor already is. Targets Round 8's +mechanism 1: a `Prep::Candidates`-acquired zero-match round collapses every OTHER term in the arm to +zero, so the whole prediction used to read as `GATHER_FIXED_COST_NS` alone — confirmed independently +(fresh 31,030-row sample, 9,890 zero-match, every non-fixed term exactly 0). Held-out paired-diff +(hash-of-query split, 9,890 zero-match rows): calibration half (n=4,944) sets the constant to its +median measured `plan_self_ns`, 42.0; held-out half (n=4,946) reads 4,577 improved / 369 regressed / 0 +tied, total absolute ns error 530,256 → 103,110 (5.1x), median ratio 0.248 → 1.000, within-25% 0.1% → +57.7%. `GatheredScan`/`card` agreement-gate cell moves from 11% to 30% within [0.8, 1.25] (median 0.57 +→ 0.77) — still FAIL by the median bar (0.77 < 0.8) but the largest single-round movement of this +number since Round 0, unlike every range-family round's "same 15-18%, unchanged" result; the by-unique +`GatheredScan`/`card` cell flips FAIL → PASS (0.69 → 0.80). See Round 9 below for the residual-risk +caveat this round found and verified as immaterial (a shared-`PlanFeatures` edge case in the +RANGE_ACQUIRES-forced-competitor population, checked against `bench_regret_matrix.py` and found to move +total regret by 0.0 ms). + +## Iteration ledger + +| # | Idea | Outcome | GS/card within-25% | Other cells | Notes | +|---|------|---------|--------------------|-------------|-------| +| 0 | (baseline, `engine-cost-model-cleanup` @ `97dc30c8`) | — | 16% | — | n=35,074, median 0.67, p10 0.25, p90 2.75 | +| 1 | match-density depth proxy | kept | 16% → 17% (noisy, uncontrolled) | none, within run-to-run noise | paired-diff (controlled): 946 impr / 544 regr, 29.6M → 9.86M abs `scan_units` error; `BIAS` refit 2.1 → 0.7 | +| 2 | independence-product `domain_cards` for 2+ different-index range leaves | rejected at self-check | n/a (no code shipped) | n/a | printing-space variant: 38 impr / 496 regr, 17.3M → 18.1M abs error (worse); card-space variant: 0/1500 changed (mathematically incapable of firing) — see Round 2 below | +| 3 | second clustering-bias constant (`COMPOSE_RANGE_AND_CLUSTER_BIAS`) for the same shape | kept | n/a (see Round 3 below — noisy at this cell's grain) | none, within run-to-run noise | held-out paired-diff (controlled): 433 impr / 117 regr, 8.60M → 8.02M abs `scan_units` error; new bias 1.1 against `COMPOSE_CARD_ESTIMATE_BIAS`'s 1.78 | +| 4 | downward `scan_units` scale (`COMPOSE_RANGE_AND_BROAD_SCAN_SCALE`) for the `range_too_broad_to_narrow`-fired subset of the same shape | kept | n/a (see Round 4 below — noisy at this cell's grain) | none, within run-to-run noise | held-out paired-diff (controlled): 166 impr / 15 regr / 0 tied, 5.40M → 1.90M abs `scan_units` error; new scale 0.7; `eval_domain` left untouched (measured exact, 0 error) | +| 5 | diagnostic: re-bucket remaining error by AST shape | diagnostic | n/a (no code shipped) | n/a | see Round 5 below — fresh magnitude-weighted bucketing (n=30,892) finds 74.9% of all pooled `scan_units` error sits in the `range_too_broad_to_narrow` broad-guard reset FIRING OUTSIDE `is_cross_index_range_and` (a population Round 3/4's own comment already flagged as unscaled on purpose); Rounds 1-4's target shape drops to 2.3% of pooled error, median ratio 1.00 — confirming the shipped fixes worked, just on a small slice of the cell | +| 6 | downward `scan_units` scale (`COMPOSE_BARE_RANGE_BROAD_SCALE`) for the `CardRangePopcount` arm's own `range_too_broad_to_narrow` reset (single bare range leaf, `unique=card`) | kept | 15-17% both builds, unchanged (noisy at this cell's grain, same as Rounds 3/4); the finer `GatheredScan/card_range_popcount` sub-row moved 47%→52% within [0.8,1.25], median 0.94→1.05 | none, within run-to-run noise; regret matrix unchanged (96% `printing_compose` share both builds) | held-out paired-diff (controlled): 1,704 impr / 31 regr, 93.3M → 16.0M abs `scan_units` error; new scale 0.43; `eval_domain` left untouched (96.6% of rows exactly 1.0, mean 0.975 — a real but small tail from price-field null-exclusion, not chased); flags the sibling `else` branch's `scan_units = card_est` as itself badly under-calibrated (median ratio ~0.25-0.37 by field) — not fixed this round, out of scope, noted for a future round | +| 7 | downward `scan_units` scale (`COMPOSE_SAME_RANGE_BROAD_SCAN_SCALE`) for `PrintingCompose`'s OWN `range_too_broad_to_narrow` reset, gated on a NEW `is_same_index_range_only` (bare single range leaf, or a fused same-field two-sided bound) — the rest of Round 5/6's "single:range" bucket that `CardRangePopcount` never reaches | kept | 17-18% both builds, unchanged (noisy at this cell's grain, same as every prior round); the pooled `GatheredScan/printing_compose` row (all `unique` modes) unchanged at 24% both builds — expected, small slice of a much larger diverse pool | none, within run-to-run noise; regret matrix unchanged (95% `printing_compose` share both builds) | held-out paired-diff (controlled): 6,422 impr / 33 regr, 304.8M → 57.6M abs `scan_units` error; new scale 0.52; `eval_domain` left untouched (measured exact, median/mean 1.000); population confirmed to be a SEPARATE, independently-broken slice of "single:range" from Round 6's, reached via a different acquire branch (`printing_compose`, not `card_range_popcount`) for two independent reasons — see Round 7 below | +| 8 | diagnostic: bucket candidates-acquire `GatheredScan`/`card` error by shape | diagnostic | 13% (n=22,190, median 0.60), unchanged from checkpoint — expected, no code shipped | n/a | see Round 8 below — pivots off the printing-range-index family entirely (Rounds 1-7's whole target) onto `Prep::Candidates`, the OTHER acquire branch feeding this same pooled cell. Finds `eval_domain` exact (median 1.00 against `cards_visited`) and `scan_units` also near-exact-to-UNDER-predicting (median 1.00, several high-magnitude buckets 1.2-1.8x, i.e. real work exceeds the estimate) — the OPPOSITE direction from the pooled ns-space over-cost (median 0.49-0.60), so neither size feature is the culprit; the bug is in how `GATHER_*` rate/fixed constants convert those (correct) features into ns for the `candidates` (and sibling `plane`) acquire branch specifically. Two concrete mechanisms found: (a) `GATHER_FIXED_COST_NS` (169.6ns) is ~4x too high for the 32% of the sample with zero matches (median measured 42ns); (b) card-mode's `feats.matches = count` (unconditional, `candidate_feats`, lib.rs~11776) ignores real residual selectivity — `is:vanilla`-shaped high-selectivity residuals push 2-3% of the predicted match count, and the whole per-candidate verify-tier charge (`GATHER_CARD_PASS_NS + max(tier_ns, GATHER_RESIDUAL_FLOOR_NS)` × `eval_domain`) doesn't discount for short-circuit-driven cheap-average-case cost the way real `card_pass` behaves at low match rates. A THIRD population invisible to `bench_cost_model_agreement.py`'s own flat-conjunction sampler — Or/negation/nested-paren structures via `structured_query()` — shows the opposite tail shape (median near 1.0, p90 1.25-3.48x UNDER-cost) and needs its own round. | +| 9 | lower fixed cost (`GATHER_FIXED_COST_ZERO_MATCH_NS`) for `PhysicalPlan::GatheredScan`'s zero-match rounds, gated on `matches == 0` the same way the arm's `tier_ns > 0.0` neighbor is gated — the first fix in this doc inside `cost.rs`'s cost FORMULA rather than `lib.rs` feature estimation | kept | 11% → 30% (n=38,435→38,889, median 0.57→0.77) — largest single-round movement since Round 0; by-unique `GatheredScan`/`card` cell flips FAIL (0.69) → PASS (0.80) | `GatheredScan/printing_compose` unchanged (median 1.15→1.14, 24%→24%); `GatheredScan/printing_range_scan` and `/card_range_popcount` unchanged; `bench_regret_matrix.py` total regret unchanged (27.6ms both builds); `bench_query_latency_ab.py` same-build canary swings by a comparable magnitude to the real A/B diff (-0.2µs vs -0.3µs) — no real latency effect claimed | held-out paired-diff (hash-of-query split, 9,890 zero-match rows): calibration half (n=4,944) median measured `plan_self_ns` sets constant to 42.0; held-out half (n=4,946) 4,577 impr / 369 regr / 0 tied, 530,256 → 103,110 abs ns error (5.1x), median ratio 0.248 → 1.000, within-25% 0.1% → 57.7%. Confirmed a real risk this round could not fully close within its `cost.rs`-only blast radius: `plan_cost` costs EVERY candidate plan from ONE shared `PlanFeatures` per acquire (`lib.rs:12917`), so `matches == 0` also fires for `GatheredScan` costed as a competitor/picked plan under `printing_compose`/`card_range_popcount`/`printing_range_scan` (RANGE_ACQUIRES) acquire, where `eval_domain == 0` is an unset accounting default rather than a real empty candidate list, and dispatch pays a real (sometimes large, e.g. 4,959ns median for one `printing_compose` slice) `prepare_candidates` rebuild this arm has no term for at all — pre-existing (already 29x under-predicted before this round) and NOT introduced by this fix, but made numerically worse in isolation (29x → 118x under on that slice). Checked for real routing impact directly (a same-build wheel diff on two flip cases, `date<1993-08-05`/`tix<0.01` under `printing_range_scan`) and via `bench_regret_matrix.py` (total regret 27.6ms unchanged) and `bench_cost_model_agreement.py` (no other cell moved) — no measurable regression found, but the gate is a correlated proxy, not the exact phenomenon, for this sliver of RANGE_ACQUIRES rows; flagged for a future round that can touch `lib.rs` to add an acquire-branch-aware feature | +| 28 | scope `COMPOSE_RANGE_AND_BROAD_SCAN_SCALE` (Round 4) and `COMPOSE_SAME_RANGE_BROAD_SCAN_SCALE` (Round 7) to `Mode::Card` only, leaving `Mode::Printing`/`Mode::Artwork` at the pre-existing unscaled `n_printings` ceiling | kept | not this doc's own metric (see below) | pooled `scan_units` feature accuracy (`bench_feature_accuracy.py`), the metric a fresh `main`-vs-`costcell/trunk` A/B (Round 27) found regressed: median 0.70 (UNDER-COUNTS) → 0.94 (clean), against `main`'s own 1.00 | see "Round 28" narrative below — both scales were fit exclusively on `unique=card` samples (each round's own doc says so) but applied unconditionally to all three modes; `Mode::Printing`/`Mode::Artwork`'s real `printings_examined / n_printings` reads EXACTLY 1.000 (zero spread) for this guard-fired population, so the card-only-derived scale was silently manufacturing an under-count for two modes it was never calibrated against | +| 30 | `STREAM_SMALL_TOTAL_REDO_BIAS`, a `stream_scan_units` correction for `printing_compose`'s bare `else` arm (`Mode::Card`, no legality partner) — Round 1's `scan_units` revision was inherited verbatim by `StreamedSelect`'s own feature, which structurally under-prices a SECOND, unmodeled `push_card_matches` pass `run_query_streamed`'s small-total branch pays and `GatheredScan` never does | kept, partial | n/a (this doc's own agreement-gate metric untouched; see the flip/regret numbers below instead) | `#852` ordering 88%→88% clean; Round 28's pooled `scan_units` median 1.00→1.00 clean | see "Round 30" narrative below — of 114 reproduced f3f4a017 flip queries, 50 (44%) now correctly re-route to `GatheredScan`; `StreamedSelect -> GatheredScan` regret matrix slice -7% share of traffic / -12% regret-ms; residual traced to the acquire-time `result_total` ESTIMATE itself being unreliable near `STREAM_MIN_MATCHES` for cross-index-range Ands (this doc's own Round 1 "separate, uninvestigated `domain_cards` bug" flag) — not a `cost.rs` rate problem, so chunk 2 (rate refit) is unlikely to close the rest on its own | +| 32 | new `PlanFeatures::perm_walk_span` feature (`cost.rs`/`lib.rs`) for `StreamedSelect`'s OTHER branch (`walks_permutation`, `total > STREAM_MIN_MATCHES` — different from Rounds 30/31's small-total gather): `perm_steps`'s estimate multiplied by `n_cards` unconditionally, when the real executor already bounds its walk to the filter's own interval on the sort column | kept | n/a (not this doc's metric; see below) | `#852` 88%→89% clean; Round 30/31 territory (`StreamedSelect -> GatheredScan` regret slice) flat; Round 28's `scan_units` unreachable by this change | see "Round 32" narrative below — held-out mean \|log ratio\| 1.033→1.001 pooled (both halves improve independently); `StreamedSelect/candidates` cost-model-agreement cell unchanged (median 0.59 both builds) because the targeted correlation (filter bounds the same field the query orders by) is rare under uniform traffic; shipped as a strict-generalization correctness fix (collapses to the old formula when unbounded), not for measured impact on this specific cell | +| 33 | `set_collector_ranges` (`lib.rs`, load-time precomputed per-set `collector_number_int` min/max/count), a new `compose_printing_estimate` `And`-arm tightening for the 2-source `set:X` + `cn`-range shape: `density = count / (max-min+1)` scaled by the query's own overlap, replacing the plain min-fold this shape had no other tightening for | kept | n/a (not this doc's own metric; see Round 33 narrative) | pooled cost-model-agreement cells move within noise (an untouched acquire branch, `PrintingCompose/plane`, shows the largest swing, 0.88→0.76, confirming it's sampling noise not this fix); `#852` 89%→89% clean; Round 28's `scan_units` 1.00→1.00 clean; Rounds 30/31/32's flip-query population 51/95 fixed on BOTH builds, 0 regressed | see "Round 33" narrative below — held-out validation across 550 real sets / 3,300 queries / both shapes: density estimator pooled median \|log ratio\| 0.000 (88.8% within 25%) against the fold's 0.788 (18.0% within 25%); regret matrix moved 37.4ms→33.4ms (-11%, improving); one honest documented exception -- a non-contiguous set (SLD) can now undershoot where the fold used to overshoot, still a net improvement (2.5x under vs 24x over) but a new failure direction | + +### Round 1 + +The self-check (constraint 3 in the parent doc) cleared cleanly: every new number +(`printing_matches` = `est.result.printing`, `domain_cards`, `printings_per_card`) was already +computed before `scan_all` runs, so the fix is a handful of extra float ops, not a new scan -- +confirmed by the correctness/latency gates below showing no execution-time effect. + +The surprising part showed up in the paired-diff, not the self-check: for **~37% of the and2/and3 +RANGE_FAMILIES sample**, `domain_cards` itself (the candidate-card estimate this fix multiplies by a +depth term) is already smaller than `domain_cards * printings_per_card` -- i.e. smaller than the +*maximum possible* span for that many candidate cards -- while the REAL measured +`printings_examined` exceeds even that ceiling. No depth formula operating on top of `domain_cards` +can fix those rows; the error is upstream, in `compose_printing_estimate`'s own candidate-card count +for an And of several different-index range leaves (price/cn/released_at each have their own index, +so an And across them doesn't hit the same-index fusion or the plane-based tightening this session's +prior work added -- it falls back to `calibrated_balls_into_bins` on the min-folded printing match +count). Restricted to the other ~63% (`domain_cards` not the bottleneck), the fix moves within-25% +from 7.5% to 36.7% and the median from 2.67 to 0.92 -- a real, large improvement, just capped by a +separate, uninvestigated `domain_cards` bug for multi-range-index Ands. That bug is the natural next +target, since it bounds how far ANY depth-side fix here can go. + +The primary gate (`bench_cost_model_agreement.py`, single uncontrolled 300s run each) moved the +target cell only 16% -> 17%, which looks like noise rather than signal on its own -- confirmed as +noise by checking UNRELATED cells this change cannot touch (`CardRangePopcount` 59% -> 66%, +`GatheredScan`/`candidates` 12% -> 15%, `StreamedSelect`/`printing_compose` p90 1.72 -> 2.98) moving +by comparable or larger amounts between the same two runs, purely from the two 300-second windows +sampling a different number and mix of queries. The paired, same-query-set diff (1,500 shared queries, +identical rng seed against both builds) is the only one of the two that actually isolates this +change's effect, and it shows the real number. + +### Round 2 + +Target: fix `domain_cards` itself for the ~37% subset Round 1 identified as ceiling-capped — an +And of 2+ DIFFERENT-INDEX printing-varying range leaves (`price_usd`/`price_eur`/`price_tix`/ +`collector_number_int`/`released_at`, each its own separate index), where `exact_result_total` +returns `None` (no pair-table/arith-tuple/plane-compile coverage exists for this combination) and +`est_cards` falls back to `calibrated_balls_into_bins(printing_matches, n_cards)` on the min-folded +(loosest) printing match count. + +**Self-check (constraint 3):** the only new per-query work in either variant tried is one extra +`PrintingValueIndex::range` partition-point probe per distinct range field the And names (`O(log +n)`, bounded by query length, never by match/printing/candidate count), reusing the exact same +lookup `bare_range_bounds`'s other callers already pay per leaf — paid only after `exact_cards` has +already declined. Every number traces to an existing precomputed structure (`PrintingValueIndex`, +`calibrated_balls_into_bins`/`COMPOSE_CARD_ESTIMATE_BIAS`), no new per-query scan. This cleared the +self-check; the idea failed on the empirical paired-diff instead, described below. + +**Attempt 1 — printing-space independence product.** Combined each leaf's own exact printing-match +selectivity (`k_i / n_printings`) via `Π(k_i / n_printings) * n_printings`, excluding any +combination touching 2+ of the price triple (checked directly: `price_usd<5 price_eur<4`-shaped +queries are a real correlation risk, confirmed on a 500-query price-only paired diff below, though +moot once the whole approach failed). Wired in as a `.min()` against the existing min-fold, feeding +both `est_cards`'s fallback and (to keep the pair internally consistent) the Round-1 depth term's +`density` numerator. + +Paired diff (1,500 shared and2/and3 RANGE_FAMILIES queries, unique=card, identical seed against +baseline `costcell/trunk`@`f3f4a017`): **38 improved / 496 regressed / 966 tied, total abs +`scan_units` error 17.3M → 18.1M (worse), within-25% 11.8% → 8.3% (worse)**. Price-triple-only +subset (500 queries, `usd`/`eur`/`tix` and2/and3): 0 improved / 0 regressed — the correlation guard +worked exactly as designed (no combination in that subset reached the independence branch), but +this is moot given the whole-population result. + +**Why it failed, and why no independence variant can work here:** a probability-product +combination is mathematically bounded above by its smallest single factor whenever every factor is +`<= 1` (`Π p_i <= min(p_i)`) — so ANY such combination can only ever SHRINK an estimate relative to +the tightest single-field bound, never raise it. Checking the baseline's own error DIRECTION on the +1,500-query sample confirms this is the wrong direction for the dominant failure mode here: 866/1500 +rows (58%) already UNDER-estimate `scan_units`, only 450/1500 (30%) over-estimate, and Round 1's own +investigation established that `domain_cards` for this population is a FLOOR that undershoots the +true candidate span, not a ceiling that overshoots it (`printings_examined` exceeding even `domain_cards +* printings_per_card`). A transform that can only shrink an already-too-small number cannot fix it; +it just pushes the under-estimating majority further from the truth while incidentally helping the +smaller over-estimating minority, netting negative overall — exactly the 38/496 split measured. + +**Attempt 2 — card-space independence product**, tried as "the closest variant" once attempt 1's +mechanism was understood to be structurally wrong-directioned: instead of combining printing-space +selectivities, combine each leaf's own CARD-space estimate (`calibrated_balls_into_bins(k_i, +n_cards)`), on the theory that "this card has SOME printing satisfying leaf i" is a weaker, +superset condition of the filter's real "one printing satisfies every leaf" semantics, so the +combination should be able to raise the estimate instead of shrinking it. Wired in as `.max()` +against today's fallback. Empirically: **0/1500 changed** — the `.max()` branch never won even +once. This confirms the same math from the opposite direction: each leaf's own card-space estimate +is itself `<= n_cards`, so the product-of-fractions form is bounded by the smallest such factor +regardless of which space it is computed in — moving to card space changes what each factor MEANS, +not the shape of the ceiling the combination is stuck under. + +**Conclusion:** rejected at self-check-plus-paired-diff. Both variants respected the +pre-computation constraint (no new per-query scan class) but neither can fix a floor-too-low bug by +construction — this rules out the whole "independence product" family for this specific target, not +just a tuning miss. No code committed; both attempts were reverted (`git checkout -- +card_engine/src/lib.rs` against `costcell/trunk`, confirmed clean via `git diff --stat`). + +**Next steps for a future round:** the diagnosed bug (domain_cards undershoots the true candidate +span for this shape) needs a mechanism that can RAISE the estimate, which independence-style +combination cannot do. Two candidates worth checking before another attempt: (a) a flat, +shape-specific multiplicative correction on top of `calibrated_balls_into_bins`'s output — same +precedent as `COMPOSE_CANDIDATE_SPAN_BIAS`/`COMPOSE_CARD_ESTIMATE_BIAS` — but fit with a proper +calibration/held-out split (not the same 1,500-query sample used to diagnose the bug, per this +repo's benchmark-methodology rule); (b) investigating whether the undercount's true source is +within-card printing clustering (reprints of the SAME card may jointly satisfy multiple range +conditions at a materially higher rate than corpus-wide field marginals imply, since a card's own +printings are not independent draws — the same clustering `COMPOSE_CARD_ESTIMATE_BIAS`'s 1.78 +divisor already corrects for at the SINGLE-field level), which would need a different precomputed +source than the ones checked here, not just a different combination formula. + +### Round 3 + +Target: Round 2's own "next steps" item (b) — a flat, shape-specific multiplicative correction on +top of `calibrated_balls_into_bins`'s output, fit with a genuine calibration/held-out split, for the +same And-of-cross-index-range-leaves population. A second constant, `COMPOSE_RANGE_AND_CLUSTER_BIAS`, +routes `est_cards`'s fallback (`acquire_plan_features`, the `PrintingCompose` arm) to +`calibrated_balls_into_bins_with_bias(printing_matches, n_cards, COMPOSE_RANGE_AND_CLUSTER_BIAS)` +instead of `COMPOSE_CARD_ESTIMATE_BIAS`'s 1.78, whenever the new `is_cross_index_range_and` detects +an `And` with 2+ children whose `bare_range_bounds` indexes are pairwise distinct (same-index +children, e.g. a two-sided `usd>=a usd<=b`, still fuse to one index and don't count — that population +already gets an exact `k` from `fuse_and_range_children` upstream and never reaches this fallback). + +**Self-check (constraint 3):** trivially clears. `is_cross_index_range_and` is O(children) — +`bare_range_bounds` per child is a pure match + float comparison, no index probe, bounded by query +length never match count — and only runs inside the `unwrap_or_else` closure, i.e. only after +`exact_cards` has already declined. No new per-query scan class; confirmed by `cargo test` and the +same-build latency canary below showing nothing distinguishable from noise. + +**A structural surprise mid-investigation, not in the fix itself:** the first Python re-derivation of +`scan_units` (needed to sweep the bias without a Rust rebuild, same trick Round 1 used for +`COMPOSE_CANDIDATE_SPAN_BIAS`) matched the live build's own `scan_units` on only 1,138/1,500 rows. +The other 362 are `range_too_broad_to_narrow` — a LATER, separate guard in `acquire_plan_features` +(after `domain_cards`/`scan_units` are computed) that resets both to the full corpus whenever the +And's min-folded `printing_matches` alone exceeds `MAX_NARROW_FRACTION` (0.25) of `n_printings`, +**independently of any bias**. This is not a bug this round touches — it is a separate, deliberate +"narrowing degrades to a full scan" model, verified elsewhere — but it means a clustering-bias +constant here can only ever move the NARROW subset (562/742 of the held-out half): the broad subset's +`scan_units` is bias-invariant by construction. Modeling the guard in the Python re-derivation brought +the self-check to 1,500/1,500 exact matches before any sweep was trusted, and the real Rust build's +paired diff (below) matches the Python-simulated numbers exactly, confirming the model was right. + +**Fit.** Captured, per sampled query, each leaf's own exact printing-match count (`k_i`, via an +isolated `unique=printing` sub-query per predicate — exact, no estimate) and the real +`printings_examined` GatheredScan counter, over 1,500 and2/and3 RANGE_FAMILIES queries (`unique=card`, +same population and precedent size as Rounds 1-2). Split by `hash(query) % 2` — 758 calibration / 742 +held-out. Swept the bias 0.20–1.78 in steps of 0.02 on the calibration half only; the error-vs-bias +curve is smooth and convex on BOTH halves with minima 0.04 apart (1.14 calibration, ~1.06 held-out), +which is what a genuine signal looks like rather than noise fit to one split. Picked `1.1`, inside +both minima, rather than either half's precise argmin. + +``` + calibration (n=758) held-out (n=742) +scan_units total abs 8.70M (1.78) -> 8.31M (1.1) 8.60M (1.78) -> 8.02M (1.1) +improved / regressed 417 / 148 433 / 117 +narrow-only subset (n) 576 562 +narrow-only total abs 3.15M (1.78) -> 2.76M (1.1) 3.19M (1.78) -> 2.61M (1.1) +price-triple subset (n) 382 363 +price-triple total abs 3.87M (1.78) -> 3.74M (1.1) 3.73M (1.78) -> 3.47M (1.1) +``` + +Direction matches the assignment's hypothesis: smaller than 1.78 (less division of `k`, so a HIGHER +effective ball count and a higher resulting estimate), correcting Round 2's diagnosed floor-undercount +rather than repeating `COMPOSE_CARD_ESTIMATE_BIAS`'s saturating-overcount correction. + +**Price-triple check.** The held-out price-triple subset (`usd`/`eur`/`tix`, 2+ of them) improves +proportionally in line with the whole population (213/68/82) — no sign of the correlation risk Round 2 +flagged, because this is a flat multiplicative rescaling of `calibrated_balls_into_bins`'s existing +math, not a combination formula across per-leaf estimates; there is no per-leaf independence +assumption for near-identical fields to violate. + +**Verified against the real build, not just simulation:** rebuilt the modified engine and re-ran the +same 1,500-query, same-seed sample through it directly (not the Python re-derivation) — the real +paired diff (baseline `costcell/trunk`@`ef78a984` vs modified) landed on the exact same numbers as the +simulation (758/742 split, 8.70M→8.31M and 8.60M→8.02M), confirming the Python model used to pick the +constant was not itself a source of error. + +**Why the single-run agreement gate doesn't move.** `bench_cost_model_agreement.py`'s `GatheredScan`/ +`card` cell stayed at 15% within [0.8, 1.25] on both builds (35,918 vs 35,946 rows) — expected, not a +sign the fix is inert: this cell pools every card-mode `PrintingCompose` acquire, and the affected +shape (And of 2+ different-index range leaves, narrow enough to escape `range_too_broad_to_narrow`) is +a small slice of it. The held-out paired-diff above is the controlled measurement; this cell is the +same noisy sanity check Round 1 already established is uninformative at this grain. + +### Round 3 confirmation runs + +- `bench_regret_matrix.py --seconds 120 --mode uniform`: same shape as Round 1's — regret still 96% + `printing_compose` share, `StreamedSelect -> GatheredScan` / `GatheredScan -> PrintingCompose` still + the largest picked/best mismatches, nothing resembling the 23.6x acquire-time precedent. +- `bench_query_latency_ab.py --mode realistic --sample 800 --seed 1`, baseline vs modified, interleaved + A1/B1/A2: real diff `B - A = -0.3µs, 95% CI [-0.6, -0.1]`, "B is FASTER". Same-build canary (A1 vs + A2, zero code difference): `-0.6µs, CI [-0.9, -0.4]`, also "B is FASTER" — a swing of comparable (here + larger) magnitude with nothing changed, matching Round 1's own non-interleaved-run drift finding. Read + as no detectable latency effect either way, not as a confirmed speedup. + +### Round 4 + +Target: the "broad" ~24-25% of the same `is_cross_index_range_and` population Round 3 flagged out of +scope -- the subset where `range_too_broad_to_narrow` (a LATER, independent guard, found mid- +investigation by Round 3) resets `eval_domain`/`scan_units` to the full corpus regardless of any +bias, because the And's min-folded `printing_matches` alone is too broad a fraction of `n_printings` +to trust `domain_cards`. Round 3's own report called the ~69% "real usage" figure for this subset a +mid-investigation finding, not validated -- this round re-derives it from scratch on a fresh sample +before building anything on it. + +**Re-derivation.** Sampled 1,500 and2/and3 RANGE_FAMILIES queries (`unique=card`, fresh seed) via +`query()` with `Shape(families=RANGE_FAMILIES, predicates=2 or 3)` -- family draws are distinct- +without-replacement and each of the 5 `RANGE_FAMILIES` maps 1:1 to its own printing-value index, so +every sampled query is `is_cross_index_range_and` by construction, same reasoning Round 3 used. +Detected the guard firing by its signature (`eval_domain == n_cards` and `scan_units == n_printings`, +which for this printing-varying population -- never card-invariant, never a bare collection leaf, +and Round 2 already proved `est.result.card` is never `Some` here -- means the guard fired with none +of its four exemptions applying): **372/1,500 (24.8%)**, matching the ~24-25% cited going in. + +Read the real GatheredScan counters via `explain_analyze` (`num_warmups=0, num_trials=1`; counters +are round-invariant -- checked directly by rerunning 20 queries at `(0, 1)` against `(2, 5)` with +identical `cards_visited`/`printings_examined` both times). Result, on the 372 guard-fired rows: + + real cards_visited / n_cards mean 1.000 median 1.000 (0 rows below 1.0) + real printings_examined / n_printings mean 0.697 median 0.713 + +`eval_domain` (`n_cards`) is EXACT for every one of the 372 rows, not just close -- real card-space +narrowing gives up at the same `range_too_broad_to_narrow` threshold this guard checks (the function +is shared with the real narrowing path, not just this pricing site), so a GatheredScan the router +actually runs after this fires really does visit every card. `scan_units` is the opposite: the guard's +`n_printings` ceiling is real (never measured over 1.0) but loose, at a stable ~0.70 of it -- this +re-derives the ~69% figure cleanly, and settles the "did the guard also give up on the printing side" +question the eval_domain number could not answer. + +**Fix.** Left `eval_domain` untouched (scaling an already-exact number down would reintroduce the +under-charge this guard exists to prevent -- the exact failure mode the four existing exemptions were +each added to fix, so this round does not risk it even via a downstream scale). Added +`COMPOSE_RANGE_AND_BROAD_SCAN_SCALE` (0.7), applied to `scan_units` alone, gated on +`is_cross_index_range_and(composed, indexes)` -- reused unchanged from Round 3, not reimplemented. +This is a scale on the ALREADY-DECIDED reset, not a 5th exemption: the guard's unconditional reset +still fires exactly as before; only what `scan_units` (never `eval_domain`) resets TO changes, and +only for this one shape. + +**Fit.** Split the 372 guard-fired rows by `hash(query) % 2`: 191 calibration / 181 held-out. Swept +0.60-0.84 in steps of 0.01 on the calibration half only; both halves' error-vs-scale curves are +smooth, convex, and minimize at the SAME 0.71 (closer agreement than Round 3's two minima 0.04 apart). +Picked 0.7, inside the flat bottom of both and matching the sample's own mean/median realized fraction +almost exactly. + +``` + calibration (n=191) held-out (n=181) +scan_units total abs 5.64M (1.0) -> 1.92M (0.7) 5.40M (1.0) -> 1.90M (0.7) +improved / regressed 172 / 19 166 / 15 +price-triple subset (n) 79 71 +price-triple total abs 1.91M (1.0) -> 0.79M (0.7) 1.82M (1.0) -> 0.75M (0.7) +``` + +**Price-triple check.** The held-out price-triple subset (`usd`/`eur`/`tix`, 2+ of them) improves +proportionally in line with the whole population (62 improved / 9 regressed) -- same reasoning as +Round 3's own price-triple check: a flat scale on an already-computed ceiling has no per-leaf +independence assumption for the near-identical price columns to violate. + +**Verified against the real build, not just simulation.** Rebuilt the modified engine and re-ran the +identical 1,500-query, same-seed sample through it directly -- `eval_domain` matched `n_cards` on all +372 guard-fired rows (0 mismatches) and `scan_units` matched `round(0.7 * n_printings)` exactly (0 +mismatches), and the real paired diff landed on the exact same numbers as the Python-side +re-derivation (5.64M/5.40M -> 1.92M/1.90M, 172/19 and 166/15). + +**Why the single-run agreement gate doesn't move.** `bench_cost_model_agreement.py`'s `GatheredScan`/ +`card` cell stayed at 16% within [0.8, 1.25] on both builds (33,966 vs 33,806 rows) -- expected: this +cell pools every card-mode `PrintingCompose` acquire, and the guard-fired subset of +`is_cross_index_range_and` is a small slice of it, same reasoning as Round 3. + +### Round 4 confirmation runs + +- `cargo test --manifest-path card_engine/Cargo.toml`: 167 passed, 0 failed, 56 ignored. +- `cargo clippy --manifest-path card_engine/Cargo.toml --all-targets -- -D warnings`: clean. +- `bench_regret_matrix.py --seconds 120 --mode uniform`: same shape as Rounds 1 and 3 -- regret still + 96% `printing_compose` share, `StreamedSelect -> GatheredScan` / `GatheredScan -> PrintingCompose` + still the largest picked/best mismatches, nothing resembling the 23.6x acquire-time precedent. +- `bench_query_latency_ab.py --mode realistic --sample 800 --seed 1`, baseline vs modified, interleaved + A1/B1/A2: real diff `B - A = +0.4µs, 95% CI [+0.3, +0.6]`, "B is SLOWER". Same-build canary (A1 vs + A2, zero code difference): `-0.4µs, CI [-0.5, -0.2]`, "B is FASTER" -- a swing of comparable + magnitude with nothing changed, matching Rounds 1 and 3's own non-interleaved-run drift finding. Read + as no detectable latency effect either way, not as a confirmed regression. + +### Round 5 + +Diagnostic only, no code changes -- re-run the magnitude-weighted AST-shape breakdown from scratch +against the current `costcell/trunk` tip (`8ab0b4cc`), since a full-corpus checkpoint +(`bench_cost_model_agreement.py`) still shows `GatheredScan`/`card` at 15% within 25%, essentially +unchanged from Round 0's 16%, despite three landed, held-out-validated fixes (Rounds 1, 3, 4). + +**Method.** Isolated release wheel (`maturin build --release`, extracted, `PYTHONPATH`-pinned). +Sampled with `QuerySampler(corpus, "uniform")`, reimplementing `query()`'s body inline (predicate +count → `_draw_families` → `predicate` per family) so each row keeps which FAMILIES were drawn -- +`query()` itself doesn't return them, and every other part of the sampling loop (limits, offsets, +warmups/trials, `unique`/`orderby`/`direction` drawn independently) matches +`bench_cost_model_agreement.py` exactly. Every family maps to one of six categories: `range` +(`usd`/`eur`/`tix`/`cn`/`released` -- the printing-varying, range-indexed fields this whole doc is +about), `numeric_other` (`pow`/`tou`/`cmc`/`loyalty`), `rarity`, `text` (`name`/`oracle`/`flavor`/ +`artist`), `arith` (the extended syntax), and `collection` (everything else -- type/legality/ +identity/color/set/keyword/produces/tag/border/frame/watermark/devotion). `sampler.query()` only +ever emits a flat conjunction (no Or/Not/regex), so the whole GatheredScan/card population this cell +measures is single leaves and `and2`/`and3` -- there is no Or-composed or Not-wrapped subpopulation +to bucket here; that's a property of what `bench_cost_model_agreement.py` samples, not something +this round chose. + +Per row: `predicted = acquire["scan_units"]`, `measured = plan["printings_examined"]` (`GatheredScan` +only, non-declined) -- the same pairing Rounds 3/4 used, per `scan_units`'s own doc comment at +`lib.rs:11213` ("the real `printings_examined` GatheredScan counter"). Bucket key = `structure` +(`single`/`and2`/`and3`) + sorted category tuple. Ranked by total absolute `scan_units` error per +bucket (magnitude-weighted), with each bucket's row count and median ratio reported alongside so a +high-count-but-tied bucket and a rare-but-catastrophic one are both visible. + +300s budget (same protocol/seconds as Round 0's baseline run) → **30,892 GatheredScan/card rows**, +same order of magnitude as Round 0's 35,074 and Rounds 1-4's 1,500-query calibration samples for +their narrower held-out slices. Total pooled absolute `scan_units` error: 175,122,864. (Note: this +is a `scan_units`-space ratio, same quantity Rounds 1-4 worked in, not the ns-space +measured/predicted ratio `bench_cost_model_agreement.py`'s headline 15%/16% number reports -- +the two measure different things and are not expected to match numerically.) + +**Ranked bucket table** (all buckets with n ≥ 1; buckets below 0.1% share are real but tiny): + +``` +bucket n sum |err| share median ratio within25% +single:range 3041 93,991,483 53.7% 0.64 1% +and2:collection+range 2301 17,699,351 10.1% 1.21 15% +and2:numeric_other+range 801 14,662,649 8.4% 0.74 25% +single:collection 6993 11,248,207 6.4% 1.00 68% +single:rarity 614 8,168,926 4.7% 1.08 41% +and2:range+rarity 199 5,631,260 3.2% 1.50 5% +and2:numeric_other+rarity 169 3,714,009 2.1% 0.41 21% +and2:collection+numeric_other 1628 2,553,659 1.5% 1.00 42% +and2:range+range 398 2,442,018 1.4% 1.15 35% +single:numeric_other 2355 2,405,375 1.4% 1.00 95% +and2:collection+rarity 466 2,234,802 1.3% 0.54 14% +and2:arith+range 215 1,708,742 1.0% 0.58 15% +and2:collection+collection 2442 1,497,390 0.9% 0.20 20% +and3:collection+numeric_other+range 355 1,303,555 0.7% 0.44 12% +and3:collection+range+range 232 855,513 0.5% 0.80 21% +and3:numeric_other+range+rarity 30 725,010 0.4% 0.31 17% +and3:collection+collection+range 522 526,766 0.3% 0.18 10% +and2:range+text 814 449,305 0.3% 0.86 20% +and3:numeric_other+range+range 62 434,311 0.2% 0.44 15% +and3:numeric_other+numeric_other+range 39 398,600 0.2% 0.27 10% +and3:collection+range+rarity 99 261,534 0.1% 0.77 12% +and2:arith+rarity 35 247,425 0.1% 0.43 29% +single:text 2454 239,195 0.1% 1.00 46% +and2:numeric_other+numeric_other 144 168,614 0.1% 1.00 82% +and3:range+range+range 24 154,252 0.1% 1.02 17% +[remaining 38 buckets each < 0.1% share, ~0.4% combined, mostly text/arith-involving rows with n<100] +``` + +**Confirmation: Rounds 1-4's target shape did drop, as expected.** The `is_cross_index_range_and`- +equivalent population (an `and2`/`and3` with 2+ `range`-category families -- exactly what +`is_cross_index_range_and` requires) is **807 rows (2.6% of the sample), 4,069,972 abs error (2.3% +of the pooled total), median ratio 1.00, 28% within 25%**. Before Rounds 3/4 this shape was +"tens of millions of units each" and the single dominant contributor by every account in this doc; +now it sits at a median ratio of exactly 1.00 (as good as any bucket in the table) and would not +make a top-10 list by magnitude. The three shipped fixes worked exactly as designed on their target +population -- they just never had a chance to move the pooled cell, because that population turns +out to be a small slice of it (2.6% by row count, 2.3% by error), not the ~37%+ this doc's earlier +rounds estimated from the narrower and2/and3-RANGE_FAMILIES-only calibration sample. That estimate +was never wrong on its own terms (it was scoped to the RANGE_FAMILIES-only shape from the start); +it just wasn't representative of the whole `GatheredScan`/card population once measured against it +directly. + +**What actually dominates: the SAME broad-guard reset, everywhere Round 4 didn't scale it.** Flagging +every row where `predicted == n_printings` exactly (the `range_too_broad_to_narrow` guard's +telltale signature -- both `PrintingCompose`'s and the sibling `CardRangePopcount`/ +`PrintingRangeScan` arms' resets set `scan_units` to the literal, unscaled `n_printings` when they +fire) finds **2,412 rows (7.8% of the sample) carrying 131,170,530 abs error -- 74.9% of the ENTIRE +pooled total -- at median ratio 0.45** (predicted ~2.2x too high). Zero of these 2,412 rows are +`is_cross_index_range_and` -- Round 4's scale never had a chance to touch any of them, by +construction. Split by structure: `single` (n=1,851, 91.6M), `and2` (n=534, 37.3M), `and3` (n=27, +2.2M). + +Reading `lib.rs` confirms this is not a new bug -- it is Round 3/4's own noted, deliberate scope +limit, finally showing up as the dominant term now that the target shape it excluded is fixed. Two +separate sites: + +1. **`PrintingCompose`'s own broad-guard reset** (`lib.rs:12239-12259`) scales `scan_units` by + `COMPOSE_RANGE_AND_BROAD_SCAN_SCALE` *only* `if is_cross_index_range_and(composed, indexes)`; the + inline comment at `:12248-12250` says outright: "every other query reaching this branch (a single + broad range, a broadcast legality, ...) never had this scale's calibration sample in it, so it + keeps today's unscaled `n_printings` ceiling." That "everything else" population is exactly what + this round measured. Per-bucket broad/narrow split confirms the broad slice is a small-count, + huge-magnitude minority within each mixed bucket: + ``` + and2:collection+range broad n=210 (9% of bucket rows, 78% of bucket's error) median ratio 0.30 + narrow n=2091 (91% of rows, 22% of error) median ratio 1.39 + and2:numeric_other+range broad n=188 (23% of rows, 94% of error) median ratio 0.21 + narrow n=613 (77% of rows, 6% of error) median ratio 0.93 + and2:range+rarity broad n=81 (41% of rows, 91% of error) median ratio 0.36 + narrow n=118 (59% of rows, 9% of error) median ratio 3.37 + ``` + Example rows (all `predicted == n_printings == 97,812`, the full corpus): + `tou<=5 tix>=0.02 tix<=0.04` → measured 26,834 (ratio 0.27); `tou>=2 tou<=4 year<2025` → measured + 12,491 (ratio 0.13); `r>=uncommon tix>0.02` → measured 46,256 (ratio 0.47); `r>=uncommon eur<0.49` + → measured 49,726 (ratio 0.51). + +2. **The sibling `CardRangePopcount` (`lib.rs:11801-11845`) and `PrintingRangeScan` + (`lib.rs:11846-11872`) acquire arms** -- which serve a single BARE range leaf under `unique=card` + (e.g. `usd>=0.24` alone, no `And` at all) -- have their own structurally identical + `range_too_broad_to_narrow`-gated reset to `(n_cards, n_printings)`. This is a completely separate + code path from `PrintingCompose` (confirmed live: `usd>=0.24` alone acquires via + `count_source: card_range_popcount`, not `printing_compose`), never in scope for any of Rounds + 1-4 (whose investigation was explicitly `compose_printing_estimate`/`PrintingCompose`). This is + the `single:range` bucket -- the single largest bucket in the whole table, 53.7% of pooled error + on its own, median ratio 0.64. Example rows (all `predicted == 97,812`): `usd>=0.24` → measured + 40,782 (ratio 0.42); `cn>=127` → measured 45,904 (ratio 0.47); `tix<0.12` → measured 53,260 + (ratio 0.54); `year>=2023` → measured 61,411 (ratio 0.63). + +**Secondary, smaller finding, opposite direction.** `and2:range+rarity`'s NARROW (non-broad) subset +reads median ratio 3.37 -- badly UNDER-costed, the opposite direction from everything else in this +round. Small in absolute terms (118 rows, ~9% of that bucket's 5.6M error, so well under 1M total) +-- not worth its own round yet, but worth a one-line flag for whoever next touches range+rarity +combinations, since it's a direction-flip rather than more of the same over-cost pattern. + +**What Round 6 should target.** The `range_too_broad_to_narrow` broad-guard reset, generalized +beyond `is_cross_index_range_and`, at two sites: + +- Extend (or add a sibling to) `COMPOSE_RANGE_AND_BROAD_SCAN_SCALE` inside `PrintingCompose`'s own + reset so it also scales `scan_units` when the guard fires but `is_cross_index_range_and` is false + (a lone broad range leaf mixed with a collection/numeric_other/rarity leaf, or a bare broadcast + legality/range predicate). This is exactly the pre-computation-safe pattern Round 4 already used -- + a flat multiplicative scale on an already-computed ceiling, no new per-query scan -- just widened + in scope. Needs its OWN calibration/held-out split before trusting a number: this round's median + ratio here (0.45) reads meaningfully lower than Round 4's fitted realized fraction (~0.70-0.71) for + the `is_cross_index_range_and` population, so reusing 0.7 unchanged is not obviously right -- + and the two mixed-leaf buckets above disagree with each other too (0.21-0.36 median), so a single + universal constant may not fit either; check whether the guard's realized fraction varies + systematically with the NON-range leaf's own selectivity before picking one or several constants. +- Add the analogous scale to the `CardRangePopcount`/`PrintingRangeScan` arms' own broad-guard reset + (`lib.rs:11831`, `:11862`) for a single bare range leaf -- a different acquire branch than + `PrintingCompose`, so it needs its own gate check and likely its own constant (median ratio here, + 0.64, differs again from both of the above), even though the underlying guard function + (`range_too_broad_to_narrow`) is the same shared code. This is `single:range`, the single largest + bucket by magnitude in the whole table -- the highest-leverage place to start. +- Population parity note for whoever fits this: unlike Rounds 3/4's RANGE_FAMILIES-only calibration + sample, this population spans every family category (collection/numeric_other/rarity/text mixed + with a range leaf, plus bare single range leaves with no other predicate at all) -- a proper + calibration/held-out split here should draw from the SAME uniform-mode, all-category sampling this + round used, not a re-use of the narrower RANGE_FAMILIES-only sample Rounds 1-4 built their splits + from, since that sample structurally cannot contain the `single:range` or mixed-category rows that + now turn out to matter most. + +### Round 6 + +Target: Round 5's own top recommendation -- the `CardRangePopcount` arm's own +`range_too_broad_to_narrow` broad-guard reset (`lib.rs:11831` as of Round 5's tip), the single +largest bucket in the whole pooled `GatheredScan`/`card` error table (`single:range`, 53.7% of pooled +error, n=3,041 in Round 5's sample, median ratio 0.64). A bare single range leaf under `unique=card` +(e.g. `usd>=0.24` alone) -- confirmed live via `count_source: card_range_popcount`, a completely +separate acquire branch from `PrintingCompose`'s `is_cross_index_range_and` guard Rounds 3/4 fixed. + +**Population re-derivation.** Sampled with `Shape(families=RANGE_FAMILIES, predicates=1, +unique={"card"})` (same shape `bench_card_range_estimate.py` already uses for this exact acquire +branch), filtered to `count_source == "card_range_popcount"`, varying `limit`/`offset`/`orderby`/ +`direction` per query (matching `bench_cost_model_agreement.py`'s own protocol rather than pinning +them, so the population is not an artifact of one page shape). Of all `card_range_popcount` rows, +**52-54% have the guard fire** (two independent 3,500-row samples: 54.1% and 52.1%) -- the "broad" +population this round targets. Real `GatheredScan` counters (GatheredScan is always tried as a +forced trial in `explain_analyze` regardless of which plan the router actually picks, same trick +Rounds 3/4 used) over 3,500 guard-fired rows: + +``` +eval_domain realized fraction (cards_visited / n_cards): mean 0.975 median 1.000 min 0.233 +scan_units realized fraction (printings_examined / n_printings): mean 0.447 median 0.434 min 0.159 max 0.798 +``` + +Per-field `scan_units` median: `cn` 0.41, `usd` 0.43, `eur` 0.43, `released` 0.42, `tix` 0.48 -- +stable within a ~20% relative band across all five `RANGE_FAMILIES`, not one field dominating or +diverging. + +**Self-check (pre-computation constraint).** The only change is a multiply-and-round on two numbers +already computed before this branch's `unwrap_or_else`-equivalent `if`/`else` runs (`n_printings` is +a corpus-wide constant read from `ctx`, `k`/`idx.len()` already drive the `range_too_broad_to_narrow` +call the branch makes regardless). No new per-query scan, no new index probe -- confirmed by +`cargo test` and the latency A/B below showing nothing distinguishable from noise once run-order +confounds are controlled for (see below). + +**A structural surprise, not in the target arm itself: the sibling `else` branch is also +miscalibrated, for a different reason.** The arm's own comment claims "the sibling `PrintingRangeScan` +branch below assumes the opposite (always unnarrowed) and its cells agree to within 1% -- this makes +both exact," which reads as a claim that the NARROW-subset `(card_est, card_est)` branch is exact. +Measured directly, on the guard-NOT-fired rows from the same sample: `card_est / cards_visited` +(eval_domain check) is indeed exact at the median (1.00), but `card_est / printings_examined` (scan_units +check) reads median 0.25-0.37 depending on field -- `card_est` (a DISTINCT-CARD estimate) badly +undershoots `printings_examined` (a printing count) whenever a card has multiple reprints inside the +narrowed range, which is common for `cn`/`released`/`usd`. This is the assignment's own "verify what +that comment refers to" check: it refers to the two branches' feature vectors being internally +CONSISTENT with each other (not to either being numerically accurate), and the `else` branch's +`scan_units` side is a real, separate miscalibration -- **not fixed this round** (out of the assigned +blast radius; scoped as a follow-up in the ledger table above, not silently folded into this +constant). + +**Why `eval_domain` is left untouched despite not being perfectly exact here (unlike Round 4's +population).** 96.6% of guard-fired rows read exactly 1.0; the remaining 3.4% are concentrated +entirely in `usd`/`eur`/`tix` queries at extreme thresholds (`eur>=1.05`, `tix>0.04`, ...), where most +cards have no printing with that currency at all, so the real materializing scan still narrows out +the null-complement even though the guard correctly judged the VALUE range too broad to narrow on. +Scaling the dominant 96.6%-exact regime down to chase a rare, structurally different tail would +reintroduce the under-charge the guard exists to prevent -- same call Round 4 made, but this time +independently verified rather than assumed to transfer, per the assignment's instruction. + +**Fit.** Split 3,500 guard-fired rows by `hash(query) % 2`: 1,765 calibration / 1,735 held-out. Swept +0.30-0.60 in steps of 0.01 on the calibration half only; both halves' error-vs-scale curves are +smooth and convex, minimizing one step apart (0.43 calibration, 0.44 held-out). + +``` + calibration (n=1,765) held-out (n=1,735) +scan_units total abs 96.0M (1.0) -> 15.4M (0.43) 93.3M (1.0) -> 16.0M (0.43) +improved / regressed 1,742 / 23 1,704 / 31 +``` + +Picked 0.43, inside the flat bottom of both curves. + +**Per-field constant considered and rejected as not worth it.** A per-field scale (each field's own +median as an oracle upper bound) reaches 30.3M total abs error against the flat scale's 31.4M -- only +~3.5% further reduction, except for `tix` (275 rows, smallest subgroup) where the per-field oracle +does meaningfully better (0.80M vs 1.71M). Given the modest aggregate gain and this round's mandate to +prefer a flat constant unless the fit clearly does not hold, one flat `COMPOSE_BARE_RANGE_BROAD_SCALE` +was kept; a future round revisiting `tix` specifically could reconsider. + +**Price-triple sanity (per-field, not cross-field correlation -- that check does not apply to a bare +single leaf).** `usd` (0.43), `eur` (0.43), `tix` (0.48) all sit close to the chosen 0.43; no +individual price field is a pricing outlier. + +**Verified against the real build, not just the Python-side sweep.** Rebuilt with the constant and +re-ran the identical 3,500-query sample directly against it: `scan_units` matched +`round(0.43 * n_printings)` exactly on all 3,500 rows (0 mismatches), and the real paired diff landed +on the exact same total (31,400,955 combined) as the simulation. + +**Routing-decision check (why this round is different from Rounds 3/4's structural risk).** Lowering +a feature this branch's shared `PlanFeatures` also prices COMPETING plans (`GatheredScan`/ +`StreamedSelect`) against could in principle flip the router away from `CardRangePopcount` toward a +now-artificially-cheap competitor. Checked directly: the router picked `CardRangePopcount` on +500/500 sampled bare-range queries under BOTH the baseline and modified build (same kw), and +`bench_regret_matrix.py`'s `acquire` table shows `card_range_popcount` at 0.00 mean / 0% miss in both +builds (n=659 baseline, n=658 modified) -- no misrouting introduced. + +**Why the single-run agreement gate barely moves, and why that's not evidence against the fix.** +`bench_cost_model_agreement.py`'s `GatheredScan`/`card` cell read 17% within [0.8, 1.25] on both +builds (n=33,019 baseline, n=33,251 modified) -- same story as Rounds 3/4: `card_range_popcount` is +only ~1,563-1,572 of that ~33,000-row pooled cell (~4.7%), so even fixing its single largest error +bucket cannot move a pooled median by much. The finer `GatheredScan`/`card_range_popcount` sub-row +(grouped by acquire branch, not pooled across all of `unique=card`) DID move: median 0.94 -> 1.05, +within-25% 47% -> 52% (n=1,563 / 1,572, single uncontrolled runs -- read as corroborating, not proof, +same noise caveat as every other single-run number in this doc). + +### Round 6 confirmation runs + +- `cargo test --manifest-path card_engine/Cargo.toml`: 167 passed, 0 failed, 56 ignored. +- `cargo clippy --manifest-path card_engine/Cargo.toml --all-targets -- -D warnings`: clean. +- `bench_regret_matrix.py --seconds 120 --mode uniform`: same shape as every prior round -- regret + still 96% `printing_compose` share, `StreamedSelect -> GatheredScan` / `GatheredScan -> + PrintingCompose` still the largest picked/best mismatches, `card_range_popcount`'s own regret + unchanged (0.00 mean / 0% miss, both builds) -- nothing resembling the 23.6x acquire-time precedent. +- `bench_query_latency_ab.py --mode realistic --sample 800 --seed 1`: the FIRST paired run (baseline + measured first, modified second) read `+4.1µs, 95% CI [+3.6, +4.5]`, "B is SLOWER" -- a magnitude + that, unlike every prior round's canary-comparable noise, looked like a real signal at first glance. + Investigated directly rather than accepted: (1) the specific queries showing the largest slowdowns + were `t:legendary`, `c:g`, `set:usg` and similar -- filters that never reach `CardRangePopcount` at + all, ruling out a routing-side effect from this change; (2) re-running with the build ORDER swapped + (modified first, baseline second) produced `-0.1µs`, "NO DETECTABLE DIFFERENCE"; (3) two further + same-build canaries (baseline-vs-baseline, modified-vs-modified, each a fresh pair) read `+0.8µs` + ("B is SLOWER") and `-0.4µs` ("B is FASTER") respectively -- swings of comparable or larger magnitude + than two of the three real A-vs-B diffs measured, with nothing changed. Read as run-order-dependent + machine drift (exactly the failure mode the harness's own module docstring warns about), not a + real latency effect in either direction -- consistent with the routing-decision check above finding + zero picked-plan changes on the target population. +- `cargo build`/wheel blast radius: `git diff --stat costcell/trunk` shows only `card_engine/src/lib.rs` + touched (58 lines: one new constant + its doc, five lines in the `CardRangePopcount` arm). + +### Round 7 + +Target: resolve the population-size discrepancy the assignment opened with -- Round 5's "single:range" +bucket (a single family/predicate drawn from `RANGE_FAMILIES`, `unique=card`) was 3,041 rows, 53.7% of +pooled error, but Round 6's fix only touches the `CardRangePopcount` arm, which Round 6 itself measured +at ~1,660 rows (~4.7% of the pooled cell) -- smaller than the bucket. Where does the rest go? + +**The `Prep::Candidates` hypothesis was checked first and refuted.** Reading `card_range_popcount_ +applicable` (lib.rs:9485) confirms it requires `plane.is_none()`, a bare range (`bare_range_bounds`), +AND `indexes.sort_perms.order(sort_col, descending, cards.len()).is_some()` -- both sort-permutation +directions for the query's exact orderby/direction/card-count combination. But the NEXT branch acquire +tries when that fails is not `Prep::Candidates` -- it is `PrintingCompose`, which is mode-agnostic and +requires no sort permutation at all (`printing_compose_applicable`, lib.rs:9437, and `is_printing_ +composable`'s range arm, lib.rs:6866, both gate only on `bare_range_bounds(...).is_some()`). A direct +sample confirms this empirically: of 1,184,753 bare single-range `unique=card` queries generated +(varying orderby/direction/limit/offset the way `bench_cost_model_agreement.py` does), 56.3% acquired +via `card_range_popcount` and the remaining 43.7% via `printing_compose` -- **zero** via `candidates`. + +**A second mechanism, found while building that sample, matters just as much: two-sided bounds.** +`Shape(families=RANGE_FAMILIES, predicates=1, unique={"card"})` -- Round 6's own generator for this +population -- can render its one drawn predicate as a fused two-sided bound (e.g. `eur>=0.23 +eur<=0.45`), because `QuerySampler`'s `bounded` parameter defaults to `None` (either shape, drawn at +random) rather than `False` (one-sided only). `bare_range_bounds`, `CardRangePopcount`'s own gate, +matches a single comparison and never an `And` (confirmed directly in `fuse_and_range_children`'s own +doc: "a FUSED two-sided range never arrives here at all"), so a two-sided bound reaches +`PrintingCompose` regardless of sort permutation. Round 5's AST-shape bucketer keyed "single" on the +SAMPLER's predicate count (one family drawn), not on `FilterExpr` structure -- so "single:range" +always included these two-sided `And`-shaped rows, they were just never told apart from true bare +leaves until this round asked. + +**Conclusion: the missing population is real, independently broken, and reaches `PrintingCompose`'s +OWN broad-guard reset -- exactly Round 5's "what Round 6 should target" recommendation item 1, which +Round 6 explicitly deferred** ("Extend ... `PrintingCompose`'s own reset so it also scales `scan_units` +when the guard fires but `is_cross_index_range_and` is false ... a bare broadcast legality/range +predicate"). Sampled 23,039 `printing_compose`-acquired rows from the same shape (240s budget, fresh +seed): 13,053 (56.6%) have the guard fire (`scan_units == n_printings`), split 5,300 true bare-single / +7,753 fused two-sided. Measured against the real `printings_examined` GatheredScan counter: + +``` + eval_domain/cards_visited scan_units/printings_examined printings_examined/n_printings +broad (guard fired, n=13,053) mean 1.000 mean 2.023 (median 1.917) mean 0.518 (median 0.522) +narrow (guard not fired, n=9,986) mean 0.905 (median 0.963) mean 0.381 (median 0.382) mean 0.122 (median 0.131) +``` + +`eval_domain` is exact on the broad subset (matches every prior broad-guard round). `scan_units` is +badly over-costed there (predicted ~1.9-2.0x too high), confirming Round 5's bucket-level median ratio +of 0.64 for "single:range" was a blend of this over-costed `printing_compose` slice and Round 6's +now-fixed `card_range_popcount` slice, not evidence Round 6 left its own target undone. The narrow +subset is badly UNDER-costed (median 0.38) -- a second, separate bug in `PrintingCompose`'s non-broad +branch for this same shape, structurally the same phenomenon Round 6 flagged in `CardRangePopcount`'s +sibling `else` branch (a card-count-shaped estimate undershooting a printing count) -- **not fixed this +round**, out of the assigned scope, noted below for a future round. + +**Self-check (pre-computation constraint).** The new gate, `is_same_index_range_only` (lib.rs, next to +`is_cross_index_range_and`), is O(children): it calls `bare_range_bounds` per child (a pure match plus +float comparison, no index probe) and compares index pointers, the identical technique and complexity +class `is_cross_index_range_and` already uses, only run inside the same `unwrap_or_else`-adjacent +branch after `exact_cards` has already declined. No new per-query scan, no new index probe -- confirmed +by `cargo test`/`cargo clippy` and the latency A/B below. + +**Fit.** Split the 13,053 broad rows by `hash(query|orderby|direction) % 2`: 6,598 calibration / 6,455 +held-out. Swept 0.20-0.80 in steps of 0.02 on the calibration half only; both halves' error-vs-scale +curves are smooth and convex, minimizing at the SAME 0.52 (each sub-shape's own argmin -- 0.48 bare- +single, 0.55 fused two-sided -- brackets it tightly, so one flat constant was kept rather than two). + +``` + calibration (n=6,598) held-out (n=6,455) +scan_units total abs 310.8M (1.0) -> 60.2M (0.52) 304.8M (1.0) -> 57.6M (0.52) +improved / regressed 6,560 / 38 6,422 / 33 +``` + +**Price-triple sanity (per-field, not cross-field correlation -- a flat scale on an already-computed +ceiling has no per-leaf independence assumption to violate, same reasoning as every prior broad-guard +constant).** `usd` (0.534), `eur` (0.546), `tix` (0.574) all sit in the same band as `cn`/`date`/`year` +(0.46-0.49); `tix` reads highest but not an outlier. + +**Verified against the real build, not just the Python-side sweep.** Rebuilt with the constant and +replayed the identical 13,053 rows directly against it: `scan_units` matched `round(0.52 * +n_printings)` exactly on all 13,053 rows (0 mismatches, i.e. `is_same_index_range_only` correctly +recognized every one of them), and the real paired total (117,812,900) landed on the exact same number +as the calibration+held-out simulation combined (60,175,506 + 57,637,394). + +### Round 7 confirmation runs + +- `cargo test --manifest-path card_engine/Cargo.toml`: 167 passed, 0 failed, 56 ignored. +- `cargo clippy --manifest-path card_engine/Cargo.toml --all-targets -- -D warnings`: clean. +- `bench_regret_matrix.py --seconds 120 --mode uniform`: same shape as every prior round -- `printing_ + compose` still 95% share both builds, `StreamedSelect -> GatheredScan` / `GatheredScan -> + PrintingCompose` still the largest picked/best mismatches, total regret comparable (49.9ms baseline + vs 50.7ms modified) -- nothing resembling the 23.6x acquire-time precedent. +- `bench_query_latency_ab.py --mode realistic --sample 800 --seed 1`: real diff (baseline vs modified) + `+0.7µs, 95% CI [+0.5, +0.9]`, "B is SLOWER". A same-build canary (baseline vs a second baseline run, + identical protocol, nothing changed) read `+0.5µs, CI [+0.3, +0.7]`, also "B is SLOWER" -- same sign + and comparable magnitude with zero code difference, matching every prior round's non-interleaved-run + drift finding. Read as no detectable latency effect, not a confirmed regression. +- `git diff --stat costcell/trunk` shows only `card_engine/src/lib.rs` touched (101 lines: two new + constants + their docs, one new helper function, four lines wiring it into the broad-guard branch). +- Full-table checkpoint (`bench_cost_model_agreement.py --seconds 300 --seed 0`): `GatheredScan`/`card` + 17% -> 18% within [0.8, 1.25], both within noise of each other (n=33,218 baseline, n=33,121 + modified) -- expected, same reasoning as every prior round: this cell pools every card-mode + `PrintingCompose`/`CardRangePopcount`/`candidates`/`plane` acquire, and this round's target (a single + range family reaching `printing_compose`'s broad guard) is a small slice of it. The pooled + `GatheredScan`/`printing_compose` row (every `unique` mode, n=54,827/54,658) also held steady at 24% + both builds -- same story, an even larger and more diverse pool this fix touches only a slice of. + +**Next steps for a future round.** The narrow-subset (`range_too_broad_to_narrow` NOT fired) +`printing_compose` bare-range population found mid-investigation above (median `scan_units` ratio 0.38, +n=9,986 in this round's sample) is real, separately broken, and out of this round's assigned scope -- +structurally the same "card-count-shaped estimate undershooting a printing count" bug Round 6 flagged +in `CardRangePopcount`'s sibling `else` branch, now confirmed to have a `PrintingCompose`-side +counterpart too. + +### Round 8 + +Diagnostic only, no code changes. Rounds 1-7 exhausted the printing-range-index family +(`compose_printing_estimate`/`CardRangePopcount`/`PrintingCompose`, all reached via `Prep::Range`) and +the pooled `GatheredScan`/`card` cell still reads 13-16% within [0.8, 1.25], essentially unchanged from +Round 0's baseline. This round asks where the rest of the error lives, and finds it in a completely +different acquire branch: `Prep::Candidates` (`count_source == "candidates"`), reached whenever +`prepare_candidates`/`narrow_rec` cannot resolve the query to a bare range or a fully plane-compilable +expression -- text search (`name`/`o`/`ft`/`a`), the extended arithmetic syntax (`power+toughness<6`, +`cmc>=power`), `is:`-rewrite predicates, `loyalty`, and any `Or`/negated/nested-paren structure, none of +which `is_printing_composable`/`is_broadcast_leaf_shape` accept. + +**Checkpoint** (`bench_cost_model_agreement.py --seconds 180 --seed 0`, isolated release wheel, same +protocol as every prior round): `GatheredScan`/`candidates` reads `n=22,190 median 0.60 p10 0.25 p90 +0.92 13% within 25% FAIL` -- the single largest acquire-branch row in the whole per-plan table by row +count, well below the `[0.8, 1.25]` bar, and **over-costed** (`median < 1`), the opposite direction +from every range-leaf fix Rounds 1-7 shipped. + +**Method.** Two throwaway samplers (not checked in), both pinning `unique=card` and varying +`orderby`/`direction`/`limit`/`offset` the way `bench_cost_model_agreement.py` does, against an +isolated release wheel: + +- **Flat-conjunction sample** (`QuerySampler.query()`'s own body, reimplemented inline per Round 5's + trick so each row keeps which families were drawn): 300s, uniform mode, seed 0 -- 125,680 queries + sampled, **45,451 kept** after filtering to `count_source == "candidates"` and a non-declined + `GatheredScan` trial. This is the same population `bench_cost_model_agreement.py` itself samples + (same generator), just larger and carrying per-row family/shape metadata the harness doesn't keep. +- **Structured-connective sample** (`QuerySampler.structured_query()`, which draws `Or`/negated/ + parenthesized/regex shapes `query()` can never produce): 240s, uniform mode, seed 1 -- 47,058 + sampled, **29,192 kept**. `bench_cost_model_agreement.py` cannot see this population at all -- + `sampler.query()` only ever emits a flat conjunction -- so it is invisible to the checkpoint number + above regardless of how large its error turns out to be. + +Per row: `predicted_ns` = `costbench.predicted_ns` (the `GatheredScan` trial's `predicted_ns`), +`measured_ns` = `costbench.plan_self_ns` (the same netting rule the checkpoint gate uses -- `candidates` +is in neither `RANGE_ACQUIRES` nor exempt, so `plan_self_ns` is the executor alone, no `ns_prepare` +added back). Feature-level: `explain`'s own `acquire.scan_units`/`acquire.eval_domain` against the +`GatheredScan` trial's real `printings_examined`/`cards_visited` counters -- the same pairing Rounds +3-7 used for the range family, applied here to `Prep::Candidates` for the first time. + +**Which feature is actually mismatched -- checked, not assumed.** Over the flat-conjunction sample: + +``` +eval_domain / cards_visited n=30,794 median 1.00 p10 1.00 p90 1.00 (essentially exact) +scan_units / printings_examined n=30,586 median 1.00 p10 0.52 p90 3.00 (noisier, but not + systematically over) +overall measured_ns / predicted_ns n=45,451 median 0.49 p10 0.24 p90 0.86 within25% 8% +``` + +`eval_domain` is exact everywhere sampled. `scan_units` is close to exact at the pooled median and, in +several of the highest-magnitude buckets below, **under**-predicts (real `printings_examined` bigger +than the estimate) -- the opposite direction from the pooled ns-space over-cost. Neither size feature +is the mismatched one; the bug is downstream, in how `GatheredScan`'s rate/fixed constants +(`cost.rs`'s `PhysicalPlan::GatheredScan` arm) convert these already-correct features into nanoseconds +for this acquire branch specifically. + +**Ranked bucket table** (flat-conjunction sample, `structure:sorted-category-tuple`, same taxonomy +style as Round 5 but rebuilt for this population -- `arith`/`text`/`collection`/`broadcast`/`range`/ +`rarity`/`legality`/`loyalty` categories, since Round 5's range-family taxonomy under-describes a +population dominated by families no printing-range machinery ever sees): + +``` +bucket n share(abs ns err) med_ns med_scan_units med_eval_domain within25% +single:arith 2188 25.3% 0.66 1.00 1.00 0% +and2:arith+range 681 16.6% 0.71 1.74 1.00 30% +single:collection 724 13.9% 0.66 1.00 1.00 4% +single:text 8420 6.6% 0.58 1.00 1.00 14% +and2:collection+range 289 5.8% 0.61 1.70 1.00 12% +and2:range+text 2799 4.4% 0.63 1.16 1.00 19% +and2:arith+rarity 142 4.2% 0.59 1.72 1.00 27% +and2:arith+broadcast 662 2.7% 0.66 1.00 1.00 1% +and2:broadcast+collection 269 2.6% 0.49 1.00 1.00 1% +and2:arith+collection 1291 2.6% 0.52 1.00 1.00 8% +``` + +(`arith` = extended syntax over `power`/`toughness`/`cmc` compounds, never `is_broadcast_leaf_shape`- +eligible since that gate requires a bare `NumField`, not a `NumExpr::Add`, so every arith predicate +lands in `candidates` unconditionally; `collection` = `type`/`keyword`/`tag`/`produces`/`set`/`border`/ +`frame`/`watermark`/`devotion`; `broadcast` = `color`/`identity`/`cmc`/`pow`/`tou` singleton leaves that +usually escape to `Prep::Plane` but land here when paired with a non-composable partner.) + +**Not shape-concentrated -- broad-based instead.** Every top-10 bucket reads `median_ns` in a tight +0.49-0.71 band regardless of which families are involved -- text-only, collection-only, and every +arith combination all cluster together. This is the opposite of Rounds 3-7's range-leaf findings, +where the fix was scoped to one precise shape; here the shape taxonomy is not the axis that +separates fixed from broken. `scan_units`'s per-bucket median tells the same story from a different +angle: it reads exactly 1.00 (agreeing with the real count) for every bucket where the query's +predicates carry high selectivity relative to the corpus, and 1.16-1.74 (UNDER-predicting) for the +`range`/`rarity`-paired buckets -- i.e. the one feature that DOES vary across buckets moves in the +wrong direction to explain a uniform over-cost. + +**What actually separates fast-and-cheap from over-costed: `eval_domain` SIZE and match rate, not +shape.** Cutting the same sample by predicted `eval_domain` decile: + +``` +eval_domain range n median ns_ratio +0 13,635 0.25 (deciles 0-2, exactly zero candidates) +(0, 2] 4,545 0.39 +(2, 9] 4,545 0.46 +(9, 23] 4,545 0.53 +(23, 57] 4,545 0.60 +(57, 161] 4,545 0.62 +(161, 937] 4,545 0.68 +(937, 31724] 4,546 0.68 +``` + +and by verify-cost tier (`residual_tier_ns100`, from `filter.rs`'s `verify_cost_tier`): + +``` +tier n share(abs ns err) median ns_ratio +MASK_COMPARE (400) 7,102 49.0% 0.49 +0 / all_match_known 15,590 39.3% 0.57 +SET_LOOKUP (900) 16,490 9.6% 0.47 +TEXT_SCAN (2,300) 5,578 1.4% 0.38 +REGEX_MACHINERY (5,000) 691 0.7% 1.59 +``` + +The two biggest tiers by magnitude (MASK_COMPARE, all_match_known) are not the two most *miscalibrated* +by ratio -- they dominate by ROW COUNT (88% of rows between them), same "volume, not tier-specific +miscalibration" pattern the doc has seen before. The real signal is the monotonic decay above: ratio +degrades steadily as `eval_domain` shrinks toward zero, which points at **two separate, compounding +mechanisms** rather than one shape-specific bug: + +1. **`GATHER_FIXED_COST_NS` (169.6ns) is ~4x too high for zero-match rounds.** 14,657 of the 45,451 + sampled rows (32%) have `matches == 0` -- every multiplicative term in `PhysicalPlan::GatheredScan`'s + `cost.rs` formula vanishes, so `predicted_ns` collapses to exactly `GATHER_FIXED_COST_NS` (median + predicted 169.6ns, matching the constant to the decimal). Real measured cost for these rounds: median + 42.0ns -- a clean, isolated, shape-independent 4x over-charge with no other term involved. Cheap in + absolute ns per query, but 32% of the whole `candidates` population by row count, so it alone would + move a meaningful share of the within-25% pass rate. + +2. **Card-mode's `feats.matches = count` (unconditional, `candidate_feats`, `lib.rs` ~11776) ignores real + residual selectivity, and the per-candidate verify-tier charge doesn't discount for it either.** + Printing/artwork mode already has a residual-pass-rate discount here (`RESIDUAL_PASS_RATE_PRINTING`/ + `_ARTWORK`); card mode has none -- `matches` is the full candidate count regardless of whether + `all_match_known` holds. Concrete example, resampled 41 times in this run (`is:vanilla`, a static + `tag`-family value): `eval_domain = pred_matches = 17,437` (`residual_card_invariant = true`, tier + `MASK_COMPARE`), but `real_matches_pushed = 343` -- **2.0%** of predicted. `predicted_ns ≈ 601,657`, + `measured_ns ≈ 95,000`, ratio **0.16** -- worse than the zero-match mechanism above, and at a LARGE + `eval_domain`, contradicting a naive "small eval_domain only" read of the decile table. Not an + `is:`-specific artifact: the same `eval_domain >= 2,000` + `MASK_COMPARE` slice (n=756, 684 distinct + queries) reads median ratio 0.67, and the non-`is:` members alone (`t:creature year>2001` ratio 0.39, + `cmc>=power year>=1997` ratio 0.34, `name:s eur<=5.06` ratio 0.48, ...) show the same direction and + comparable magnitude. Over the whole residual-present population (`tier > 0`, n=29,861): + `real_matches_pushed / pred_matches` reads median 1.000 (most queries genuinely do have most + candidates match) but **p10 0.033** -- a real, fat left tail of 30x-overestimated match counts, not + a single outlier. `GATHER_PUSH_PER_MATCH_NS` (2.24 ns/match) explains only part of the gap in the + `is:vanilla` example (~39K ns of the ~507K ns predicted-minus-measured gap); the dominant term is + `eval_domain * (GATHER_LOOP_PER_CARD_NS + GATHER_CARD_PASS_NS + max(tier_ns, GATHER_RESIDUAL_FLOOR_NS))` + (~449K ns of that gap) -- i.e. the flat per-candidate verify-tier charge itself is too high whenever + the residual is this selective, plausibly because a real `card_pass` short-circuits cheaply on most + candidates at low match rates in a way `verify_cost_tier`'s single-node "worst child wins" model + cannot see, and `GATHER_RESIDUAL_FLOOR_NS` (18.89, calibrated -- per its own doc comment -- against + `bench_streamed_loop`'s always-true `DateCmp` design, a HIGH-match-rate population) may not transfer + to a low-match-rate residual the way that comment's own precedent ("the third time this file has + caught the same artifact") would predict. Both `residual_card_invariant = 0` (n=25,677, median ratio + 0.47) and `= 1` (n=4,184, median ratio 0.38) show the same direction, so this is not exclusive to + card-invariant residuals either. + +**Pooling check (the task's explicit ask): does the over-cost direction hold uniformly, or does it +mask an opposite error?** Within the flat-conjunction sample, YES it holds uniformly at the AST-shape +level (every top-10 bucket's median sits in 0.49-0.71, no bucket flips sign) -- but the +`scan_units`-feature check above already found the masked opposite: several buckets' `scan_units` +*feature* under-predicts (1.16-1.74x) inside the SAME rows whose *time* prediction over-costs, meaning +a naive "fix scan_units" reading of this cell would move the wrong lever. The real masking is +structural rather than per-bucket: `bench_cost_model_agreement.py`'s flat-conjunction sampler cannot +produce the population below at all, so its 13% headline is blind to it entirely, not merely diluting it. + +**A third, structurally invisible population: `Or`/negation/nested-paren connectives.** Sampled via +`structured_query()` (`STRUCTURES`, never reachable through `sampler.query()`), 29,192 candidates rows: + +``` +structure n share(abs ns err) median ns p10 p90 within25% +regex 3,812 33.4% 0.54 0.23 1.28 7% +neg-or 3,895 18.8% 0.96 0.36 2.27 23% +or3 2,203 16.3% 0.72 0.36 1.41 27% +and-of-ors 2,650 9.7% 1.05 0.40 3.48 22% +or2 1,668 8.0% 0.68 0.34 1.28 21% +neg-and 2,519 5.9% 0.58 0.29 1.25 17% +paren-or 2,637 4.9% 1.07 0.40 2.62 21% +and-or 2,238 2.4% 0.68 0.25 1.96 19% +and2/and3/and4/single (this run) 7,570 0.5% 0.25-0.63 0.24-0.41 0.69-0.93 4-10% +``` + +This population's median ratios (0.54-1.07) look far closer to the `[0.8, 1.25]` bar than the flat- +conjunction population's do -- but the p90 column tells the opposite story: 1.25-3.48x, a severe +UNDER-cost tail, the OPPOSITE direction from the flat-conjunction over-cost. Pooling this in with the +flat population (which the real `bench_cost_model_agreement.py` never does, since it cannot sample +`Or`/negation at all) would report something close to "fine," masking a tail that is large enough by +row-count share (regex alone is 33.4% of THIS sample's pooled error) to plausibly drive real routing +regret -- a query whose true cost is 2-3x its prediction can lose an argmin to a plan that looks +cheaper on paper but isn't. This population needs its own round; it cannot be fixed by the same lever +as the flat-conjunction findings above (median direction is opposite), and no existing harness tracks +it at all. + +**What Round 9 should target, in order:** + +1. **`GATHER_FIXED_COST_NS` for zero-match `candidates`-acquired `GatheredScan` rounds** (mechanism 1 + above) -- cleanest, most isolated, no shape dependency, same "precomputed floor constant" pattern as + every prior round's fix; likely the highest-confidence, lowest-risk first move given how cleanly it + isolates (predicted collapses to exactly one constant, real measured is a flat ~42ns). +2. **Card-mode's `feats.matches` / the per-candidate verify-tier charge at low real match rates** + (mechanism 2) -- larger in magnitude (dominates the top-10 bucket table) but needs a genuine + calibration/held-out split against the real `card_pass` short-circuit behavior before trusting a + constant, not just a flat scale reused from mechanism 1; the price-triple-style correlation check + from Rounds 2-3 has no equivalent risk here (no independence-product combination proposed), but the + held-out split discipline from every prior round still applies. +3. **The `Or`/negation/nested-paren population**, once 1-2 are shipped and re-measured -- needs its own + sampler wired into whatever harness tracks it going forward, since `bench_cost_model_agreement.py`'s + own generator structurally cannot see it. + +### Round 9 + +Took Round 8's item 1 (`GATHER_FIXED_COST_NS` for zero-match `candidates`-acquired `GatheredScan` +rounds). First fix in this doc that lives in `cost.rs`'s cost FORMULA rather than `lib.rs` feature +estimation -- Rounds 1-8 all fixed a feature (`scan_units`/`eval_domain`) feeding an otherwise-correct +formula; here the features are already exact and the RATE/FIXED constant converting them to ns is +wrong. + +**Independent re-confirmation of Round 8's diagnosis.** Fresh sample (not Round 8's own, a new +throwaway sampler, uniform mode, seed 0, 240s, isolated release wheel): 31,030 `GatheredScan`/ +`candidates` rows, 9,890 (31.9%) with `matches == 0` -- matching Round 8's reported 32% closely. +Checked every non-fixed term individually rather than trusting the "collapses to one constant" claim: + +``` +field nonzero_count / n max +eval_domain 0 / 9,890 0 +scan_units 0 / 9,890 0 +artwork_seen_printings 0 / 9,890 0 +cards_visited 0 / 9,890 0 +printings_examined 0 / 9,890 0 +matches_pushed 0 / 9,890 0 +``` + +Every term this arm multiplies by really is zero (not just small) for this population, so +`predicted_ns` reads EXACTLY `169.6` (min == max == median across all 9,890 rows) -- confirmed, not +assumed. Real measured `plan_self_ns`: median 42.0, p10 41.0, p90 84.0 (bimodal: card/printing modes +cluster at ~42, artwork at ~84 -- see below). Ratio (measured/predicted): median 0.248, 0/9,890 (0.0%) +within [0.8, 1.25] -- confirms the ~4x over-charge exactly as Round 8 reported. + +**Calibration.** Hash-of-query split (`sha256(q) % 2`), same rule as every prior round. Calibration +half (n=4,944): the L1-optimal single constant is the median measured `plan_self_ns`, `42.0` -- chosen +without looking at the held-out half. Held-out half (n=4,946): + +``` + before (169.6) after (42.0) +median ratio 0.248 1.000 +within-25% 0.1% 57.7% +total abs ns error 530,256 103,110 (5.1x reduction) +paired diff: 4,577 improved / 369 regressed / 0 tied +``` + +**Per-mode split, not chased.** `PlanFeatures` carries no `unique`/mode field this arm can read, so +one pooled constant is what `cost.rs` alone can express. Held-out breakdown by mode: card (n=1,657) +83.6% within-25%, printing (n=1,625) 90.0%, artwork (n=1,664) 0.3% -- artwork's real zero-match cost +reads a flat ~2x higher (~84ns vs. card/printing's ~42ns, plausibly `exec_gathered_scan`'s +unconditional per-printing dedupe check setup), so a single pooled constant necessarily leaves +artwork's ratio at ~2.0 (previously ~0.495 -- same log-magnitude, flipped sign, and still a net win on +absolute ns error: |84-169.6|=85.6 -> |84-42|=42.0). Splitting this by mode needs a new `PlanFeatures` +field, which needs a `lib.rs` change -- out of scope for a `cost.rs`-only round, noted for later. + +**A gate-precision risk found and checked, not assumed safe.** `matches == 0` is not exclusive to the +`candidates`/`plane` acquire branches Round 8 scoped its diagnosis to. `explain_analyze` costs every +CANDIDATE plan from one shared `PlanFeatures` per acquire (`plan_cost(plan, &facts.feats)`, called +once per plan in a loop at `lib.rs:12917`), so `GatheredScan`'s own `matches == 0` also fires when it +is costed under a `printing_compose`/`card_range_popcount`/`printing_range_scan` (RANGE_ACQUIRES) +acquire branch -- and there, `eval_domain == 0` is not a real empty candidate list, it is this +branch's shared `feats` never having computed one for `GatheredScan` specifically (the acquire chose a +different plan and never ran `prepare_candidates`). If `GatheredScan` is later picked or forced as a +competitor, dispatch pays a REAL `prepare_candidates` rebuild (`plan_self_ns` adds `ns_prepare` back in +for RANGE_ACQUIRES, per `costbench`'s netting rule) that no term in this arm prices at all -- sampled +directly: 375 `printing_compose`-acquired `GatheredScan`/`matches==0` rows, 358 with `eval_domain==0`, +median measured 4,959ns against a predicted 169.6ns (29x under, PRE-EXISTING, not caused by this +round). Lowering the fixed cost to 42.0 makes this already-broken slice numerically worse in isolation +(29x -> 118x under) — same direction, no new sign flip. + +Checked for a REAL regression, not just reasoned about: this population is not purely diagnostic — +`GatheredScan` is the actually-`picked` plan in 93/96 (97%) of sampled zero-match `printing_compose`- +acquire rows. Directly diffed a same-build wheel and found 2 genuine routing flips in a 107-row sample +of RANGE_ACQUIRES zero-match rows where a competing plan's predicted cost sat between the old (169.6) +and new (42.0) constant (`date<1993-08-05` and `tix<0.01` under `printing_range_scan`: `PrintingRangeScan` +predicted 150.0 picked at baseline, `GatheredScan` predicted 42.0 picked after this round's change). +Ran the two tools built to catch exactly this: + +- `bench_regret_matrix.py --seconds 60 --seed 0`: total regret 27.6ms baseline, 27.6ms modified (18,181 + vs 18,349 multi-plan queries — wall-clock-budget variance, not a code effect); no new row in the + `picked -> best` mismatch table. +- `bench_cost_model_agreement.py --seconds 300 --seed 0`: `GatheredScan/printing_compose` unchanged + (n=58,444→59,178, median 1.15→1.14, within-25% 24%→24%); `GatheredScan/printing_range_scan` unchanged + (median 1.09→1.08, 61%→62%); `GatheredScan/card_range_popcount` unchanged (median 0.96→0.97, 51%→51%). + 12/17 acquire-branch cells inside [0.8, 1.25] both builds (unchanged); by-unique table improves 9/12 + -> 10/12 (`GatheredScan/card` flips FAIL -> PASS). + +So the affected RANGE_ACQUIRES slice is real, pre-existing, and made worse in isolated ratio terms, but +too small (2 flips in 107 sampled rows; the whole slice is ~2% of its own already-passing pooled cell) +to move any reported cell or the regret total. `matches == 0` is therefore a correlated proxy, not the +exact phenomenon Round 8 scoped ("`Prep::Candidates` zero-match"), and a future round that can touch +`lib.rs` should add an acquire-branch-aware feature (or a `real_candidates_built: bool`) to gate this +cleanly rather than relying on this round's empirical "checked, found immaterial" result indefinitely. + +### Round 28 + +Round 27 ([reference-engine-cost-model-cleanup-final-ab.md](reference-engine-cost-model-cleanup-final-ab.md)) +ran the first fresh, paired `main`-vs-`costcell/trunk` A/B this whole 27-round effort had done and +found a real, previously-invisible regression: `bench_feature_accuracy.py`'s pooled `scan_units` +feature (graded against the real `printings_examined` counter, not against a rate-fit like +`bench_cost_model_agreement.py`) reads clean on `main` (median 1.00) but `UNDER-COUNTS` on +`costcell/trunk` (median 0.70). This section is the follow-up round tasked with finding and fixing it. + +**Bisection.** Built an isolated release wheel at every `Engine:` commit between `main` and +`costcell/trunk`'s tip (17 candidates) and ran `bench_feature_accuracy.py`'s pooled `scan_units` +reading at each. Clean through Round 6 (`ce860337`, pooled median 0.94). The very next commit, +`e1c40466` ("A Broad-Guard Scale for PrintingCompose's Own Bare/Fused Range Reset", this doc's own +Round 7 above), drops it to 0.69 — the exact commit that tips the pooled metric from PASS to FAIL. +Every commit after that (Rounds 9, 14/15's verify-bypass work, Round 22's `best_other` gate, Round +24's `PairTotals` extension) holds steady at 0.68-0.70, confirming Round 7 is the trigger, not a later +round compounding an already-broken number. + +**Mechanism.** `e1c40466`'s own fit — and Round 4's `COMPOSE_RANGE_AND_BROAD_SCAN_SCALE` fit above it +— were each calibrated exclusively against `unique=card` samples (Round 4: "Sampled 1,500 and2/and3 +RANGE_FAMILIES queries (`unique=card`..."; Round 7: "the same shape" as Round 6's own +`unique={"card"}` generator). But the guard both scales live in (`acquire_plan_features`, the branch +starting `let (eval_domain, scan_units) = if ... range_too_broad_to_narrow(...)`) runs *after* the +`match mode { Mode::Printing => ..., Mode::Card => ..., Mode::Artwork => ... }` block, unconditional on +`mode` — so both scales were applied to `Mode::Printing`/`Mode::Artwork` too, shapes neither +calibration sample ever contained. + +Checked directly rather than assumed: a fresh sample of this exact guard-fired population, split by +`unique`, reading `printings_examined / n_printings` (the real, unscaled ground truth): + +``` +('broad', 'card'): n=303 p10=0.520 p50=0.520 p90=1.127 +('broad', 'printing'): n=230 p10=0.520 p50=0.520 p90=0.520 +('broad', 'artwork'): n=956 p10=0.520 p50=0.520 p90=0.520 +``` + +`Mode::Printing`/`Mode::Artwork` read **exactly** 0.520 at every percentile — zero spread, because +`printings_examined == n_printings` on every single row: those two modes' materializing kernels never +short-circuit, so a query broad enough to fire this guard really does walk the *entire* candidate +printing span, always. `Mode::Card`'s own kernels do short-circuit per candidate (the property both +scales were fit to exploit), which is why its own ratio has real spread (p90 1.127, not pinned to +0.520). Applying `COMPOSE_SAME_RANGE_BROAD_SCAN_SCALE`/`COMPOSE_RANGE_AND_BROAD_SCAN_SCALE` to +Printing/Artwork mode was therefore manufacturing a clean, deterministic ~0.52x/0.7x under-count out +of a population whose true ratio is 1.0 — not a mixed population, not noise, a mode-scoping bug with a +single, uniform failure mode. + +This is *also* why the pooled metric moved as much as it did despite `scan_units [printing_compose]`'s +own per-acquire median barely changing (0.39 → 0.39 across the fix): the guard-fired subset is only +~4-12% of `printing_compose`'s rows per mode (dwarfed by the pre-existing, separately-tracked "narrow" +bucket — the `range_too_broad_to_narrow`-NOT-fired population Round 7 above already named and +deferred: "a card-count-shaped estimate undershooting a printing count... not fixed this round"). But +before this fix, those rows sat *above* 1.0 (the OLD unscaled `n_printings` ceiling, ~1.9-2.0x +over-counted per Round 7's own measurement) — moving them down to 0.52 didn't change the sub-bucket's +own median, but it did remove ~1,500-2,200 rows from *above* the global rank used to compute the +POOLED median, letting that rank fall into the dense, already-under-counted "narrow" bucket below it. +Round 7's fix was a real, validated improvement for the `unique=card` population it targeted — the +mode-scoping bug is what let a genuine fix for one mode quietly worsen the pooled number by removing a +compensating error for two others. + +**Fix.** Gated both `is_cross_index_range_and`'s and `is_same_index_range_only`'s scale branches on +`matches!(mode, Mode::Card)`; `Mode::Printing`/`Mode::Artwork` now fall to the existing `else` branch +(the unscaled `n_printings` ceiling, already correct for every other shape reaching this guard). Zero +new computation — `mode` is already a bound local, the added check is a single enum-tag comparison — +so this carries no acquire-time cost, per this doc's own pre-computation constraint. + +**Results** (isolated release wheels, `bench_feature_accuracy.py --seconds 300 --seed 0`, `main` @ +`ca016410`, branch tip @ `865fb03e`): + +``` + pooled scan_units median verdict +main 1.00 (clean) +costcell/trunk (unfixed) 0.70 UNDER-COUNTS +costcell/trunk (fixed) 0.94 (clean) +``` + +The regression is closed: 0.94 sits inside the same `[0.8, 1.25]` agreement band `main`'s own 1.00 +does, with no verdict flag. The residual gap between 0.94 and `main`'s 1.00 is the two *other*, +already-documented, separately-tracked contributors this round did not touch: the "narrow"-bucket +`PrintingCompose` under-count Round 7 itself named and deferred above, and the era-correlated +print-position confound for bare existential leaves +([local-engine-domain-cards-existential-arith-and.md](local-engine-domain-cards-existential-arith-and.md)'s +Round 25 section, "confirmed real, confirmed severe... out of \[that round's\] blast radius"). Both are +real, both pre-date this fix, and neither is a regression introduced by any commit on this branch — +fixing either would need touching `domain_cards`'s own broad-range estimate for bare ranges (the +`RangeCardCounts::distinct_cards` undercount this doc's own `scan_all` comments already name), which +nine prior rounds of this same effort found hard and did not attempt; left open, matching this doc's +own "Next steps for a future round" note under Round 7. + +**Correctness gates.** `cargo test --manifest-path card_engine/Cargo.toml`: 177 passed, 0 failed +(debug); `--release`: 176 passed, 0 failed. `cargo clippy --manifest-path card_engine/Cargo.toml +--all-targets -- -D warnings`: clean. `git diff --stat costcell/trunk`: `card_engine/src/lib.rs` only +(19 lines). + +**Confirmation pass**, before (unfixed tip) vs after (fixed), plus `main` where noted: + +- `bench_regret_matrix.py --seconds 120 --mode realistic` (routed-phases builds): mean regret/query + 0.94µs (tip) → 0.95µs (fixed) → 0.95µs (`main`) — unchanged within noise; `picked -> best` SHARE + table proportionally identical (`Perm` 69%→69%, `Gather` 15%→14%, `StreamedSelect -> GatheredScan` + 50%→47% of a shrinking pie), no anomalous transition. +- `bench_cost_model_agreement.py --seconds 300 --seed 0`: 12/17 (tip) → 13/17 (fixed) cells inside + `[0.8, 1.25]`; every reported cell moved by less than 0.05 in ratio, one boundary flip + (`GatheredScan/candidates` 0.78→0.81) consistent with sampling noise, not a real shift — matches + this tool's own documented insensitivity to a feature-only fix (a rate elsewhere absorbs it). +- `bench_query_latency_ab.py --mode realistic --sample 800 --seed 1`, two order-alternated rounds plus + a same-build canary: round 1 (tip, fixed) `+0.9µs` "B is SLOWER"; round 2 (tip, fixed) `-0.3µs` "B is + FASTER"; canary (fixed vs fixed) `0.0µs`, CI `[-0.2, +0.3]`, no detectable difference. Opposite signs + of comparable magnitude across the two real rounds, both inside the canary's own noise band — no + detectable latency effect, expected for a zero-new-computation accuracy fix. +- `bench_pairwise_ordering.py --seconds 60`, realistic and uniform, `GatheredScan` vs `PrintingCompose`: + realistic overall 89%→89% (`[printing_compose]` 91%→90%, `[plane]` 84%→86%); uniform overall 87%→87% + (`[printing_compose]` 86%→86%). Essentially unchanged in both modes — unlike Round 7's own change, + this fix does not touch the ordering that mattered to `#852`. + +### Round 30 + +**Regression found by a prior diagnostic round, confirmed by bisection + literal replay (not just +correlation):** Round 1's own `scan_all` fix above (the match-density depth proxy) was legitimate and +already validated for `GatheredScan`'s `scan_units` -- but `StreamedSelect`'s own feature, +`stream_scan_units`, defaults to inheriting `scan_units` verbatim (`mk_plan_feats`'s doc: "only an +acquire that knows P3 examines fewer printings overrides it") unless the `printing_compose` acquire's +own override logic (`lib.rs`, the `feats.stream_scan_units = if tier == 0 {...} else if +filter_touches_legality(...) {...} else {...}` block) says otherwise. For a printing-varying leaf with +no legality partner (`price_usd`/`cn`/`released_at`, or an And of them), that block falls to its bare +`else { scan_units as u32 }` arm -- so Round 1's legitimate downward revision to `scan_units` rode +straight through into `stream_scan_units` too, with no acquire branch ever taught the difference. This +grew the `StreamedSelect -> GatheredScan` misroute (router picks P3 when P4 is actually faster) from +1,284 to 1,618 occurrences (mean regret 17.0us -> 21.4us) on matched-size `bench_regret_matrix.py +--mode realistic` runs -- the single largest remaining regret slice on the branch (43% share) going +into this round. + +**Mechanism, confirmed directly against real dispatch counters** (not assumed from reading the code +alone): `run_query_streamed` (P3's executor) runs a first pass (`card_match_count`, over every +candidate) that is structurally identical to `GatheredScan`'s own single pass in `Mode::Card` -- both +break at the first printing satisfying the residual under `Prefer::Default`, confirmed by matching +`printings_examined` counters exactly (2,449 on both plans, `f:pioneer cn>=30 cn<=39`). What differs is +a SECOND pass this first pass's counter never sees: `run_query_streamed`'s `total <= *STREAM_MIN_MATCHES` +branch re-derives `card_pass` and re-walks the printing span for every MATCHING card a second time to +select the page (`push_card_matches`, called again, its return value discarded -- so +`printings_examined`, and therefore any `scan_units`-shaped feature, structurally cannot see this +second pass no matter how it's computed). `cost.rs`'s `StreamedSelect` arm already has a term for this +branch's OWN O(n_cards) "scan every stored count" overhead (`STREAM_SMALL_TOTAL_FLOOR_PER_CARD_NS * +n_cards`), but that floor is a per-CORPUS constant that cannot vary with a query's own `matches` count -- +it was fit on a population where `matches` was small enough that the actual per-card REDO was +negligible next to the floor. On `f:pioneer cn>=30 cn<=39` (853 real matches, close to the +`STREAM_MIN_MATCHES` ceiling of 1,024) the floor alone (32.4us, `n_cards=31,724 * 1.02`) materially +undershoots the real `ns_finish` (65.3us) -- the remainder is exactly this unpriced redo, and it is why +`StreamedSelect`'s real dispatch (108.9us) is 2.3x `GatheredScan`'s (46.6us) despite identical +`printings_examined`. + +**Fix** (`card_engine/src/lib.rs`, the `printing_compose` acquire's `feats.stream_scan_units` bare +`else` arm): adds a `STREAM_SMALL_TOTAL_REDO_BIAS` (`1.32`, a new lib.rs constant, NOT a `cost.rs` rate) +scaled term on top of the inherited `scan_units`, `Mode::Card` only. The redo-candidate count is the +acquire-time `result_total` estimate when it sits at or below `STREAM_MIN_MATCHES` (mirroring the same +threshold `compose_paging_with_total` already gates its own decline prediction on, a few hundred lines +up in the same function), else capped at `feats.limit` (the permutation-walk branch's own bound) rather +than dropped to zero outright -- a hard cliff at the threshold would turn the acquire estimate's own +noise into an all-or-nothing coin flip, which matters here specifically: this round's own concrete +example's acquire-time estimate (1,983) sits ABOVE the 1,024 threshold despite its REAL total (853) +landing inside the small-total branch. + +**Calibration.** Bias fit against `ns_finish` minus the existing floor's own contribution (isolating +the previously-unpriced redo specifically, not re-deriving the floor), converted to `stream_scan_units` +units via the existing, untouched `STREAM_SCAN_PER_ROW_NS` (5.97), over a held-in/held-out split +(hash-of-query, 1,875/1,949 rows) of `unique=card` `printing_compose` rows where the acquire-time +estimate gates the correction AND real dispatch confirms the small-total branch actually ran +(`perm_steps == 0`, `matches_pushed > 0`). Median fitted bias 1.32 "printing units" per redone +candidate; held-out total absolute error on the unpriced remainder: 2.18e7 -> 2.06e7 (a real but +partial reduction -- this population's per-query redo cost is heavy-tailed (implied bias p10 -38.7, p90 +12.7), dominated by per-query residual complexity this feature vector has no term for, not by candidate +count alone). A 4x/8x/30x sweep of the bias against the live routing-outcome metric below (not just the +ns-error metric) showed diminishing returns fast -- 5.9%->7.3% of a broader current-trunk +misroute sample fixed for a doubling of the false-positive rate on already-correct `StreamedSelect` +picks -- so the median (lowest false-positive rate, still measurably useful) was kept rather than +chasing the sweep. + +**Flip-query validation.** Reproduced the ORIGINAL flip population exactly as the diagnostic round's +own `flip_finder_f3f4a017.py` does (BEFORE=`97dc30c8`, AFTER=`f3f4a017`, same seed/sample window): 114 +queries found this run (consistent with the diagnostic round's own ~120, sampling noise). Replayed +against this round's FIX build (current `costcell/trunk` tip + the patch above): + +``` +of 114 reproduced f3f4a017 flip queries: + now correctly pick GatheredScan (fixed): 50 (44%) + still (wrongly) pick StreamedSelect (unchanged): 64 (56%) + pick something else: 0 +``` + +**Regret matrix** (`bench_regret_matrix.py --seconds 300 --mode realistic --seed 0`, isolated release +wheels, baseline = unfixed `costcell/trunk` tip `4e101d7f` vs fix = this round's patch on top). The two +300s windows sampled different absolute query counts (121,724 vs 108,533 multi-plan queries -- system +load from other concurrent work on this box, not a code-speed effect; rates/shares below are the fair +comparison, not raw `n`): + +``` +StreamedSelect -> GatheredScan n share of traffic mean regret SHARE -> ~ms +baseline (unfixed) 2,407 1.98% of 121,724 sampled 23.00us 53% ~55.4ms +fix 1,995 1.84% of 108,533 sampled 24.33us 56% ~48.5ms +``` + +~7% fewer misroutes as a share of traffic, ~12% less absolute regret-ms attributed to this specific +transition. Total pool regret (all transitions) 104.3ms -> 86.2ms (mean/query 0.86us -> 0.79us), roughly +consistent in direction with the targeted slice, not dramatically larger -- no sign the fix disturbed +other transitions. (No dedicated same-build latency canary was run this round on top of this -- the +regret figures come from forced per-plan trial minimums, not wall-clock query timing, which is less +exposed to the sampling-count variance noted above, but a canary would still be the stronger claim; flag +this as the one gap in this round's own validation rigor.) + +**Regression guards.** + +- `#852` (`GatheredScan` vs `PrintingCompose` ordering, `bench_pairwise_ordering.py --seconds 300 + --mode realistic --seed 0`): overall 88% -> 88%, unchanged. By acquire: `[plane]` 83% -> 82%, + `[printing_compose]` 91% -> 92% -- both within noise, no real shift. Clean. +- Round 28's `scan_units` feature-accuracy fix (`bench_feature_accuracy.py --seconds 120 --mode + realistic --seed 0`): pooled `scan_units` median 1.00 -> 1.00, identical distribution shape in both + builds -- expected, since this round's patch touches only `stream_scan_units`, never `scan_units` + itself. Clean. + +**Correctness gates.** `cargo test --release` (`card_engine`): 176/176 passed. `cargo test` (debug): +177/177 passed. `cargo clippy --all-targets -- -D warnings`: clean. Blast radius: `card_engine/src/lib.rs` +(the `printing_compose` acquire branch only) plus this doc; `cost.rs`, `estimator.rs`, `filter.rs` +untouched. + +**Verdict.** Real, positive, but partial. On the population this round diagnosed and targeted directly +(the reproduced f3f4a017 flip set), 44% now route correctly again. On the broader regret matrix, the +`StreamedSelect -> GatheredScan` transition's regret is down ~7-12%, not back to `main`'s pre-regression +baseline. The residual is NOT well-explained by `cost.rs`'s rates (chunk 2's stated scope) -- it traces +to the acquire-time `result_total` estimate itself being unreliable near the `STREAM_MIN_MATCHES` +threshold for cross-index-range-leaf Ands (this round's own concrete example: real total 853, estimate +1,983, off by 2.3x), which is the SAME "separate, uninvestigated `domain_cards` bug for multi-range-index +Ands" this doc's own Round 1 section flagged as "the natural next target" and never chased. A future +round fixing that upstream cardinality estimate would likely close more of this residual than any +`cost.rs` rate refit; chunk 2 (the rate refit) still looks worth doing on its own merits but should not +be expected to finish closing this specific misroute on its own. + +### Round 31 + +**The gap Round 30 flagged but couldn't close.** Round 30 fit `STREAM_SMALL_TOTAL_REDO_BIAS` against +`ns_finish` minus the existing floor's own contribution -- a wall-clock RESIDUAL, converted to +`stream_scan_units` units via the untouched `STREAM_SCAN_PER_ROW_NS` rate -- because no structural +counter existed for the redo pass's real work. `push_card_matches` (`lib.rs:6186`) already computes +and returns a `u32` "examined" count per call, mirroring `card_match_count`'s own `(c, examined)` +pattern -- both calls inside `run_query_streamed`'s `total <= *STREAM_MIN_MATCHES` branch's second +loop (~line 13940) simply discarded it as a bare statement. + +**Step 1: the counter.** Added `PhaseStats::redo_examined: u64`, a new field zero everywhere except +this one branch (following the `set_printings`/`perm_steps` precedent: doc-declared scope, zeroed +explicitly at the other two exits of `run_query_streamed` -- the empty/past-the-end return and the +permutation walk). The small-total loop now accumulates `push_card_matches`'s return value into a +local (`n_redo_examined`) and passes it to the `publish` closure, which now takes a fourth parameter +alongside `perm_steps`. The permutation walk's OWN `push_card_matches` call (after `'walk: for cid in +walk.iter()...`) is deliberately left uninstrumented: that branch runs above `STREAM_MIN_MATCHES`, +already prices to `limit`, and its own per-step cost already flows into `ns_loop`/`ns_finish` via the +walk's wall-clock timing, the same population `perm_steps` was already calibrated against -- this +round's scope is specifically the small-total branch's previously-unpriced second pass. Surfaced to +Python exactly like `printings_examined`/`perm_steps`: a new `d.set_item("redo_examined", ...)` line in +`plan_trial_to_pydict`, and a matching entry in `scripts/costbench.py`'s `PLAN_KEYS` schema assertion. + +**Free, confirmed directly, not assumed.** `push_card_matches` already computed this value on every +call in this loop; capturing it is a return-value read, not a new pass or a new computation -- no +counter, no extra field write, nothing added to what the loop already does. Confirmed both ways: (a) +by inspection -- the diff is exactly "capture the return instead of discarding it" -- and (b) directly: +temporarily reverting the capture to a bare statement (matching pre-round code) makes the new +regression test fail on its very first assertion (`redo_examined > 0`), and restoring it passes again, +with `cargo test --release` timing unaffected in either direction (the change is a single local +accumulate plus one extra `u64` in an already-stack-allocated struct). + +**Regression test** (`card_engine/src/tests.rs`, +`redo_examined_counts_only_the_small_total_redo_pass`): one synthetic corpus, two disjoint match groups +(500 cards under `STREAM_MIN_MATCHES`, 2,000 over it), asserting `redo_examined > 0` and +`>= matches_pushed` on the small-total exit (`perm_steps == 0`), `== 0` on `GatheredScan` for the +identical query, and `== 0` on the walk exit (`perm_steps > 0`) for the large group. Verified to +actually catch a revert: reverting the capture to a bare statement fails the test's first assertion +with `redo_examined read 0`, exactly the Round 30 gap this round closes. + +**Step 2: the refit.** Sampled `unique=card` `printing_compose` rows from `bench_regret_matrix.py +--mode realistic`'s own corpus (isolated release wheel, `--seed 13`, hash-of-query +held-in/held-out split: 2,916/3,006), gated on the same real-dispatch confirmation Round 30 used +(`perm_steps == 0`, `matches_pushed > 0`) PLUS a guard Round 30's own gate missed: `page_offset < +matches_pushed`, ruling out the OTHER `perm_steps == 0` exit (`page_offset >= total` returns before the +redo loop ever runs but still reports the counting pass's `matches_pushed`) -- without it, 816/6,747 +rows silently poisoned the fit with a real redo pass that never happened. + +`redo_candidates` mirrors the acquire branch's own logic exactly: the acquire-time `matches` estimate +when it's at/under `STREAM_MIN_MATCHES`, else capped at the page `limit`. Two real summary statistics +of `real_redo_examined / redo_candidates` over the calib half, and they disagree: + +``` +median (per-row ratio) 1.0 p10=0.15 p90=10.2 +candidate-weighted mean (sum/sum) 2.237 +``` + +Held-out total absolute error on the real `redo_examined` counter itself (the POINTWISE metric): + +``` +old (1.32, Round 30's wall-clock fit) 2.467e6 +median (1.0) 2.380e6 <- best pointwise fit +weighted mean (2.237) 2.752e6 +p75 (3.831) 3.336e6 +``` + +By pointwise error alone, the median (1.0) wins -- a real, ground-truth-validated improvement over +1.32. But this population's ratio is heavily right-skewed (p10 0.15, p90 10.2: most rows sit near or +under 1.0, but a long tail runs into double digits), and the flip-query population this bias exists to +fix draws disproportionately from that tail -- a query only flips to the wrong plan when its real redo +cost was under-priced, which is exactly what the tail rows are. Checked directly rather than assumed: +replaying the same reproduced f3f4a017 flip set (below) against both candidates, the median +ACTIVELY REGRESSES queries Round 30's own 1.32 already fixed correctly, gaining nothing back. This is +the same false-positive/false-negative asymmetry Round 30's own 4x/8x/30x bias sweep found against its +noisier wall-clock-derived distribution -- resolved here against a real ground-truth counter instead of +a guessed multiplier. The ratio is flat (~1.0-1.3) across every candidate-count bucket (0-50, 50-150, +150-400, 400-1024), so the skew is in per-query residual complexity, not candidate count -- a flat +linear bias remains the right shape, matching Round 30's own conclusion. + +**Fix.** `STREAM_SMALL_TOTAL_REDO_BIAS` set to **2.237** (the candidate-weighted mean), not the +pointwise-optimal 1.0 -- kept because it is the real, structurally-grounded statistic that does not +regress the live routing outcome, following the same "live outcome over pointwise ns-error" precedent +Round 30 itself set with its own bias sweep. + +**Flip-query validation.** Reproduced the ORIGINAL flip population exactly as `flip_finder_f3f4a017.py` +does (BEFORE=`97dc30c8`, AFTER=`f3f4a017`, same seed/sample window), then replayed the SAME reproduced +list against three FIX builds in one script (removing the sampling-window noise a separately-run +validation would carry): Round 30's own tip (`9668dfa4`, bias 1.32), this round's pointwise-optimal +median (1.0), and this round's shipped weighted-mean (2.237). + +``` +of 118 reproduced f3f4a017 flip queries: + round30 (bias=1.32): fixed 52 still wrong 66 + round31 median (bias=1.0): fixed 43 still wrong 75 (regresses 9 of round30's 52, gains 0) + round31 weighted-mean (2.237): fixed 64 still wrong 54 (regresses 0 of round30's 52, gains 12) +``` + +The shipped bias (2.237) regresses none of Round 30's 52 correct fixes and closes 12 more -- 64/118 +(54%) now route correctly, up from Round 30's own 52/118 (44%) on this exact reproduced population (the +114/50 figure in Round 30's own doc entry came from a separate sampling run; both are the same +population modulo the classification-timing noise this whole method carries, already flagged in Round +30's own verdict). + +**Regret matrix** (`bench_regret_matrix.py --seconds 300 --mode realistic --seed 0`, isolated release +wheels with `routed-phases`, before = `costcell/trunk` tip `9668dfa4` i.e. Round 30's own shipped fix, +after = this round's patch): + +``` +StreamedSelect -> GatheredScan n share of traffic mean regret SHARE -> ~ms +before (Round 30's fix) 2,363 1.87% of 126,203 sampled 23.01us 49% ~54.8ms +after (Round 31's refit) 2,129 1.73% of 123,143 sampled 9.80us 25% ~20.7ms +``` + +Mean regret on this transition drops by 57% (23.01us -> 9.80us) and its SHARE of all lost time nearly +halves (49% -> 25%) -- ~54.8ms -> ~20.7ms attributed, a **62% reduction**, dwarfing Round 30's own +55.4ms -> 48.5ms (~12%). Total POOL regret (every transition) also drops, 111.9ms -> 82.7ms (mean/query +0.89us -> 0.67us) -- consistent in direction with the targeted slice, not an isolated artifact. + +One nearby transition moved the other way and is worth naming rather than burying: `PrintingCompose -> +StreamedSelect` (compose picked, but StreamedSelect was really best) grew from 12% to 24% share (mean +34.28us -> 40.61us, n 396 -> 483, ~13.4ms -> ~19.8ms, +6.4ms) -- a real, expected side effect of raising +`stream_scan_units`: making StreamedSelect look pricier tips a few close compose-vs-stream calls the +other way when StreamedSelect actually was faster. Every other transition moved by less than 2 points of +SHARE in either direction. The target slice's ~34ms improvement outweighs this ~6ms give-back by 5:1, +and the total-pool number (111.9ms -> 82.7ms, -29.2ms net) confirms the net effect across the whole +matrix is a real improvement, not a wash. + +**Regression guards.** + +- `#852` (`GatheredScan` vs `PrintingCompose` ordering, `bench_pairwise_ordering.py --seconds 300 + --mode realistic --seed 0`): overall 88% -> 88%, unchanged. By acquire: `[plane]` 83% -> 83%, + `[printing_compose]` 91% -> 91% -- identical in both builds, no shift at all. This round's own + target pair, `GatheredScan` vs `StreamedSelect`, also held steady (97% -> 97% overall, 92% -> 92% + `[printing_compose]`, 99% -> 99% `[candidates]`) -- the ordering `stream_scan_units` exists to get + right did not regress even though its predicted GAP shrank (gap meas/pred 1.08 -> 0.55 overall): + the model now predicts a LARGER gap than measured on this pair (conservative, not wrong-signed), + and argmin correctness -- which side of the gap wins -- is what this guard actually checks. Clean. +- Round 28's `scan_units` feature-accuracy fix (`bench_feature_accuracy.py --seconds 120 --mode + realistic --seed 0`): pooled `scan_units` median 1.00 -> 1.00, identical distribution in both + builds -- expected, since this round's patch touches only `stream_scan_units`, never `scan_units` + itself. Clean. +- Round 30's own fix: the flip-query check above IS this guard -- 0 of the 52 queries Round 30 fixed + regressed under this round's refit. + +**Correctness gates.** `cargo test --release` (`card_engine`): 178/178 passed (177 + this round's new +regression test). `cargo test` (debug): 179/179 passed. `cargo clippy --all-targets -- -D warnings`: +clean. Blast radius: `card_engine/src/lib.rs` (the new counter, its plumbing, and the +`printing_compose` acquire branch's redo-bias constant), `card_engine/src/tests.rs` (one new +regression test), `scripts/costbench.py` (the `PLAN_KEYS` schema entry for the new field), this doc. +`cost.rs` untouched, per this round's own scope. + +**Verdict.** Real, significantly larger, and better-grounded than Round 30's own fix. Cumulatively +(Round 30 + Round 31 together), the `StreamedSelect -> GatheredScan` transition's attributed regret +goes 55.4ms (Round 30's own "before") -> 48.5ms (Round 30's fix, ~12% closed) -> ~20.7ms (this round, +~62% closed relative to Round 30's own before-state) -- five times the closure Round 30's wall-clock-fit +bias achieved, using the SAME feature-level lever, just fit against real structural ground truth instead +of a noisy residual. On the reproduced flip-query population this ledger entry has tracked since Round +30: 44% (52/118) -> 54% (64/118) correctly routed, with zero regression of Round 30's own fixes. + +It is not fully closed. 46% of the reproduced flip population (54/118) still wrongly picks +`StreamedSelect`, one nearby transition (`PrintingCompose -> StreamedSelect`) grew by ~6.4ms as a real +side effect of raising `stream_scan_units` (a 5:1 trade against the ~34ms gained, not free), and Round +30's own diagnosed DEEPER root cause -- the acquire-time `result_total` cardinality estimate itself +being unreliable for cross-index-range-leaf `And`s near the `STREAM_MIN_MATCHES` threshold (a +`domain_cards` estimation bug, the same "natural next target" this doc's own Round 1 section flagged +and no round has yet chased) -- is completely untouched by this round. This round improved WHAT the +bias is fit against (real counter vs. wall-clock residual) and refit the constant accordingly; it did +not touch `redo_candidates`' own input (the acquire-time estimate that feeds it), which is where the +residual almost certainly still lives. + +On the parent punch-list's chunk 2 (`cost.rs` rate refit, `STREAM_SCAN_PER_ROW_NS` itself): this +round's own data argues against urgency there, not for it. The real ratio read flat across every +candidate-count bucket (no saturation, no shape mismatch a rate change would fix), and a feature-level +fix alone -- with no `cost.rs` change at all -- closed 5x more of this regression than Round 30's own +attempt. A rate refit was never tested directly this round and remains formally open, but the +evidence so far suggests the acquire-time cardinality estimate (not the per-unit rate) is the more +promising next target, exactly as Round 30's own verdict already concluded. + +### Round 32 + +**A different term than Rounds 30/31** (`walks_permutation`, the branch taken when `total > +STREAM_MIN_MATCHES`, as against Rounds 30/31's small-total gather), flagged by `cost.rs`'s own +`perm_steps` comment: the estimate (`page_span * n_cards / matches`, capped at `n_cards`) assumes +matches spread uniformly across the WHOLE corpus, but the real executor (`exec_streamed_select`) +starts and ends the walk at `walk_bounds`'s segment -- the slice the filter's own interval on the SORT +COLUMN admits, which the comment's own regrade table already showed matters (`unbounded` p90 6.43 vs +`sort-column bound` p90 5.31) without ever explaining why the bounded variant was never shipped, or +distinguishing it from the third, explicitly-rejected variant (a realized `inv_perm` span, correctly +declined for costing 0.51ns/matching card -- a real per-candidate hot-path cost this effort's +pre-computation constraint forbids). + +**Why it was never shipped: not infeasible, just never circled back to.** Read `walk_bounds` and its +caller (`exec_streamed_select`, `lib.rs:10707`) and the acquire pipeline in full before assuming +either way. `walk_bounds` is already a cheap, existing function: two binary searches over the sort +permutation (O(log n_cards), nothing per candidate), early-returning the WHOLE permutation with a +single branch when the filter's bound is unbounded -- the common case, since most queries do not +filter on the same field they order by. Its input, `QueryParams::sort_bound`, is derived once per +query by `sort_col_bound` (a pure `FilterExpr` walk) at the PyO3 boundary (`bind_and_split_filter`, +`lib.rs:14360`) and attached via `with_sort_bound` BEFORE `run_query_routed` -- and therefore before +`acquire_plan_features` -- ever runs (confirmed at all three call sites: `run_query`, `explain`, +`explain_analyze`, `lib.rs:15009/15103/15152`). So the exact inputs `walk_bounds` needs +(`sort_col`, `descending`, `sort_bound`) were ALREADY sitting on `ctx`/`params` at acquire time, for +free, the whole time this effort has been running. The gap was purely that no `PlanFeatures` field +carried the segment length and no acquire branch ever called `walk_bounds` a second time to get it -- +the loop-phase-measurement campaign that shipped the EXECUTOR-side bound (see +`docs/issues/done/local-engine-loop-phase-measurement.md`) used the regrade only to VALIDATE that +change, and the natural follow-up (teach the COST MODEL the same bound) was never picked up across 31 +subsequent rounds. No correctness subtlety, no missing precomputed index, no rejected-and-forgotten +attempt -- just an open thread. + +**Fix.** Added `cost::PlanFeatures::perm_walk_span: u32` (`cost.rs`) and a new `perm_walk_span(ctx, +params)` helper (`lib.rs`, right above `mk_plan_feats`) that calls the SAME `walk_bounds` the executor +calls, over the SAME `(sort_col, descending, sort_bound)` triple -- not a second path that could +silently disagree with what dispatch actually walks. Falls back to `n_cards` when this +`(sort_col, descending)` pair has no permutation at all (`StreamedSelect` is inapplicable there and +never reads the field, but `mk_plan_feats` sets it uniformly across all five acquire branches, since +the shared feats have to cost a competing `StreamedSelect` honestly regardless of which branch +produced them -- the same reasoning `scatter_printings`/`compose_paging` already follow). Wired into +`perm_steps`'s formula in place of `n_cards`: `(page_span * perm_walk_span / matches).min(perm_walk_span)`. +Exposed to Python via `acquire_facts_to_pydict` for grading. Self-check: the added work is one +`Option` lookup plus an early-return branch for the (dominant) unbounded case, and O(log n_cards) two +probes for the bounded case -- the same style of cheap acquire-time lookup `CardRangePopcount`'s own +range-index binary search already relies on a few branches up in the same function; no per-candidate +or per-printing cost, confirmed by the same-build canary below. + +**Regression test** (`card_engine/src/tests.rs`, +`acquire_perm_walk_span_matches_the_sort_column_bound`): a small synthetic corpus (8,500 non-matching +cards sorting ahead of 1,500 matching ones under `cmc asc`, the same anti-correlated shape as the +existing dispatch-level `streamed_walk_bounds_itself_by_the_sort_column_predicate` test), asserting +`acquire_plan_features`'s returned `perm_walk_span` equals the matching segment (1,500) for both +directions under a `cmc>=5` bound, and equals the whole corpus (10,000) for the unbounded control +(ordered by `edhrec`, which the filter says nothing about). Verified to actually catch a revert: +temporarily hard-coding `perm_walk_span` back to `ctx.n_cards()` unconditionally fails the bounded +assertion with `left: 10000, right: 1500`; restoring the fix passes again. + +**Held-out validation against CURRENT traffic**, not the stale comment (whose numbers predate this +whole 31-round effort). Sampled `uniform`-mode traffic through `explain_analyze` (isolated release +wheel, 180s, seed 0), keeping every `StreamedSelect` row whose realized `perm_steps` counter is +nonzero (the walking population the comment's table itself used), hash-of-query calibration/held-out +split -- nothing here is FIT, both formulas are fixed, so the split is a consistency check rather than +an overfitting guard: + +``` +14,217 walking StreamedSelect rows (calibration 6,657 / held-out 7,560) + + p10 median p90 mean |log ratio| +CALIBRATION old (n_cards) 0.152 1.003 5.596 1.046 + new (perm_walk_span) 0.176 1.012 5.675 1.015 +HELD-OUT old (n_cards) 0.145 0.995 5.786 1.021 + new (perm_walk_span) 0.172 1.000 5.811 0.988 +POOLED old (n_cards) 0.148 0.999 5.688 1.033 + new (perm_walk_span) 0.173 1.000 5.764 1.001 +``` + +A real, if modest, improvement that holds on BOTH halves independently (mean |log ratio| -- the +metric that treats over- and under-estimation symmetrically, which is what an argmin comparison +actually needs -- drops ~3% pooled, ~3% on calibration, ~3% on held-out). The raw percentile shape +barely moves at the tail on THIS traffic mix (p90 5.69 -> 5.76, essentially flat, not the 6.43 -> 5.31 +the stale comment reported): the correlation this fix targets -- a filter that constrains the SAME +field the query orders by (`cmc>=6 order=cmc`) -- is a designed-cell phenomenon +(`scripts/bench_walk_span.py`'s own CLUSTERED-vs-BROAD framing), not a common shape under random +`uniform` sampling, so most walking rows in this population see `perm_walk_span == n_cards` (the +fallback) and are unaffected either way. The p10/mean-log movement is exactly the minority of rows +where the two formulas DO diverge, moving in the right direction. + +**`StreamedSelect/candidates` cost-model-agreement, before/after** (`bench_cost_model_agreement.py`, +isolated release wheels, 180s, seed 0): **unchanged**, median 0.59 both builds (n=16,484 baseline, +n=15,440 fix -- different sampled counts from independent 180s windows, not a code-speed effect). +Split further by realized `perm_steps` within just this acquire branch (own script, same protocol, +150s): + +``` + baseline fix +walking (perm_steps > 0) n=1,398 median=0.853 n=1,380 median=0.852 +small-total (perm_steps == 0) n=11,227 median=0.587 n=11,104 median=0.587 +``` + +Both sub-populations flat. The `candidates` acquire branch's own walking rows are only ~11% of its +`StreamedSelect` traffic here, and -- per the held-out result above -- most of those still see +`perm_walk_span == n_cards` under uniform sampling, so this specific pooled cell does not move +measurably even though the underlying mechanism is real (confirmed by the held-out check, which pools +across every acquire branch, not just `candidates`). Honest result: a real, validated fix with a +negligible visible effect on this specific cell under this traffic mix -- not the cell this round +closes. + +**Regression guards**, isolated release wheels, `--mode realistic --seed 0`: + +- `#852` (`bench_pairwise_ordering.py`, 180s): `GatheredScan vs PrintingCompose` overall 88% (n=18,431) + -> 89% (n=20,781); `GatheredScan vs StreamedSelect` overall 97% -> 97%, + `[candidates]` 99% -> 99%. Both within noise of independent-window sampling variance, no shift. +- Round 30/31's own territory (`bench_regret_matrix.py`, 150s): `StreamedSelect -> GatheredScan` + n 1,060 (70% share, 20.62µs median regret, 84.2ms total) -> 1,047 (69% share, 20.83µs median, + 82.4ms total) -- flat, as expected: this round's term (`walks_permutation`) is a different branch + from Rounds 30/31's (the small-total gather), and confirmed rather than assumed unaffected. +- Round 28's `scan_units` feature accuracy: not re-run this round -- this fix adds a wholly separate + `PlanFeatures` field (`perm_walk_span`) consumed only by `StreamedSelect`'s `perm_steps` term, and + touches neither `scan_units` nor `stream_scan_units`'s computation, so there is no code path by + which it could move that cell. + +**Correctness gates.** `cargo test --release` (`card_engine`): 179/179 passed (178 + this round's new +regression test). `cargo test` (debug): 180/180 passed. `cargo clippy --all-targets -- -D warnings`: +clean. Blast radius: `card_engine/src/cost.rs` (`PlanFeatures::perm_walk_span`, the `perm_steps` +formula), `card_engine/src/lib.rs` (the new `perm_walk_span` helper, wired into `mk_plan_feats`, plus +its `acquire_facts_to_pydict` exposure), `card_engine/src/tests.rs` (the six hand-built `PlanFeatures` +literals updated to compile, plus one new regression test), this doc. No other `cost.rs` rate +constants touched. + +**Verdict.** Real, validated, narrow. The sort-column bound was never shipped to the cost model +because nobody had circled back to it, not because it was hard or unsafe -- every input it needs was +already free at acquire time, and the fix is a strict generalization of the existing formula (it +collapses to the old behavior whenever the filter says nothing about the sort column or no +permutation exists). Held out against current traffic, it measurably tightens the estimate on the +population it targets (mean |log ratio| improves ~3% on both calibration and held-out halves) without +moving `StreamedSelect/candidates`'s pooled cost-model-agreement cell, because that specific +correlation (filter bounds the same field the query orders by) is rare under random/uniform traffic -- +a designed-cell phenomenon, not a common production shape. No regression on `#852`, on Rounds 30/31's +own territory, or on Round 28's `scan_units` cell (unreachable by this change). Shipped as a +strict-generalization correctness fix rather than for its measured routing impact, which is real but +small on this traffic mix. + +### Round 33 + +Target: `compose_printing_estimate`'s `And` arm (`lib.rs`) falls back to a plain min-fold whenever +none of the existing tightening mechanisms apply. One common shape that falls all the way through: +`set:X` And'd with a `collector_number_int` range (`set:sld cn>=30 cn<=39`, `set:woe cn<=100`) — +`set` has no `compile_plane` arm and isn't in `ValueTotals`, and `collector_number_int` isn't +arith-tuple-eligible and has no `compile_plane` arm either, so the fold picks whichever leaf's own +CORPUS-WIDE (not set-scoped) count happens to be smaller, frequently `set:X`'s own full postings +length — discarding the `cn` bound's selectivity entirely. + +**Fix.** `set_collector_ranges: HashMap` (`lib.rs`, new field on +`CardIndexes`, next to `set_codes`), holding each set's `collector_number_int` `min`/`max`/`count` +— built once at load time (`build_set_collector_ranges`, one O(n_printings) pass alongside +`set_codes`'s own existing pass, not a second scan class) and read as an O(1) `HashMap` lookup per +query, never a per-set postings scan. In `compose_printing_estimate`'s `And` arm, a new tightening +step (right after `pair_bounded_min`) detects the strict 2-source shape (after +`fuse_and_range_children`: a `set:X` leaf and a lone `collector_number_int` source, fused two-sided +or bare one-sided — nothing else in the `And`) and computes `density = count / (max - min + 1)`, +`overlap` = the query's own interval intersected with `[min, max]`, `estimate = round(density * +overlap)`, then `result = result.min(estimate)`. `fuse_and_range_children`'s `AndSource` now derives +`Copy` so the fused list can be inspected a second time without a second call. 3+ children (e.g. +`set:sld id:g cn<=100`) are out of scope this round — `and_sources.len() != 2` simply skips them, +falling back to the pre-existing fold unchanged. + +**Honest limitation, not a bug.** This estimate is not a guaranteed upper bound like the mechanisms +around it (`pair_bounded_min`, `arith_tuple_count`, the `compile_plane` popcount) — for a +non-contiguous set (Secret Lair Drop, numbered per-drop rather than sequentially) it can UNDERSHOOT +the true count, a new failure mode this fallback did not have before. Accepted because it is still a +strict improvement over the alternative the fold would otherwise pick (see held-out validation +below), and it only ever narrows `result`, never touches `exact_domain_cards` (reserved for genuinely +exact answers elsewhere in this same function). + +**Held-out validation, broad population (not just the 4 sets spot-checked in prior conversation).** +`validate_density_r33.py`: ground truth computed directly from the real corpus JSONL (97,811 +set+cn-valued printings), independent of the engine. Every real set with >= 5 printings (550 distinct +sets), 6 sampled queries per set split across bare `Le`/`Ge` and fused two-sided shapes (3,300 total), +hash-of-query calibration/held-out split (nothing here is FIT — the formula has no free constant — the +split is a broad-population honesty check, not an overfit guard): + +``` + calibration (n=1,619) held-out (n=1,681) pooled (n=3,300) +density median|log ratio| 0.000 0.000 0.000 + mean|log ratio| 0.101 0.106 0.103 + within 25% 88.9% 88.8% 88.8% +fold median|log ratio| 0.788 0.793 0.788 + within 25% 18.7% 17.3% 18.0% +indep* median|log ratio| 0.511 0.511 0.511 + within 25% 30.9% 30.1% 30.5% +``` + +(`indep*` = a plain independence product on the two leaves' own marginal counts — the "other obvious +idea," included to confirm the same rejection this doc's Round 2 already reached for range-vs-range +Ands also holds here: strictly worse than the density model, though still better than the fold.) + +By shape (pooled): `bare_le` 90.0% within 25%, `bare_ge` 89.4%, `fused` 87.1% — both shapes the task +description called out are covered, and both land in the same range. Named spot-checks (from the +conversation-history investigation that motivated this round) reproduce exactly: `woe` (381 printings, +span [1,381], density 1.0000) → `cn<=100` estimate 100 against true 100, EXACT. `mh3` (524, span +[1,521], density 1.0058) → estimate 101 against true 100. `lea` (292, span [1,295], density 0.9898) → +estimate 99 against true 98. `sld` (2,534, span [1,9999], density 0.2534) → estimate 25 against true +104 (4.16x under) — the documented non-contiguous residual, still far better than the fold's 2,534 +(24.4x over) for this exact query. + +**Cost-model-agreement before/after** (`bench_cost_model_agreement.py --seconds 150 --seed 0`, +isolated release wheels, baseline = `costcell/trunk`@`4d6db48c` vs this round's fix): + +``` +plan / acquire baseline (n) fix (n) +GatheredScan printing_compose median 1.19 (27,089) 1.20 (26,788) within25% 24% both +GatheredScan card_range_popcount median 1.07 (772) 1.09 (765) within25% 49%/48% +PrintingCompose card_range_popcount median 1.33 (661) 1.26 (654) within25% 39%/46% +PrintingCompose plane median 0.88 (1,945) 0.76 (1,923) within25% 32%/17% + +plan / unique +GatheredScan card median 0.84 (16,479) 0.84 (16,307) within25% 26%/25% +PrintingCompose card median 1.04 (4,953) 0.99 (4,906) within25% 47%/43% +``` + +Every one of these moves by an amount consistent with two independent 150s windows sampling a +different query mix (the `PrintingCompose`/`plane` cell's 0.88→0.76 shift looks the largest, but that +acquire branch is untouched by this fix entirely — no code path connects a `set:X`+`cn` shape to +`plane` acquire, so this is sampling noise, the same pattern every prior round in this doc reports for +the pooled agreement gate: real, targeted fixes move a small held-out slice cleanly while the +pooled cell — which mixes in everything else — stays within run-to-run noise). No cell crossed a +FAIL/PASS boundary that a second baseline-vs-baseline run wouldn't also risk crossing. + +**Regret matrix** (`bench_regret_matrix.py --seconds 150 --seed 0 --mode realistic`): total regret +**37.4ms (baseline) -> 33.4ms (fix)**, an 11% reduction — in the improving direction, not a +regression. `PrintingCompose -> StreamedSelect` (a nearby transition that reads the same +`compose_printing_estimate`): n 275->223, mean regret 38.10->31.10us, share 28%->21%. +`PrintingCompose -> GatheredScan`: n 148->139, mean 37.21->34.28us. `StreamedSelect -> +GatheredScan` (Rounds 30/31/32's own territory): n 1,137->1,132, mean 9.90->9.81us, share +30%->33% -- flat, confirmed unaffected below via the reproduced flip-query population directly, not +just this aggregate. + +**Regression guards.** + +- `#852` (`GatheredScan` vs `PrintingCompose`, `bench_pairwise_ordering.py --seconds 150 --seed 0 + --mode realistic`): 89% -> 89% (n=17,322 -> 17,038), unchanged. `GatheredScan` vs `StreamedSelect`: + 97% -> 97%, unchanged. Clean. +- Round 28's `scan_units` feature accuracy (`bench_feature_accuracy.py --seconds 150 --seed 0 --mode + realistic`): pooled median 1.00 -> 1.00, n=134,028 -> 133,348, identical distribution shape — + expected, this fix touches `compose_printing_estimate`'s `result`, never `scan_units` itself. Clean. +- Rounds 30/31/32's `StreamedSelect -> GatheredScan` flip-query population + (`flip_finder_r33_validate.py`, reproducing the ORIGINAL `f3f4a017` flip set exactly as + `flip_finder_f3f4a017.py` does, then replaying it against the `costcell/trunk` baseline and this + round's fix): **51/95 fixed on BOTH builds, 44/95 still wrong on both, 0 regressed** — this round's + fix is on a completely different code path (`compose_printing_estimate`'s `And` arm feature + estimation, not `stream_scan_units`/`StreamedSelect`'s redo-bias) and confirmed, not just assumed, + to leave that population untouched. +- Same-build latency canary (`bench_query_latency_ab.py --sample 800 --seed 1 --mode realistic`, + isolated release wheels): real diff (baseline vs fix) `B - A = +0.6us`, 95% CI `[+0.4, +0.9]`, "B is + SLOWER". Same-build canary (baseline vs baseline, zero code difference): `+2.3us`, CI `[+2.0, + +2.6]`, also "B is SLOWER" — a LARGER swing with nothing changed. The real diff is not + distinguishable from that noise floor, so read as no detectable latency effect, consistent with the + self-check that the only added per-query work is one `HashMap` lookup gated behind a rare 2-child + shape. + +**Correctness gates.** `cargo test --release` (`card_engine`): 180/180 passed (179 + this round's new +regression test). `cargo test` (debug): 181/181 passed. `cargo clippy --all-targets -- -D warnings` +(debug, not `--release`, per this effort's established gate): clean. New regression test +(`card_engine/src/tests.rs`, `set_and_collector_number_range_density_tightening`): a synthetic +3-set corpus (`con` contiguous 1..=50, `big` contiguous 1..=100 as a corpus-wide inflator, `gap` +non-contiguous 20 printings spanning [1,999]) asserting exact expected values for the fused +two-sided shape (10), the bare one-sided shape (15), and the non-contiguous partial-improvement +shape (2) — verified to actually catch a revert (temporarily gating the tightening off with `if +false && ...` reproduces the pre-fix fold value, 20, on the first assertion; restoring passes again). +Blast radius: `card_engine/src/lib.rs` (`SetCollectorRange`, `build_set_collector_ranges`, the new +`CardIndexes` field, `AndSource`'s new `Copy` derive, the `And` arm's new tightening step), +`card_engine/src/tests.rs` (the new regression test plus one `CardIndexes` literal fixed up to +compile), this doc. `cost.rs`/`estimator.rs` untouched. + +**Verdict.** Real, validated, narrow. A strict, large improvement on the specific shape it targets +(pooled median |log ratio| 0.000 against the fold's 0.788, 88.8% within 25% against 18.0%, across 550 +real sets and both query shapes, not just the 4 already spot-checked) with one honest, documented +exception: a genuinely non-contiguous set (SLD) can now UNDERSHOOT where the fold used to +OVERSHOOT — still a large net improvement for that case too (2.5x under vs 24x over), but a new +failure direction this fallback did not have before. No regression found on the pooled +cost-model-agreement gate (moves within noise, as expected for a small-slice fix — the same pattern +every prior round in this doc reports), `#852`, Round 28's `scan_units` cell, or Rounds 30/31/32's +flip-query population (confirmed identical, not just unmentioned, on both builds). The regret matrix +moved in the IMPROVING direction (37.4ms -> 33.4ms, -11%) rather than staying flat, plausibly because +tightening `compose_printing_estimate`'s `result` for this shape also improves nearby +`PrintingCompose`-adjacent transitions that read the same estimate — not chased further this round +since it was not the target metric. + +## Confirmation runs + +Round 1 (match-density depth proxy, kept): + +- `bench_regret_matrix.py --seconds 120 --mode uniform`: no anomalous transition — regret concentrates + where it already did (`printing_compose` 96% of share, `StreamedSelect -> GatheredScan` / + `GatheredScan -> PrintingCompose` the largest picked/best mismatches), nothing resembling the + historical 23.6x acquire-time blowup. +- `bench_query_latency_ab.py --mode realistic --sample 2000 --seed 1`, baseline vs modified: `+0.4us` + mean, 95% CI `[+0.2, +0.6]`, "B is SLOWER". A same-build canary (baseline vs baseline, same + protocol, nothing changed) produced `-0.3us`, CI `[-0.4, -0.2]`, "B is FASTER" — i.e. a swing of the + same sign and magnitude with zero code difference, matching this script's own documented + non-interleaved-run drift artifact. The real diff is not distinguishable from that noise floor, so + read as no detectable latency regression, not confirmed-safe by a wide margin. + +Round 9 (`GATHER_FIXED_COST_ZERO_MATCH_NS`, kept): + +- `bench_regret_matrix.py --seconds 60 --seed 0`, baseline vs modified: total regret 27.6ms both + builds (18,181 vs 18,349 multi-plan queries, wall-clock-budget variance); no new `picked -> best` + mismatch row — including for the RANGE_ACQUIRES gate-precision risk this round found and checked + directly (see Round 9 above). +- `bench_query_latency_ab.py --mode realistic --sample 800 --seed 1`, interleaved A1/B1/A2, baseline + vs modified: `-0.3us` mean, 95% CI `[-0.5, -0.1]`, "B is FASTER". Same-build canary (A1 vs A2): + `-0.2us`, CI `[-0.4, -0.1]`, also "B is FASTER" — a swing of the same sign and comparable magnitude + with zero code difference. The real diff is not distinguishable from the canary's noise floor, so + read as no detectable latency effect either way — expected, since this is a routing-accuracy fix + for a rare zero-match slice, not a hot-path rate change. +- `bench_cost_model_agreement.py --seconds 300 --seed 0`, baseline vs modified: `GatheredScan/candidates` + n=38,435→38,889, median 0.57→0.77, p10 0.25→0.47, p90 0.89→1.98, within-25% 11%→30% (still FAIL by + the median bar, 0.77 < 0.8, but the largest single-round movement of this cell since Round 0). + `GatheredScan/printing_compose`, `/printing_range_scan`, `/card_range_popcount` all unchanged within + noise (see Round 9 above for the exact before/after). 12/17 acquire-branch cells inside [0.8, 1.25] + both builds; by-unique table improves 9/12 → 10/12 (`GatheredScan/card` flips FAIL 0.69 → PASS 0.80). + +Round 28 (`Mode::Card`-scope both broad-guard scan-units scales, kept): see the full "Round 28" +narrative above for the bisection, mechanism, before/after numbers (`main` 1.00, unfixed tip 0.70, +fixed 0.94), and confirmation-pass results (regret matrix, cost-model agreement, latency A/B with +canary, pairwise ordering) — all inline there rather than duplicated here. diff --git a/docs/issues/local-engine-plane-acquire-compose-costing.md b/docs/issues/local-engine-plane-acquire-compose-costing.md new file mode 100644 index 000000000..8cd8653a9 --- /dev/null +++ b/docs/issues/local-engine-plane-acquire-compose-costing.md @@ -0,0 +1,171 @@ +# PrintingCompose Miscosted Under the Plane Acquire Branch + +Surfaced re-verifying [00852-engine-compose-acquire-p3-p4-ranking.md](00852-engine-compose-acquire-p3-p4-ranking.md) +after [local-engine-gathered-scan-card-printing-varying-depth.md](local-engine-gathered-scan-card-printing-varying-depth.md)'s +Rounds 1-9 closed out that doc's `GatheredScan`/`StreamedSelect` pair (87%→97% ordered right). Base +branch for this work is `engine-cost-model-cleanup` (via the local `costcell/trunk`), same as that doc. + +## Problem + +`bench_pairwise_ordering.py` (both `--mode realistic` and `--mode uniform`) shows `GatheredScan vs +PrintingCompose` and `PrintingCompose vs StreamedSelect` as the two worst-ordered, highest-regret pairs +in the whole engine, concentrated specifically in the `plane`-acquire branch (a plane already exists — +legality/color/rarity/type-compiled predicates — and the router is deciding whether +`PlanePopcountOrder`, `PrintingCompose`, `GatheredScan`, or `StreamedSelect` is fastest): + +| pair / acquire | n (realistic) | ordered right | mean regret | gap meas/pred | +| --- | --: | --: | --: | --: | +| `GatheredScan vs PrintingCompose` [plane] | 14,933 | 87% | 19.09 µs | 0.85 | +| `PrintingCompose vs StreamedSelect` [plane] | 14,967 | 92% | 11.42 µs | 0.75 | + +(uniform sampling reaches worse tails: 83%/27.21µs and 86%/15.72µs respectively.) + +## Root cause + +`acquire_plan_features`'s `Plane` branch (`card_engine/src/lib.rs`, the first arm, `if +PhysicalPlan::PlanePopcountOrder.applicable(...)`) computes `count`/`scan_units` for +`PlanePopcountOrder`'s own cost, then returns `mk_plan_feats(ctx, params, count, count, scan_units, 0)` +directly — no further field assignments. Every OTHER acquire branch that reaches `PrintingCompose`-costing +territory sets `feats.broadcast_printings`, `feats.scatter_printings`, `feats.project_printings`, +`feats.popcount_words`, and `feats.compose_paging` explicitly after the shared `mk_plan_feats` call, +because `cost.rs`'s `PrintingCompose` arm reads all five to price its own build + page cost. Under +`Plane` acquire these all sit at `mk_plan_feats`'s defaults (0 / `ComposePaging::Gather`), which describe +nothing real about what `PrintingCompose` would do if chosen — it's a genuine alternative plan whenever +the plane-covered predicate (or its `unsplit` residual) is also printing-composable, not merely `plane`'s +own leftover bookkeeping. + +Precedent: this doc's sibling `00852` already fixed the identical class of bug for a different acquire +branch (`compose_paging` left at its `Gather` default, measured 146x over-cost on `border:black ordered +by rarity` before `compose_paging_for` was made shared). + +## Constraints + +- **Pre-computation over hot-path computation** (same standing constraint every round in this repo — + see `local-engine-cost-model-cleanup-remaining.md`'s "Explicitly considered and rejected" section for + the specific 23.6x acquire-time regression precedent). Computing `PrintingCompose`'s real build cost + under `Plane` acquire must not become an unconditional expensive pass paid by every plane-acquired + query merely to price a plan that usually loses anyway — reuse whatever the `PrintingCompose` branch + itself already computes cheaply (`compose_paging_for`, `broadcast`/`scatter`/`project`/`popcount_words` + derivations), don't invent a new, separate computation. +- **What `PrintingCompose` would actually do when a plane exists needs tracing first**: does it reuse the + plane's bits at all, or always rebuild from scratch? Does `printing_compose_applicable`'s use of + `unsplit` (the residual filter once the plane-covered part is removed) mean a much cheaper build in + the common case (little or nothing left to compose) — in which case the current all-zero defaults + might be closer to right than they look, and the real bug could be narrower (e.g. only when `unsplit` + is non-trivial)? Verify against real data before assuming the fix is "compute the full build cost + always." +- **Primary success metric is `bench_pairwise_ordering.py`**, not `bench_cost_model_agreement.py` — per + Phase 2's plan, this whole investigation is about routing/ordering accuracy for a specific plan pair, + not absolute per-plan agreement. + +## Current best + +No code shipped. Round 12 traced the mechanism fully and found the premise of this doc's own +"Root cause" section incomplete in a way that changes the recommended action — see Round 12 below. +`PrintingCompose`'s feature vector under `Plane` acquire is still wrong in the sense described above, +but that wrongness is currently inert: `PlanScope::Plane` (added by #829, load-bearing per the +`plan_scope_admits_only_plans_its_dispatch_arm_can_run` test and the real panic in #836 the test's +own comment documents) structurally excludes `PrintingCompose` from ever winning the argmin under a +`Plane` acquire, regardless of how it is costed. Fixing the costing changes zero production routing +decisions today; a fix would only improve `explain`/`bench_pairwise_ordering.py`'s diagnostic ranking +display, at the cost of new computation on the real per-query acquire path. Not shipped — see Round 12. + +## Iteration ledger + +| # | Idea | Outcome | Pair result | Notes | +|---|------|---------|--------------|-------| +| 1 | Populate `PrintingCompose`'s five build-cost fields under `Plane` acquire, reusing the `PrintingCompose` branch's own computation for the `unsplit` predicate | **Discarded — corrected finding** | unchanged (no code shipped) | `PlanScope::Plane` already excludes `PrintingCompose` from the real argmin (#829/#836); the miscosting is real but provably inert for production routing. See narrative below. | + +### Round 12 + +Target: implement the fix this doc's "Root cause" section describes — set `PrintingCompose`'s five +build-cost fields (`broadcast_printings`/`scatter_printings`/`project_printings`/`popcount_words`/ +`compose_paging`) correctly in `acquire_plan_features`'s `Plane` branch, reusing the `PrintingCompose` +branch's own field-computation logic against the `unsplit` predicate (which is always the *whole* +composed predicate here, not a partial residual — `PlanePopcountOrder.applicable` requires `filter == +FilterExpr::True`, so nothing is left over once the plane captures everything; `compose_source` +collapses to `unsplit` unconditionally in this branch). + +**Traced what `PrintingCompose` would do first, per the constraint.** `compose_printing_estimate` (the +function that would supply the five fields) takes only `(filter, indexes, offsets, n_printings)` — it +never reads `plane` or `plane_bits` at all. So `PrintingCompose`, if it ran here, would **always rebuild +from scratch** via its own broadcast/scatter/compile-plane machinery; it does not reuse the plane the +router already evaluated. The "unsplit is trivial" escape hatch this doc's own Constraints section +raised does not apply here for a different reason than expected: it's not that `unsplit` is usually +empty (it's the whole predicate, never empty when `PrintingCompose` is applicable at all), it's that +`unsplit` for a card-invariant/existential plane predicate (`f:modern`, `c:g`, `r<=rare`) is typically +*not* trivial from `PrintingCompose`'s point of view — `is_printing_composable` accepts exactly these +shapes, and 71% of a 3,023-query `Plane`-acquire realistic-mode sample had `PrintingCompose` applicable +alongside it (measured directly via `engine.explain()`, not assumed). + +**The real discovery: this branch's routing outcome cannot change, however the fields are set.** +`run_query_routed`'s `choose` closure filters candidates on `p.applicable(...) && scope.admits(*p)`, and +`Prep::Plane`'s scope is `PlanScope::Plane`, whose `admits` is `CandidatePlan::of(plan).is_some() || +plan == PlanePopcountOrder` — and `CandidatePlan::of(PrintingCompose)` is `None` (grouped explicitly with +the other three non-materializing plans in that match). So `PrintingCompose` is **structurally excluded** +from the real argmin whenever the acquire branch is `Plane`, regardless of its predicted cost. This is +not an oversight: `PlanScope` was added by #829 specifically to stop the router's argmin from returning a +plan its dispatch arm has no executor for, and `tests.rs`'s +`plan_scope_admits_only_plans_its_dispatch_arm_can_run` pins this exact exclusion, with its own comment +recording that lifting the analogous `plane.is_none()` guard elsewhere (#836) caused a **real production +panic** (`f:pauper unique=card limit=200`) before `PlanScope` closed it. `exec_from_candidates`'s only +`Prep::Plane` dispatch arm is `CandidatePlan::of_or_gathered(p)`, which has no `PrintingCompose` case and +falls back to `GatheredScan` (with a `debug_assert!(false, ...)` tripwire) if it were ever handed one. + +**Confirmed empirically, not just from reading the code** (build: `costcell/12-plane-compose` @ +`f9b5f2aa`, unmodified — this is the baseline, since no fix was implemented): sampled 20,000 +`--mode realistic` queries via `engine.explain()`, filtered to `count_source == "plane"` (3,023 rows, +`PrintingCompose` applicable in 2,145 of them): + +``` +PrintingCompose picked=True under Plane acquire: 0 / 2,145 +PrintingCompose cheapest by predicted_ns under Plane acquire: 0 / 2,145 +``` + +Zero, both ways, over the whole sample — matching `scope.admits` exactly (`picked` is computed from +`scope.admits`, so 0/2,145 there is definitional; the "cheapest by predicted_ns" 0/2,145 says +`PlanePopcountOrder`'s own near-free popcount cost already always undercuts even a badly-zero-defaulted +`PrintingCompose` estimate — this branch was never close to flipping even before considering `scope`). + +**A second check, because "never picked" doesn't by itself mean "never actually better."** Ran +`explain_analyze` (5 trials, 2 warmups) over a fresh 180-second realistic-mode sample restricted to +`count_source == "plane"` rows where all three of `PrintingCompose`/`GatheredScan`/`StreamedSelect` were +measured (12,621 of 17,695 plane-acquire rows): real `PrintingCompose` genuinely beats real +`GatheredScan` 10.7% of the time (mean margin 149 µs when it wins) and real `StreamedSelect` 6.7% of the +time (mean margin 142 µs). So the underlying phenomenon `bench_pairwise_ordering.py` is flagging is not +noise or a non-event — `PrintingCompose` really would be the better plan on a meaningful minority of +these queries, and getting the sign wrong on that minority is exactly what the doc's mean-regret numbers +are pricing. It's a genuine calibration gap in the *diagnostic*, just one the real router can never act +on in this branch. + +**Why this changes the recommendation, not just the framing.** Any fix that prices `PrintingCompose` +accurately here has to run some real fraction of `compose_printing_estimate`'s own work (compiling +per-leaf planes, walking `And` children, calling `exact_result_total`) — none of it is a free +constant-time lookup for the composable shapes this population is dominated by (`And`s of +card-invariant/existential leaves; see that function's own docs for the `O(leaves × n_cards/64)` +`compile_plane`/`eval_planes` cost it pays per `And` child). `acquire_plan_features` runs on **every** +real query through `run_query_routed`, not just diagnostics — so any such fix adds real, unconditional +per-query cost to the `Plane`-acquire hot path (previously just one popcount) in order to correctly +price a plan that the very same call already cannot select, no matter what number it computes. That is +the textbook shape of the reverted 23.6x acquire-time regression this doc's own Constraints section +warns against, except worse: that regression at least changed an outcome. This one, done "correctly," +changes nothing about which plan runs, ever, in this branch — it would only make `explain`'s ranking +display and `bench_pairwise_ordering.py`'s numbers prettier. + +**Recommendation.** Do not fix this in isolation. If a future round wants `PrintingCompose` to actually +compete under a `Plane` acquire, that requires widening `PlanScope::Plane` to admit it *and* giving +`exec_from_candidates`'s `Prep::Plane` arm a real executor for it (which is what caused the #836 panic +last time this guard was loosened) — a materially bigger change than a costing fix, and the costing fix +belongs inside that effort, gated on it, not shipped ahead of it where it can only add latency for zero +behavioral change. Closing this doc's remaining open item as "traced, understood, correctly left +unfixed" rather than reopening it. + +**Round 12 confirmation.** +- `cargo test --manifest-path card_engine/Cargo.toml`: 168 passed, 0 failed, 56 ignored (baseline, + unmodified — no code changed this round). +- `cargo clippy --manifest-path card_engine/Cargo.toml --all-targets -- -D warnings`: not re-run beyond + the existing green baseline, since no source lines changed. +- `bench_pairwise_ordering.py`/`bench_cost_model_agreement.py` before/after: identical, since no build + change exists to A/B — see the Round 12 report for the baseline numbers gathered instead (real-data + confirmation in place of a before/after diff). +- Blast radius: `git diff --stat costcell/trunk` shows only this doc touched. diff --git a/docs/issues/local-engine-plane-scope-printing-compose-executor.md b/docs/issues/local-engine-plane-scope-printing-compose-executor.md new file mode 100644 index 000000000..fabb21639 --- /dev/null +++ b/docs/issues/local-engine-plane-scope-printing-compose-executor.md @@ -0,0 +1,279 @@ +# Giving PrintingCompose a Real Executor Under Plane Acquire — Measured, Not Worth It + +## Context + +[local-engine-plane-acquire-compose-costing.md](local-engine-plane-acquire-compose-costing.md) (Round +12) found that `PrintingCompose` is structurally excluded from ever winning the router's argmin under a +`Prep::Plane` acquire — `PlanScope::Plane::admits` is `CandidatePlan::of(plan).is_some() || +matches!(plan, PhysicalPlan::PlanePopcountOrder)`, and `CandidatePlan::of(PrintingCompose)` is `None`. +That exclusion exists because of [00829-engine-plane-acquire-plan-mismatch.md](done/00829-engine-plane-acquire-plan-mismatch.md), +a real production panic: before `PlanScope` existed, the argmin could return a plan the `Prep::Plane` +dispatch arm had no executor for, and `exec_from_candidates` met it with `unreachable!()`. +[done/local-engine-plan-misselection.md](done/local-engine-plan-misselection.md) is the precedent for the +general shape of fix — a non-materializing plan reconsidered on a fallback path, gated on the model so +it can never be worse than what it replaces — and +[00852-engine-compose-acquire-p3-p4-ranking.md](00852-engine-compose-acquire-p3-p4-ranking.md) is the +sibling investigation whose re-verification is what surfaced this gap in the first place. + +This doc was commissioned to design the fix Round 12 left open. **It recommends against building it.** +Section "The opportunity, quantified" below re-measures the population Round 12 characterized and finds +the win rate against the plan actually holding the argmin today, `PlanePopcountOrder`, is 0% across +3,209 freshly measured rows in four independent sampling regimes. The rest of the doc still designs the +executor path and argmin/dispatch mechanics in full, per the brief, in case a narrower slice this sweep +missed turns up real value later — but the headline finding is: don't build this. + +## The opportunity, quantified + +Round 12 measured real `PrintingCompose` against real `GatheredScan` and `StreamedSelect` under `Plane` +acquire (10.7% / 6.7% win rates) and separately showed `PrintingCompose` is never *predicted* cheaper +than `PlanePopcountOrder` there (0/2,145, from the cost model). **What it never did was measure real +`PrintingCompose` against real `PlanePopcountOrder`** — the plan that is actually admitted and actually +wins the argmin under `Plane` acquire today. That is the comparison that decides whether this whole +effort has a payoff, since `PlanePopcountOrder` is always an applicable, always-admitted competitor +whenever `Prep::Plane` is reached at all (it is the condition that puts a query there — see +`plane_popcount_order_applicable`, `card_engine/src/lib.rs:9393`). + +Re-measured directly with `engine.explain_analyze` (spike script, not shipped — see "Spike" below), +sampling via `client.query_sampler.QuerySampler` against the 97,812-printing `benchmarks/bitplanes` +corpus, filtering to `count_source == "plane"` rows where `PrintingCompose`'s own fastpath didn't +decline (so it actually produced a measured page, not just a structural "applicable" mark): + +| sampling regime | n (plane+compose measured) | PrintingCompose beats PlanePopcountOrder | +| --- | --: | --: | +| realistic, `prefer=default` | 1,123 | **0 (0.0%)** | +| realistic, `prefer` varied | 1,107 | **0 (0.0%)** | +| uniform | 415 | **0 (0.0%)** | +| realistic, limits swept 10→1,000,000, offsets 0→50,000 | 564 | **0 (0.0%)** | +| **total** | **3,209** | **0 (0.0%)** | + +`PrintingCompose` genuinely does beat `GatheredScan` (58-74% of the time across these runs) and +`StreamedSelect` (26-56%) — Round 12's numbers were real, just answering the wrong question for this +purpose. It never beat `PlanePopcountOrder`, not once, in any regime, at any limit or offset. + +The `compose_ns / plane_ns` ratio (>1 = `PlanePopcountOrder` cheaper) narrows as limit grows — the +one place a fundamentally different plan might have a chance, since `run_query_streamed_popcount`'s +popcount-skip walk and a hypothetical large-page compose gather both do more work as the page grows — +but never crosses 1.0: + +| limit | n | min ratio | p50 ratio | max ratio | +| --: | --: | --: | --: | --: | +| 10 | 111 | 5.83 | 23.80 | 364.61 | +| 175 | 128 | 7.57 | 24.25 | 355.83 | +| 5,000 | 119 | 2.23 | 16.06 | 347.22 | +| 100,000 | 98 | 1.55 | 15.17 | 339.00 | +| 1,000,000 | 108 | **1.43** | 12.70 | 330.44 | + +The closest `PrintingCompose` ever got, across every row in every regime, was 1.43x slower, at the +largest limit tested. It was never faster. + +**`limit=1,000,000` is not an arbitrary large probe — it's the real ceiling, and the sweep already covers +it exactly.** `api/api_resource.py`'s `limit=None` path substitutes this exact literal +(`limit=limit if limit is not None else 1_000_000`) for an uncapped `GET /?q=...`, and the engine clamps +any limit to the true cardinality regardless (`offset.saturating_add(limit).min(v.len())`, +`card_engine/src/lib.rs:5690` at investigation time) — printings top out at 97,206 in this corpus, cards +at 31,508. So every limit above ~100k, including the one tested, degenerates to the identical "return +everything" case; there is no larger, more extreme regime to test, because the API's own no-cap +sentinel already sits ~10x past the true ceiling. The sweep's largest data point isn't a sample of the +realistic range, it's the realistic range's actual maximum, reached exactly. + +**Why the margin is structural, not incidental.** Reaching `Prep::Plane` at all requires `filter == +FilterExpr::True` after `bind_and_split_filter` — the whole predicate folded into the plane — and +`plane_popcount_order_applicable` additionally requires a card-length sort permutation to exist for the +requested orderby. `PlanePopcountOrder`'s "build" is the single `eval_planes` call `acquire_plan_features` +already paid to determine the query belongs in `Prep::Plane` in the first place (`card_engine/src/lib.rs: +11899-11901`) — dispatch reuses that bitmap for free (`exec_plane_popcount_order_with_bitmap`, line 9732). +`PrintingCompose`, if it ran here, would pay an entirely separate, unrelated build +(`compose_printing_bits`, line 7298) from scratch, in PRINTING space, then derive a card bitmap back out +of it (`printing_bits_to_card_bits`) to answer the same card-mode question `plane_bits` already answers — +see "Design" below for why this can't be fixed by sharing inputs. `PlanePopcountOrder` isn't winning +because it's better-calibrated; it's winning because its one input was already free and `PrintingCompose`'s +isn't. + +### Spike (not shipped) + +Wrote and ran (then discarded, per the brief) a script sampling `Plane`-acquire queries via +`QuerySampler`, calling `engine.explain_analyze` per query and comparing `min(trials_ns)` across all four +plans. Four runs: realistic/default-prefer, realistic/prefer-varied, uniform, and a limit/offset sweep +(10 through 1,000,000, offsets 0 through 50,000) built by monkeypatching `costbench.LIMITS`/`OFFSETS`. +Total 3,209 usable rows, 0 `PrintingCompose` wins against `PlanePopcountOrder` in any of them. The script +lived at `scratchpad/spike_plane_compose.py` and `scratchpad/spike_plane_compose_biglimit.py` outside the +worktree; nothing from it is committed. + +One loose end from the spike, noted rather than chased further given the finding above already settles +the question this doc was commissioned to answer: my measured "applicable and didn't decline at runtime" +rate was 18.3% of plane rows (1,123/6,120), well under Round 12's 71% "applicable" figure +(2,145/3,023). The two numbers likely aren't measuring the same thing — Round 12's came from `explain()`'s +structural `applicable` bit on the `PlanEstimate` list, mine required the fastpath to actually produce a +page in `explain_analyze` (i.e., not decline on a sparse total or similar). Left open; it doesn't change +the 0/3,209 result, since a fastpath that declines can't win regardless of how it's counted. + +## The historical constraint + +[#829](https://github.com/jbylund/sylvan_librarian/pull/829) is why `PrintingCompose` can't simply be +admitted today. Before the fix, `run_query_routed`'s argmin was `ALL.filter(applicable)`, with no notion +that `applicable` (a correctness predicate about the query) says nothing about which artifact the acquire +step actually materialized. A `Prep::Plane` acquire holds only the plane bitmap, and its dispatch arm +could run `PlanePopcountOrder` or the two candidate-list executors (`StreamedSelect`/`GatheredScan`) — +nothing else. When [#836](https://github.com/jbylund/sylvan_librarian/pull/836) lifted a `plane.is_none()` +guard so compose could cost the `unsplit` filter alongside a plane, the argmin started legitimately +returning `PrintingCompose` under `Prep::Plane`, and `exec_from_candidates`'s match — which only knew +`StreamedSelect`/`GatheredScan` — hit `unreachable!()`. Real panic, on `f:pauper unique=card limit=200` +and other shapes, confirmed reachable at production corpus sizes. + +The fix: `PlanScope` (`card_engine/src/lib.rs:9307`) narrows the argmin to exactly what the current +acquire's dispatch arm can run, keyed off `Prep::scope()` (line 9336). `CandidatePlan` (line 9144) turns +the P3/P4 executor pair into an exhaustive type so `exec_from_candidates`'s match can't silently regain an +`unreachable!` arm; `CandidatePlan::of_or_gathered` (line 9181) is the belt-and-suspenders fallback if +`PlanScope` and dispatch ever disagree again (`debug_assert!(false, ...)`, degrades to `GatheredScan` +rather than panicking). The test that pins this invariant is +`plan_scope_admits_only_plans_its_dispatch_arm_can_run` (`card_engine/src/tests.rs:3657`) — it asserts, +for every `PhysicalPlan`, that `PlanScope::Plane.admits(plan) == (CandidatePlan::of(plan).is_some() || +plan == PhysicalPlan::PlanePopcountOrder)`. **Any change that widens `PlanScope::Plane` must change this +assertion in the same commit**, and must not do so without also giving the `Prep::Plane` dispatch arm a +matching executor — that pairing is the entire lesson of #829, and it's what a hypothetical +implementation of this doc's design would have to get right. + +## Proposed design (for the record, not recommended for implementation) + +### Can PrintingCompose reuse the plane bits instead of rebuilding? + +Traced, not assumed. `plane_bits` (`Prep::Plane`'s artifact) is a **card**-space bitmap: `eval_planes` +(`card_engine/src/planes.rs:1531`) compiles the query's `PlaneExpr` into `n_cards` bits, one per card, +existentially — "does this card have some printing that satisfies the predicate." `PrintingCompose`'s own +build, `compose_printing_bits` (`card_engine/src/lib.rs:7298`), produces a **printing**-space bitmap — +`n_printings` bits — by recursively composing leaf indexes (`legality_leaf_bits`, `rarity_cmp_leaf_bits`, +`border_leaf_bits`, `broadcast_card_bits_to_printings`, range scatters, …) over `compose_source(filter, +unsplit, plane)`, which under a `Plane` acquire is `unsplit` — the whole pre-split predicate, never a +residual (Round 12 already established this: `PlanePopcountOrder.applicable` requires `filter == +FilterExpr::True`, so nothing is left over for compose to see as a "residual"). Neither `compose_printing_bits` +nor its cost-estimate counterpart `compose_printing_estimate` (line 7680) reads `plane` or `plane_bits` at +all — confirmed by Round 12 for the estimate function and confirmed again here for the actual build. +There is no code path today, estimate or execution, that starts `PrintingCompose` from an existing plane. + +Could one be added? For the one sub-case where it's structurally possible — a single card-invariant +broadcast leaf (`is_broadcast_leaf_shape`, e.g. `pow<=2`), where compose's own build step +(`broadcast_composable_card_bits` → `broadcast_card_bits_to_printings`) independently re-derives a card +bitmap that `eval_planes` already computed — reusing `plane_bits` in place of `broadcast_composable_card_bits`'s +output would work and would be strictly cheaper than compose's own path. But it doesn't get you a plan +that can *beat* `PlanePopcountOrder`: for `Mode::Card` (the only mode `Prep::Plane` ever serves — +`plane_popcount_order_applicable` requires `mode == Mode::Card`), the next thing `PrintingCompose` does +with its card bitmap is find one representative printing per matching card — and with `filter == True`, +that's "any printing, picked by `prefer`," the identical rule `push_card_matches`/`exec_plane_popcount_order_with_bitmap` +already applies to `plane_bits` directly. A plane-bits-reusing `PrintingCompose` doesn't converge to +"cheaper than `PlanePopcountOrder`" — it converges to *being* `PlanePopcountOrder`, reached by a more +expensive path (still deriving `card_bits`, an exact total via `compose_total_for_mode`, and a paging +decision `PlanePopcountOrder` skips entirely). For legality leaves (existential, sometimes divergent +across a card's printings — see `docs/issues/00667-engine-legality-divergent-carveout.md`), the situation +is the same or worse: `eval_planes` already resolves the card-existential answer correctly, including the +divergent-format carveout, and there is nothing left for a printing-space rebuild to add once `filter == +True` removes any per-printing residual to verify. + +**Conclusion: there is no version of "let PrintingCompose start from the plane" that produces a plan +distinct from, and cheaper than, `PlanePopcountOrder`.** The ceiling of this idea is re-implementing +`PlanePopcountOrder` through a more roundabout path. This is the mechanistic explanation for the 0/3,209 +empirical result above, not just a correlation with it. + +### Argmin/dispatch mechanics (if someone pursues this anyway) + +Named for a future implementer, in case a narrower slice (some predicate shape or page geometry this +sweep didn't sample) is later found to have real value: + +- **`PlanScope::admits`** (`card_engine/src/lib.rs:9322-9330`) would need a third disjunct: + `PlanScope::Plane => CandidatePlan::of(plan).is_some() || matches!(plan, PhysicalPlan::PlanePopcountOrder + | PhysicalPlan::PrintingCompose)`. +- **Not `CandidatePlan::of`** (line 9152) — that type is deliberately exhaustive over exactly the two + candidate-list executors (`exec_from_candidates`'s repertoire); `PrintingCompose` is a different kind of + plan (self-composing, non-materializing-via-candidate-list) and doesn't belong in it. +- **`run_query_routed`'s `Prep::Plane` dispatch arm** (`card_engine/src/lib.rs:12618-12632`) would need a + new arm ahead of the generic `(p, Prep::Plane) => exec_from_candidates(...)` fallthrough: + `(PhysicalPlan::PrintingCompose, Prep::Plane) => { ... }`, calling `printing_compose_fastpath` on + `compose_source(filter, unsplit, plane)` and falling back into the existing `exec_from_candidates(..., + plane_bits as candidate list)` path on `None` (a decline) — mirroring how `Prep::Range`'s own + `PrintingCompose` arm (line 12654) handles its fastpath declining. +- **`acquire_plan_features`'s `Plane` branch** (lines 11899-11937) would need to additionally price + `PrintingCompose` whenever it's applicable alongside the plane, by calling the same field-computation + logic the `PrintingCompose.applicable` branch further down already has (lines 12016-12530-ish) — not + duplicating it. See "Cost-model piece" below for why this is a real cost concern once (if) the plan is + admissible, separate from Round 12's finding that it's currently inert. +- **Test to update in the same commit**: `plan_scope_admits_only_plans_its_dispatch_arm_can_run` + (`card_engine/src/tests.rs:3657`) — its `PlanScope::Plane` assertion must grow the `PrintingCompose` + disjunct exactly when the dispatch arm above ships, never before or after. +- **Test to add**: something in the shape of a `plane_acquire_admits_printing_compose_and_dispatch_can_run_it` + test — force the argmin's `PlanScope::Plane` to admit `PrintingCompose`, feed it a plane-acquire query + where `PrintingCompose` is applicable, and assert dispatch produces a real page (not the + `CandidatePlan::of_or_gathered` `debug_assert!(false, ...)` fallback path) — the #829-shaped regression + this change must never reintroduce is "argmin picks a plan dispatch silently downgrades or panics on." + `force_plan_differential_agreement` (`card_engine/src/tests.rs:3025`) already asserts every plan returns + identical rows over a random corpus and would need `PrintingCompose` exercised under a plane-holding + sort spec if it doesn't already. + +## Cost-model piece + +Round 12 found `acquire_plan_features`'s `Plane` branch leaves `PrintingCompose`'s five build-cost fields +(`broadcast_printings`/`scatter_printings`/`project_printings`/`popcount_words`/`compose_paging`) at +`mk_plan_feats`'s defaults, and concluded this is currently **inert** — `PlanScope::Plane` excludes +`PrintingCompose` from the argmin regardless of how it's costed, so fixing the costing changes zero +routing decisions today, only `explain`'s diagnostic ranking. That conclusion still holds, and this round +adds a stronger one: even if `PlanScope::Plane` were widened (making the costing live), the costing +couldn't change the outcome either, because `PrintingCompose` measures slower than `PlanePopcountOrder` +in every sampled row — an accurate cost model would (correctly) rank it last just as reliably as the +current, wrong-for-different-reasons zero-defaulted one does. If someone implements the widening anyway +(narrower slice found, or for `explain`/diagnostic accuracy on its own merits), the fields need populating +by reusing the `PrintingCompose.applicable` branch's own computation against `compose_source(filter, +unsplit, plane)` — see Round 12's trace for the specific reuse points (`compose_printing_estimate`, the +`is_broadcast_leaf_shape`/legality/range leaf arms) rather than re-deriving them. + +## Risks and staging + +Not applicable in the sense of "here's how to roll this out safely," because the recommendation is not to +roll it out. Recorded for whoever revisits this: + +- **Blast radius if built anyway**: `PlanScope::admits` (1 line), one new `tests.rs` assertion, one new + dispatch arm in `run_query_routed` (~10-20 lines including its decline fallback), and — the expensive + part — folding `PrintingCompose`'s build-cost fields into `acquire_plan_features`'s `Plane` branch by + sharing logic with its ~300-line `PrintingCompose.applicable` branch. Call it 50-80 net new lines plus + whatever refactor is needed to share the two branches' field computation without duplicating it. +- **What could go wrong**: a second #829-shaped panic if the `PlanScope` widening and the dispatch arm + ship in different commits (or if a future refactor moves one without the other — the test catches this + only if it's kept in lockstep, which is a discipline, not a guarantee); an unconditional new cost + computation on every `Plane`-acquire query (a large share of `unique=card` traffic — every legality, + color, rarity, or type predicate that folds entirely into the plane) in exchange for zero routing + benefit, which is a straightforwardly bad trade given this round's numbers. +- **Precedent for gating experimental engine behavior**: `COMPOSE_SIGMA_ENABLED` + (`card_engine/src/lib.rs:6339`, `guard_env("CARD_ENGINE_COMPOSE_SIGMA_ENABLED", 0u8) != 0`, default off) + is exactly this codebase's existing pattern for a change that's implemented but not trusted by default — + and `PRINTING_COMPOSE` itself (line 6380, default *on*) is the same mechanism used as a kill switch for + an already-shipped feature. If this were built despite the finding above, it should ship behind a + same-shaped flag (default off), not unconditionally — but the honest recommendation is not to build it + rather than to gate it. + +## Explicitly out of scope / open questions + +- **Why Round 12's 71% "applicable" figure and this round's 18.3% "applicable and didn't decline" figure + disagree.** Not chased down — see "Spike" above. Doesn't affect the conclusion, since a fastpath that + declines contributes zero wins either way. +- **Resolved: whether `PlanePopcountOrder` is ever unavailable under `Plane` acquire, leaving + `PrintingCompose` to compete against only `GatheredScan`/`StreamedSelect` — the one scenario where + Round 12's real 10.7%/6.7% win-rate margins (149µs/142µs) could represent a genuine, capturable + routing gap this round's `PlanePopcountOrder`-relative measurement wouldn't see.** Checked directly: + sampled 14,291 real `plane`-acquire queries (realistic traffic, fresh run) — `PlanePopcountOrder` was + applicable in **100%**, 0/14,291 missing. Combined with the "Re-verified" pairwise data on `#852` + (`GatheredScan vs PlanePopcountOrder` and `PlanePopcountOrder vs StreamedSelect`, both 100% ordered + right, 0.00µs mean regret), there is no real subset where this degrades to a two-plan comparison — + `PlanePopcountOrder` is essentially always present and already correctly chosen. Round 12's real + margins were genuine but measured against plans that were never the actual incumbent; there is no + ~100µs+ improvement on the table via routing here. +- **Whether some predicate shape entirely outside this sweep's sampler (both `QuerySampler` modes, plus a + manual limit/offset sweep) could flip the result.** Nothing in 3,209 rows across four regimes did. The + limit dimension specifically is closed, not just inductively suggestive: `limit=1,000,000` is the API's + own uncapped-query sentinel and already exceeds the corpus's true printing/card cardinality, so there is + no larger limit a real query could ever issue — the margin's narrowing-but-never-crossing trend was + chased to its actual ceiling, not just a large sample point. What remains open is predicate SHAPE + (leaf types, `And`/`Or` combinations) outside what the two `QuerySampler` modes generate, which is a + real, separate axis this sweep didn't fully control for. +- **Non-default `prefer` was tested (regime 2) and didn't change the qualitative result**, but only one + sample was run varying it; if this doc is ever revisited, that's already covered rather than a gap. +- **Whether `PrintingCompose`'s own acquire path (`Prep::Range(CountSource::PrintingCompose)`) — where it + competes on equal footing and does win a meaningful share, per #852 — has any remaining routing + problems of its own.** Out of scope here; that's #852's territory, and it's already closed as resolved + per its own "Re-verified" section. diff --git a/docs/issues/reference-engine-cost-model-cleanup-final-ab.md b/docs/issues/reference-engine-cost-model-cleanup-final-ab.md new file mode 100644 index 000000000..26d58e2b6 --- /dev/null +++ b/docs/issues/reference-engine-cost-model-cleanup-final-ab.md @@ -0,0 +1,499 @@ +# Cost Model Cleanup — Final A/B Against `main` (Round 27) + +Round 27, and the last one before this branch splits into PRs. Twenty-six rounds of work on +`costcell/trunk` are documented across +[local-engine-gathered-scan-card-printing-varying-depth.md](local-engine-gathered-scan-card-printing-varying-depth.md) +(Phase 1, Rounds 0-10), [#852](00852-engine-compose-acquire-p3-p4-ranking.md) (resolved as a side +effect), [local-engine-plane-acquire-compose-costing.md](local-engine-plane-acquire-compose-costing.md) ++ [local-engine-plane-scope-printing-compose-executor.md](local-engine-plane-scope-printing-compose-executor.md) +(Rounds 12-13, "don't build this"), +[done/local-engine-gathered-scan-undercosted-arith-existential-and.md](done/local-engine-gathered-scan-undercosted-arith-existential-and.md) ++ [local-engine-domain-cards-existential-arith-and.md](local-engine-domain-cards-existential-arith-and.md) +(Rounds 15-25), and [reference-engine-cost-model-state-2026-08.md](reference-engine-cost-model-state-2026-08.md) +(Round 26's whole-engine survey). All of that validation was against the branch's own history — each +round measured its own before/after, and Round 26 measured only `costcell/trunk` against itself. +**Nothing in the tree so far measures the whole branch against `main` in one sitting.** This doc is +that measurement: two isolated builds, five harnesses, one sitting, no code changed. + +## Method + +Two release wheels, built with `maturin build --release` (never `maturin develop`, which rewrites the +shared `.venv`'s `card_engine.pth` and would flip every other session's `import card_engine` — see +the shared-checkout note in project memory). Each wheel unzipped to its own scratch directory and +selected per-invocation via `PYTHONPATH`, verified by printing `card_engine.__file__` and hashing the +`.so` before any measurement — confirmed four distinct binaries (plain + `routed-phases` for each +build), and confirmed the shared `.venv`'s own `import card_engine` still resolves to the primary +checkout throughout, untouched. + +- **`main`** @ `ca016410`, built in a fresh detached worktree. +- **`costcell/trunk`** @ `ddba298a`, built in this round's own worktree (`costcell/27-final-ab`). + +Corpus: `benchmarks/bitplanes/corpus.jsonl` (97,812 printings), read-only, from the primary checkout — +never written to; every harness was pointed at a `--shm-path` under scratch instead of the corpus's +own directory (the default `--shm-path` would have written a `.store` file next to the read-only +corpus). + +`bench_regret_matrix.py` needs a `routed-phases` build for its decline-row population; both `main` +and `costcell/trunk` wheels were built both ways (plain for the other four tools, `routed-phases` for +the regret matrix) so the two sides are always compared like-for-like. + +### Canary: the measurement doc's own warning, reproduced + +[reference-cost-model-measurement.md](reference-cost-model-measurement.md) warns that a same-build, +same-seed pair false-positived at the old (2, 7) trial defaults and reads clean at 30. On this +machine — a shared dev box with a visible background load (pants test workers, MCP servers, browser +automation processes) at the time of this run — **30 was not enough**: three same-build pairwise +checks at the tool's own default trial count (30) read `-1.1`, `+2.0`, `+3.1` µs against a ~51-54 µs +mean latency, each with a bootstrap CI excluding zero in a different direction. Raising `--trials` to +60 (the measurement doc's prescribed remedy when a canary fires) tightened individual pairs to +~0.3 µs, but a 3-pair pooled same-build check at 60 trials (n=2,371 shared queries) still read a small +systematic `+0.5 µs, CI [+0.4, +0.7]` — traced to an order effect (the second run in a pair reads +slightly slower than the first on this machine), not random noise, since all three pairs shared the +same first/second ordering. + +Because that order effect is real, every `main`-vs-`costcell/trunk` latency round below alternates +which build runs first, so the effect cancels in the pooled result rather than biasing it. See +[Latency](#latency) for why this matters to the headline number. + +## Cost/feature accuracy + +### `bench_cost_model_agreement.py --seconds 300 --seed 0` + +| | `main` | `costcell/trunk` | +|---|---|---| +| queries sampled | 87,212 | 97,575 | +| cells within `[0.8, 1.25]` (by acquire) | 13/17 | 12/17 | +| cells within `[0.8, 1.25]` (by unique) | 10/12 | 10/12 | + +One FAIL flip, both directions checked: + +- **New FAIL, immaterial**: `PlanePopcountOrder / plane` — median `0.81` (main, PASS) → `0.80` + (trunk, FAIL). This is a boundary artifact, not a real change: the displayed medians round to the + same two decimals `main` passed on. Confirmed inert by both the regret matrix (`plane` acquire is + 0% of all SHARE, mean regret `0.00 µs` on both builds) and pairwise ordering (`PlanePopcountOrder` + wins its argmin 100% of the time under `plane` acquire on both builds) — matches Round 26's own + "Explicitly not candidates" finding for this exact cell. +- **No FAIL→PASS flips.** `GatheredScan / candidates` moved from median `0.61` (main, 15% within 25%) + to `0.79` (trunk, 31% within 25%) — real, substantial movement toward agreement — but stays a hair + under the `0.8` floor on both builds, so the verdict column doesn't change. +- Everything else moved by roughly the sampling-driven ~12% larger `n` (trunk sampled more queries in + the same 300s wall-clock budget) with proportionally similar ratios — no other qualitative shift. + +### `bench_feature_accuracy.py --seconds 300 --seed 0` (mode=uniform, default) + +**Fixed in Round 28** ([local-engine-gathered-scan-card-printing-varying-depth.md](local-engine-gathered-scan-card-printing-varying-depth.md)'s +own Round 28 section has the full bisection, mechanism, and confirmation pass) — recorded here as this +survey originally found it, plus the resolution: + +| feature (pooled) | `main` median | `costcell/trunk` median (as surveyed here) | `costcell/trunk` median (post Round 28 fix) | verdict | +|---|---|---|---|---| +| `scan_units` | 1.00 (no flag) | 0.70 | 0.94 | UNDER-COUNTS → clean | + +`main`: 697,375 feature-rows, `scan_units` reads clean (median 1.00, no verdict flag). `costcell/trunk` +as surveyed by this round: 705,768 rows, `scan_units` reads `0.70` and is flagged `UNDER-COUNTS` pooled +and across nearly every `unique`/`prefer` slice. This reproduced Round 26's own number for this exact +cell (`reference-engine-cost-model-state-2026-08.md`, "Feature accuracy" section: median 0.70, +"already covered, the era-correlated print-position confound... plus the printing-varying range depth +work") to two decimal places, and this A/B added the piece Round 26 didn't have: `main` does not have +this problem. **Round 28 bisected it precisely**, rather than accepting the "byproduct of this +branch's own fixes" framing as the final word: the actual trigger was a single commit +(`e1c40466`, this branch's own Round 7) whose broad-guard `scan_units` scale — fit exclusively against +`unique=card` samples, exactly like its sibling `COMPOSE_RANGE_AND_BROAD_SCAN_SCALE` from Round 4 — +was applied unconditionally to `Mode::Printing`/`Mode::Artwork` too, where the real +`printings_examined / n_printings` ratio for that guard-fired population reads an exact, zero-spread +1.0 (those modes' kernels never short-circuit). Scoping both scales to `Mode::Card` only closed the +gap: pooled `scan_units` median `0.70` (UNDER-COUNTS) → `0.94` (clean, inside `main`'s own `[0.8, +1.25]` band), confirmed via a fresh isolated-wheel `main`-vs-fixed-tip A/B at this same `--seconds 300 +--seed 0` protocol. The residual `0.94` vs. `main`'s `1.00` is the two other, already-documented, +un-touched-by-this-fix contributors (`PrintingCompose`'s "narrow"-bucket under-count, named and +deferred by Round 7 itself, and the era-correlated existential-leaf confound Rounds 17/20/25 already +characterized as out of their own blast radius) — real, pre-existing, not introduced by any commit on +this branch, and not attempted this round; see Round 28's own section for why (the root cause is +`domain_cards`'s documented broad-range undercount for bare ranges, which nine prior rounds already +found hard to fix directly). + +No other feature changed materially in the pooled table. + +## Regret + +### `bench_regret_matrix.py --seconds 180 --mode realistic` (routed-phases builds) + +| | `main` | `costcell/trunk` | Δ | +|---|---|---|---| +| multi-plan queries | 75,112 | 80,499 | — | +| total regret | 120.3 ms | 71.4 ms | **-41%** | +| mean regret/query | 1.60 µs | 0.89 µs | **-44%** | + +SHARE by compose paging branch (the mechanism most of this branch's rounds targeted): + +| branch | `main` SHARE | `main` mean | `costcell/trunk` SHARE | `costcell/trunk` mean | +|---|---|---|---|---| +| `Perm` | 46% | 5.99 µs | 68% | 4.86 µs | +| `OrderbyWalk` | 42% | 14.88 µs | 10% | 1.90 µs | +| `Gather` | 8% | 0.19 µs | 15% | 0.18 µs | + +`OrderbyWalk`'s absolute contribution collapsed from ~50.5 ms to ~7.1 ms (SHARE is a fraction of a +shrinking pie, so read the absolute too) — the single largest driver of the whole-branch improvement. +`Perm`'s absolute contribution also fell slightly (~55.3 ms → ~48.6 ms). Neither branch's *own* +cost-formula fix is what's recorded as shipped in the docs read above (Round 26 names `OrderbyWalk`'s +fix as still "fully-designed, unshipped"); the reduction is consistent with the Sigma decision rule +(`docs/issues/local-engine-compose-perm-sigma-decision-rule.md` and the Step 4-7 commits in this +branch's recent history) steering more queries away from the branch transitions where `OrderbyWalk`'s +miscalibration would have been exposed, rather than fixing the miscalibration itself. + +The `#852` story, specifically — `picked → best` transitions: + +| transition | `main` n | `main` SHARE | `costcell/trunk` n | `costcell/trunk` SHARE | +|---|---|---|---|---| +| `PrintingCompose → GatheredScan` | 1,072 | 43% | 180 | 8% | +| `StreamedSelect → GatheredScan` | 1,135 | 18% | 1,573 | **49%** | + +The misroute `#852` targeted dropped 83% in raw occurrence count (1,072 → 180) and from the single +largest SHARE to a minor one. But `StreamedSelect → GatheredScan` — the compound-existential-plane +`GatheredScan` cost-formula miscalibration Round 26 explicitly parked as "needs a saturating/banded +rate, not a flat linear one" — grew to the largest single slice on `costcell/trunk`, both in SHARE and +in absolute terms (~21.7 ms → ~35.0 ms). This matches Round 26's own ranking of it as the largest +still-open item, now visible for the first time against a genuine `main` baseline rather than only +against the branch's own history. + +## Latency + +### `bench_query_latency_ab.py --mode realistic --trials 60 --sample 800`, 4 rounds, order-alternated + +| round | seed | order | B - A | 95% CI | verdict | +|---|---|---|---|---|---| +| 1 | 1 | main, trunk | -1.9 µs | [-2.4, -1.4] | trunk faster | +| 2 | 2 | trunk, main | -0.1 µs | [-0.8, +0.5] | no detectable difference | +| 3 | 3 | main, trunk | -0.4 µs | [-1.1, +0.3] | no detectable difference | +| 4 | 4 | trunk, main | +0.6 µs | [-0.5, +1.5] | no detectable difference | +| **pooled** | all 4 | alternated | **-0.4 µs** | **[-0.8, -0.1]** | trunk marginally faster | + +Pooled over 3,158 queries shared across all four rounds: `costcell/trunk` reads a mean latency of +52.0 µs against `main`'s 52.4 µs — nominally outside the bootstrap's zero-crossing, in the expected +direction, but the magnitude (~0.8% of mean latency) is the same order of magnitude as this machine's +own measured same-build noise floor (the pooled canary read `+0.5 µs` under an *unbalanced* run order; +see Method). Only 1 of the 4 individual rounds was independently significant. + +**Reconciling this with the 41% regret-matrix win**: regret is concentrated in a specific, minority +population — compose-paging-branch mismatches under `printing_compose` acquire, which the regret +matrix shows is ~13-27% of all multi-plan queries (`n=20,458`/`75,112` on main, `21,931`/`80,499` on +trunk) and produces the vast majority of the SHARE. Pooled over *all* realistic-mode traffic — +dominated by cheap `candidates`/`plane` lookups where nothing changed — that improvement is real but +small enough, at an 800-query-per-round sample, to sit right at the edge of what this environment can +resolve from noise. A user issuing the specific query shapes the regret matrix flags would feel a +real, measurable improvement; a user issuing a uniformly-sampled realistic query would not reliably +notice one at this sample size. + +## Pairwise ordering + +### `bench_pairwise_ordering.py --seconds 300`, realistic and uniform, both builds + +The `#852` cell head-to-head against `main` (not against the branch's own Round-0 baseline, which the +brief for this round flagged as measured after some fixes had already shipped): + +| mode | pair / acquire | `main` ordered-right | `main` mean regret | `costcell/trunk` ordered-right | `costcell/trunk` mean regret | +|---|---|---|---|---|---| +| realistic | `GatheredScan` vs `PrintingCompose` `[printing_compose]` | 80% | 8.09 µs | 90% | 3.03 µs | +| uniform | `GatheredScan` vs `PrintingCompose` `[printing_compose]` | 91% | 3.97 µs | **87%** | **5.25 µs** | + +Against a real `main` baseline, `#852`'s realistic-mode improvement is **80% → 90%**, not the +**69% → 97%** the tracking docs' own internal comparison reports — confirming the round's brief was +right to be suspicious of that number; the internal baseline was measured on a `costcell/trunk` +ancestor that already carried some of Round 0-10's fixes, which inflates the apparent delta. The real, +`main`-relative improvement is smaller but still genuine and in the right direction. + +**Under `uniform` mode — the sampler built specifically to reach rare tails — the same pair got +worse**: 91% → 87% ordered right, mean regret nearly doubling (3.97 → 5.25 µs). This is the branch's +one clear pairwise-ordering regression: the fixes are tuned to realistic-traffic-shaped populations +(the `QuerySampler`'s traffic weighting) and give up a small amount of accuracy on the query shapes +`uniform` mode is designed to surface. Pooled (not sliced by acquire), the same direction holds: +`GatheredScan` vs `PrintingCompose` overall reads 91% → 87% under uniform, 84% → 89% under realistic. + +The structurally-inert `[plane]` pairs (`PlanePopcountOrder` always wins its argmin regardless of how +any competitor is priced) were re-confirmed on both builds, both modes — 100% ordered right, +0.00-0.01 µs regret throughout, consistent with Round 12/13's original finding. + +## Honest verdict + +The aggregate effect is real, but noisier and smaller than the round-by-round narrative alone would +suggest, and it is not uniformly positive. + +**What holds up:** +- Regret fell 41% in total, 44% per query, on a realistic traffic mix — the single most important + number here, and it is not a wash: the reduction is dominated by the `OrderbyWalk` paging branch + collapsing from 42% to 10% SHARE, a real, large, `main`-relative win. +- The `#852` misroute (`PrintingCompose → GatheredScan`) dropped 83% in occurrence and from the + largest SHARE to a minor one — genuinely fixed, just not by as much as the branch's own internal + comparison claimed (80%→90% ordered-right against `main`, not 69%→97%). +- A small, marginally-significant end-to-end latency win (-0.4 µs pooled, in the expected direction) + survived a canary-verified, order-alternated measurement — real, but small enough that a single + realistic user request would rarely notice it. + +**What doesn't, or is smaller than advertised:** +- `#852`'s own internal "69%→97%" figure does not survive a head-to-head against `main` — the true + number is 80%→90%, because the internal baseline had already absorbed some of Round 0-10's fixes. +- `scan_units` pooled feature accuracy got measurably worse (`main` 1.00 clean → `costcell/trunk` 0.70, + UNDER-COUNTS) — a real, `main`-relative regression, low-severity per this round's own regret-matrix + cross-check, and **fixed by Round 28**: a mode-scoping bug (a `unique=card`-only-calibrated scale + applied to Printing/Artwork mode too) traced to Round 7's own `e1c40466`, closed by scoping it to + `Mode::Card`. Pooled median now `0.94`, inside the same band `main`'s `1.00` sits in — see + `local-engine-gathered-scan-card-printing-varying-depth.md`'s Round 28 section. +- Pairwise ordering for `GatheredScan` vs `PrintingCompose` under `printing_compose` acquire got worse + under `uniform` sampling (91%→87%) even as it improved under `realistic` sampling (80%→90%) — the + branch traded rare-tail accuracy for common-case accuracy on this one cell, which is a defensible + trade given realistic traffic is what users send, but it is a trade, not a pure win. +- `StreamedSelect → GatheredScan` grew to the largest single regret slice on `costcell/trunk` + (18%→49% SHARE, ~21.7ms→~35.0ms absolute) — not a regression introduced by this branch (Round 26 + already named and parked it), but proof that the branch's 26 rounds did not touch the largest + remaining opportunity, which is now more visible precisely because everything else shrank around it. +- The 41% regret win does not translate into a latency difference an average realistic query would + reliably notice, because the affected population is a minority of realistic traffic. + +**Overall**: the effort was worthwhile and the routing-regret number is genuinely, substantially +better against `main`, not just against the branch's own history — but a skeptical reviewer reading +only the round-by-round docs would come away expecting a bigger, cleaner, more uniform win than what +a fresh `main`-relative measurement actually shows. Ship it, but do not carry the `69%→97%` or +"41% regret reduction ≈ 41% faster" framings into the PR descriptions; use the numbers in this doc. + +## Round 29: Post-Fix Comprehensive State Check + +Diagnostic snapshot, no code changes. Round 27 (above) ran the first `main`-vs-`costcell/trunk` A/B +and found a real regression alongside the wins: `bench_feature_accuracy.py`'s pooled `scan_units` +read clean on `main` (1.00) but `UNDER-COUNTS` on the branch (0.70). Round 28 +([local-engine-gathered-scan-card-printing-varying-depth.md](local-engine-gathered-scan-card-printing-varying-depth.md)) +bisected that to its own Round 7 commit (`e1c40466`), fixed it by scoping two broad-guard `scan_units` +scales to `Mode::Card`, and ran a *confirmation* pass scoped to the regression itself (scan_units, +regret matrix, cost-model-agreement, a latency A/B, pairwise ordering — all at shorter `--seconds` +than Round 27's own sweep). This round redoes Round 27's FULL sweep — every feature, the whole +cost-model-agreement table, the regret matrix ranked by share, both pairwise-ordering modes, and a +canary-gated latency A/B — against the now-fixed tip, so the branch has one honest, current, +complete picture before it splits into PRs. + +**Method.** Same protocol as Round 27: two isolated release wheels (`maturin build --release`, never +`develop`), `main` @ `ca016410` built in a fresh detached worktree, `costcell/trunk` @ `288402a0` built +in this round's own worktree (`costcell/29-final-state-check`) — four wheels total per side (plain + +`routed-phases`, the latter only for the regret matrix), each verified by `card_engine.__file__` and a +distinct `.so` hash before use. Corpus: the same read-only `benchmarks/bitplanes/corpus.jsonl` +(97,812 printings), every harness pointed at its own `--shm-path` under scratch. All five harnesses run +at `--seconds 300 --seed 0` (`--mode realistic` for the regret matrix; both `realistic` and `uniform` +for pairwise ordering), matching or exceeding Round 27's own budget. + +### Feature accuracy — the full table, not just `scan_units` + +`bench_feature_accuracy.py --seconds 300 --seed 0` (mode=uniform, the tool's default). `main`: +466,524 feature-rows. `costcell/trunk`: 463,359 feature-rows. + +| feature (pooled) | `main` median | verdict | `trunk` median | verdict | +|---|---|---|---|---| +| `compose_scan_printings` | 1.28 (n=1,754) | OVER-COUNTS | 1.47 (n=1,741) | OVER-COUNTS | +| `printings_walked` | 0.85 (n=46,172) | clean | 0.85 (n=45,875) | clean | +| `matches` | 1.00 (n=141,410) | clean | 1.00 (n=140,447) | clean | +| `eval_domain` | 1.00 (n=136,638) | clean | 1.00 (n=135,697) | clean | +| `scan_units` | 1.00 (n=140,550) | clean | 0.94 (n=139,599) | clean | + +Round 28's fix holds: pooled `scan_units` is clean on both builds, `trunk`'s 0.94 sitting inside the +same `[0.8, 1.25]` band `main`'s 1.00 does. `compose_scan_printings` is flagged OVER-COUNTS on **both** +builds at comparable magnitude (1.28 vs 1.47) — pre-existing on `main`, not something the branch +introduced. + +**Slicing by acquire/mode surfaces real per-slice differences the pooled row hides — but they are not +new.** Diffing every flagged (UNDER/OVER-COUNTS) cell between the two full tables: + +- **One genuine incidental fix**: `scan_units / card / prefer=default` — `main` 1.37 (OVER-COUNTS) → + `trunk` 1.00 (clean). +- **`scan_units [card_range_popcount]`**: `main` 1.00 (clean, n=5,195) → `trunk` 0.43 (UNDER-COUNTS, + n=5,164). This is not a new regression — `0.43` is `COMPOSE_BARE_RANGE_BROAD_SCALE`, the exact + constant Round 6 shipped (`card_engine/src/lib.rs:11628`, applied at line 12258), fit against real + ns-time error (93.3M → 16.0M abs error, held-out validated) rather than against this literal + `printings_examined` counter. Round 6's own doc named this exact tradeoff at the time ("flags the + sibling `else` branch... as itself badly under-calibrated... not fixed this round"). A feature-level + bias deliberately buried in a cost-accurate rate, exactly the risk `bench_feature_accuracy.py`'s own + docstring warns about — not something this round re-litigates. +- **`scan_units [printing_compose]` and its `/card`, `/printing`, `/artwork` slices**: `main` reads + 1.63 (OVER, card) / 1.09 (clean, printing) / 1.00 (clean, artwork); `trunk` reads 0.52 (UNDER, card) / + 0.38 (UNDER, printing) / 0.39 (UNDER, artwork). This is Round 7's own already-named, already-deferred + "narrow bucket" `PrintingCompose` under-count (the `domain_cards` broad-range estimate for bare + ranges) — Round 28 itself named this as the residual gap between its fixed `0.94` and `main`'s `1.00`. + What's new **this round** is the full per-mode quantification: the narrow-bucket effect is not + card-specific, it spans all three `unique` modes at comparable severity (0.38-0.52), which no prior + round's narrower pooled/card-only view had shown directly. +- A few smaller slices move the same way for the same reason (`scan_units / card`, `/orderby=rarity`, + `/orderby=usd`) — all downstream of the same narrow-bucket population, not independent findings. + +Per the task brief for this round, both of these are the two already-documented residual gaps — +**known, deferred, unrelated to this round** — reported here with exact current magnitudes, not +re-investigated. + +### Cost-model agreement — the full table + +`bench_cost_model_agreement.py --seconds 300 --seed 0`. `main`: 62,916 queries sampled. `trunk`: +62,693 queries sampled. + +- **By acquire branch**: `main` 9/17 cells inside `[0.8, 1.25]`; `trunk` 10/17. One flip, FAIL → PASS: + `GatheredScan / candidates` — `main` 0.71 (27% within 25%) → `trunk` 0.98 (43% within 25%). This + continues the movement Round 27 already flagged as "real, substantial" (0.61→0.79, still short of the + floor) — this fresh sample crosses fully into agreement. +- **By distinct-on (`unique`)**: `main` 10/12 inside band; `trunk` 9/12 — one flip the other way, + PASS → FAIL: `GatheredScan / artwork` — `main` 1.02 (clean) → `trunk` 1.54 (UNDER-COSTED). This is + the by-unique face of the known, already-parked "compound-existential-plane `GatheredScan`" + miscalibration (Round 25/26, needs a saturating/banded rate) — see the Regret section below, where + the same mechanism shows up as the branch's single largest remaining regret slice. Named per this + round's brief, not re-investigated. +- Net: one cell fixed, one cell newly visible as failing (both attributable to already-tracked + mechanisms, not new problems) — acquire-level count improves by one, by-unique count worsens by one. + A wash in cell-count terms, not a regression in either underlying mechanism. +- Compose-paging predicted-vs-taken proportions (`Perm`/`OrderbyWalk`/`Gather`/decline counts under + each RANGE_ACQUIRES branch) are within a few rows of each other on both builds — no material shift. + +### Regret — ranked by share + +`bench_regret_matrix.py --seconds 300 --mode realistic --seed 0` (`routed-phases` builds). + +| | `main` | `costcell/trunk` | Δ | +|---|---|---|---| +| multi-plan queries | 81,935 | 82,018 | — | +| total regret | 114.3 ms | 80.2 ms | **-30%** | +| mean regret/query | 1.39 µs | 0.98 µs | **-30%** | + +Smaller than Round 27's own `-41%/-44%` (measured at `--seconds 180`, no explicit seed) — regret is +heavy-tailed and dominated by rare, extreme single-query misses (this round's own sample: `main`'s +largest single-query regret was 545.9 µs; `trunk`'s was 2,348.2 µs, one query in the +`StreamedSelect → PrintingCompose` transition), so the exact percentage is sensitive to which rare-tail +queries a given `--seconds`/`--seed` combination happens to sample. The **direction** — `trunk` +substantially lower total regret than `main` — replicates across both rounds' independent measurements. + +Compose-paging branch SHARE: `main` `Perm` 57% / `OrderbyWalk` 28% / `Gather` 10% / `Decline` 5%; +`trunk` `Perm` 69% / `OrderbyWalk` 10% / `Gather` 13% / `Decline` 8%. `OrderbyWalk`'s collapse (28%→10% +of a smaller pie; ~32 ms → ~8 ms absolute) is again the largest single driver — same mechanism Round 27 +found, though `main`'s own OrderbyWalk share reads differently between the two rounds (42% in Round +27's 180s run vs. 28% here), another heavy-tail sampling effect, not a moved target. + +`picked → best` transitions, ranked by SHARE: + +| transition | `main` n | `main` SHARE | `trunk` n | `trunk` SHARE | +|---|---|---|---|---| +| `StreamedSelect → GatheredScan` | 1,284 | 19% | 1,618 | **43%** | +| `PrintingCompose → GatheredScan` | 1,040 | 30% | 159 | 7% | +| `PrintingCompose → StreamedSelect` | 474 | 22% | 296 | 16% | +| `PrintingCompose(declined) → GatheredScan` | 435 | 21% | 280 | 15% | +| `GatheredScan → PrintingCompose` | 296 | 6% | 347 | 12% | +| `GatheredScan → StreamedSelect` | 445 | 2% | 399 | 2% | + +**`#852`'s misroute (`PrintingCompose → GatheredScan`) is robustly fixed**: 1,040 → 159 occurrences +(-85%), 30% → 7% SHARE — the largest slice on `main` is now a minor one on `trunk`, matching Round 27's +direction almost exactly (that round found -83%, 1,072→180). + +**`StreamedSelect → GatheredScan` — the known, already-parked compound-existential-plane `GatheredScan` +miscalibration (Round 25/26, "needs a saturating/banded rate, not a flat linear one") — is now +unambiguously the largest slice**: 19%→43% SHARE, and in absolute terms `main`'s ~21.7 ms → +`trunk`'s ~34.5 ms, which reproduces Round 27's own absolute-ms finding (~21.7ms→~35.0ms) almost +exactly even though the total-regret percentage this round differs. **Honest fraction closed vs. +open**: the one identified, targeted pathology (`#852`) is fixed; the single largest remaining one is +untouched by any of this branch's 30 commits and is now more prominent only because everything else +around it shrank. Named per this round's brief as known/deferred/unrelated — not re-investigated here. + +### Pairwise ordering — both modes + +`bench_pairwise_ordering.py --seconds 300 --seed 0`, `realistic` and `uniform`. + +The `#852` cell specifically, `GatheredScan vs PrintingCompose [printing_compose]`: + +| mode | `main` ordered-right | `main` mean regret | `trunk` ordered-right | `trunk` mean regret | +|---|---|---|---|---| +| realistic | 81% (n=11,390) | 6.90 µs | 93% (n=11,460) | 2.68 µs | +| uniform | 90% (n=16,249) | 3.92 µs | 90% (n=16,271) | 4.30 µs | + +**Realistic-mode improvement is stable and, if anything, slightly better than Round 27's reported +80%→90%**: this fresh sample reads 81%→93%. **Round 27's claimed uniform-mode regression for this +exact cell (91%→87%) does NOT reproduce here** — this round reads a flat 90%→90%. Given `uniform` mode +is deliberately built to reach rare tails, and this same doc has already shown (in the regret matrix, +above) that rare-tail metrics swing hard between independently-seeded 300s samples, the most honest +read is that Round 27's uniform-mode "regression" for this cell was itself sample noise, not a stable +property of the branch — flagged explicitly rather than carried forward as settled. (The gap-size +calibration did drift worse, 1.20→1.42 `gap meas/pred`, even though which plan wins stays right just as +often — a real, smaller, separate observation.) + +Pooled (non-acquire-sliced) `GatheredScan vs PrintingCompose`: realistic 82%→88%; uniform 90%→90% +(flat, both n≈20,700-20,800). + +**One other pair moved notably**: `GatheredScan vs StreamedSelect` under `uniform` mode improved +89%→95% (n≈33,600 both builds), a genuine secondary win (realistic mode: 95%→97%, smaller but same +direction). The structurally-inert `[plane]` pairs (`PlanePopcountOrder` always wins its argmin) stay +at 100% ordered-right, ~0.00-0.01 µs regret on both builds and both modes — re-confirmed inert, same as +Rounds 12/13/27. + +### Latency, with the canary stated explicitly + +Same-build canary first, per this round's own gate: `trunk-plain` vs itself, `--mode realistic +--sample 800 --trials 60 --seed 99`: **B - A = -1.2 µs, 95% CI [-1.5, -0.8], "B is FASTER"**. **Not +clean** — this reproduces Round 27's own finding of a real second-run-reads-faster order effect on +this shared box. Because of that, every real comparison below alternates which build runs first. + +Four order-alternated rounds, `--mode realistic --sample 800 --trials 60`, `main-plain` vs +`trunk-plain`: + +| round | seed | order | B - A | 95% CI | verdict | +|---|---|---|---|---|---| +| 1 | 1 | main, trunk | -2.34 µs | [-2.9, -1.9] | trunk faster | +| 2 | 2 | trunk, main | -2.33 µs | [-2.9, -1.8] | trunk faster | +| 3 | 3 | main, trunk | -0.63 µs | [-1.4, +0.0] | no detectable difference | +| 4 | 4 | trunk, main | -1.50 µs | [-2.5, -0.6] | trunk faster | +| **pooled** | all 4 | alternated | **-1.70 µs** | **[-2.06, -1.35]** | **trunk faster** | + +Pooled over 3,189 paired queries: `main` mean 53.7 µs (median 34.0 µs), `trunk` mean 52.0 µs (median +33.2 µs) — trunk reads about 3.2% faster. All four rounds point the same direction regardless of which +build ran first or second (main-first rounds 1/3 average -1.49 µs; trunk-first rounds 2/4 average +-1.92 µs — if the canary's own order bias were the whole story, alternating order should have flipped +this asymmetry, not left it in the same direction), and 3 of 4 rounds are individually significant. + +**This is a larger, more consistent signal than Round 27 found** (that round's pooled result was +-0.4 µs, CI [-0.8, -0.1], only 1 of 4 rounds individually significant, and explicitly called +"within...this environment's own noise floor"). This round's pooled -1.70 µs exceeds the same-build +canary's own -1.2 µs bias in magnitude, and holds across all four order-alternated rounds — a real, +reproducible, though still modest (~3% of mean latency, a query most users would not consciously +notice) wall-clock win. The exact magnitude clearly varies session-to-session on this shared box more +than the regret-matrix story alone would suggest — reported honestly as "trunk is measurably faster, +by an amount that itself varies between measurement sessions," not as a single fixed number. + +### Overall verdict + +`costcell/trunk` (`288402a0`) is net-positive against `main` (`ca016410`) on every axis measured this +round, and by a clearer margin on latency specifically than Round 27 found — but several exact +percentages (regret reduction, some pairwise-ordering deltas) show real run-to-run variance from +regret's heavy tail and should not be read as more precise than they are. + +- **Feature accuracy**: Round 28's fix holds (pooled `scan_units` clean on both builds). The full + per-slice sweep this round adds finds no new regression — every off-band cell traces to an + already-documented, already-deliberate tradeoff (Round 6's `card_range_popcount` scale, Round 7's + `printing_compose` narrow bucket) now quantified across all three modes for the first time, plus one + genuine incidental fix (`scan_units / card / prefer=default`). +- **Cost-model agreement**: one cell fixed (`GatheredScan/candidates`, a continuation of Round 27's own + partial finding), one newly visible as failing (`GatheredScan/artwork` by-unique, the by-unique face + of the already-parked compound-existential-plane issue) — a wash in count, not a new problem. +- **Regret**: total down substantially in both this round (-30%) and Round 27 (-41%); the exact number + is sample-sensitive but the direction is not. `#852`'s misroute is robustly fixed (-85% occurrence, + largest slice → minor slice, in both rounds). The known, parked `StreamedSelect → GatheredScan` + compound-existential-plane issue is now unambiguously the single largest remaining slice (43% SHARE, + ~34.5 ms, matching Round 27's absolute-ms finding almost exactly) — untouched by any of the 30 + commits, more prominent only because everything else shrank around it. +- **Pairwise ordering**: `#852`'s realistic-mode win is stable and reproduces (81%→93%, at least as + good as Round 27's 80%→90%). Round 27's claimed uniform-mode regression for the same cell (91%→87%) + does **not** reproduce this round (flat 90%→90%) — most likely sample noise in a rare-tail-seeking + mode, flagged rather than carried forward. A different pair (`GatheredScan vs StreamedSelect`) shows + a genuine secondary uniform-mode win (89%→95%). +- **Latency**: canary not clean (-1.2 µs), but four order-alternated rounds all point the same + direction and the pooled result (-1.70 µs, CI excluding zero) exceeds the canary's own bias — a real, + small (~3%), reproducible wall-clock win, larger than Round 27's own -0.4 µs finding. Round 27's + question of whether the branch is measurably faster than `main` is answered more confidently "yes" + this round than last, though the exact magnitude moves between sessions. +- **Known, deferred, unrelated to this round** (named per this round's brief, not re-investigated): + the compound-existential-plane `GatheredScan` cost-formula miscalibration (Round 25/26, needs a + saturating/banded rate — now confirmed as both the largest regret slice and the source of this + round's one CMA regression), and the `domain_cards`-driven "narrow bucket" `PrintingCompose` + under-count (Round 7 — now quantified across all three `unique` modes via this round's full + feature-accuracy sweep, previously only characterized pooled/card-specific). + +**Ship it.** No new regressions were found; every "worse than `main`" cell this round's fuller sweep +surfaced traces to an already-documented, already-parked, deliberate tradeoff or to regret's own +heavy-tailed sampling variance — not to anything introduced by the branch's 30 commits or by Round 28's +fix specifically. diff --git a/docs/issues/reference-engine-cost-model-state-2026-08.md b/docs/issues/reference-engine-cost-model-state-2026-08.md new file mode 100644 index 000000000..1272e7155 --- /dev/null +++ b/docs/issues/reference-engine-cost-model-state-2026-08.md @@ -0,0 +1,256 @@ +# Cost Model State of the Engine — 2026-08-31 Snapshot + +Round 26 of the accuracy/routing effort. This is a comprehensive diagnostic sweep, not a fix — no code +changed. Goal: find where the cost model is still wrong, ranked by real routing impact, so Round 27 +targets the actual biggest remaining lever. History read before measuring anything: +[local-engine-gathered-scan-card-printing-varying-depth.md](local-engine-gathered-scan-card-printing-varying-depth.md) +(Phase 1, Rounds 0-10), [00852](00852-engine-compose-acquire-p3-p4-ranking.md) (resolved), +[local-engine-plane-acquire-compose-costing.md](local-engine-plane-acquire-compose-costing.md) and +[local-engine-plane-scope-printing-compose-executor.md](local-engine-plane-scope-printing-compose-executor.md) +(Phase 2, "don't build this"), and +[done/local-engine-gathered-scan-undercosted-arith-existential-and.md](done/local-engine-gathered-scan-undercosted-arith-existential-and.md) ++ [local-engine-domain-cards-existential-arith-and.md](local-engine-domain-cards-existential-arith-and.md) +(Rounds 15-25, `PairTotals` shipped, joint refit failed a third time on clean data). + +Measured against an isolated release build of `costcell/trunk` @ `f5aed0a0` (`maturin build --release`, +extracted wheel, `PYTHONPATH`-pinned — never `maturin develop` into the shared `.venv`), corpus +`benchmarks/bitplanes/corpus.jsonl` (97,812 printings, primary checkout, read-only). + +**A surprise not in the required reading**: while cross-referencing, a second, independently-run +effort turned up covering `PrintingCompose`'s own walk/build cost model (`Perm`/`OrderbyWalk`, +"Sigma" work) — 6+ open/reference docs, 5 commits in the current `git log` head (#1009, #1060-#1065). +It is not in this doc's mandated reading list and this round did not audit it in depth, but the regret +matrix below shows it covers by far the largest share of real routing regret in the engine right now, +so it is called out explicitly rather than silently ranked against. + +## Full agreement table + +`bench_cost_model_agreement.py --seconds 300 --seed 0`, 107,516 queries sampled. + +``` +plan acquire n median p10 p90 within 25% +CardRangePopcount card_range_popcount 1700 0.81 0.70 0.89 52% +GatheredScan printing_compose 59128 1.15 0.48 5.24 24% +GatheredScan candidates 38867 0.76 0.47 1.98 30% FAIL +GatheredScan plane 5540 0.70 0.60 2.60 18% FAIL +GatheredScan printing_range_scan 2271 1.08 0.82 1.88 62% +GatheredScan card_range_popcount 1700 0.97 0.59 1.44 51% +PlanePopcountOrder plane 5540 0.77 0.54 0.97 36% FAIL +PrintingCompose printing_compose 28237 0.82 0.54 1.25 46% +PrintingCompose plane 4155 0.72 0.36 2.52 17% FAIL +PrintingCompose printing_range_scan 2090 0.70 0.03 1.02 27% FAIL +PrintingCompose card_range_popcount 1483 1.21 0.89 1.67 53% +PrintingRangeScan printing_range_scan 1042 0.92 0.52 2.55 39% +StreamedSelect printing_compose 42429 1.05 0.14 2.76 37% +StreamedSelect candidates 29017 0.82 0.54 1.11 48% +StreamedSelect plane 5540 0.92 0.83 3.13 72% +StreamedSelect card_range_popcount 1700 1.10 0.60 1.76 40% +StreamedSelect printing_range_scan 1682 1.10 0.76 2.08 53% +12/17 cells inside [0.8, 1.25] +``` + +By-unique table: 9/12 cells inside band; `GatheredScan/card` reads 0.80 (right at the boundary — the +`[0.8,1.25]` gate reads it FAIL, consistent with this cell hovering at exactly this line since Round 9). + +Annotation: + +| cell | status | disposition | +| --- | --- | --- | +| `GatheredScan`/`printing_compose` (1.15, 24%) | unchanged since Round 9 | **already covered** — Round 25's parked "residual ~1.5-2x compound-existential-plane" miscalibration, needs a saturating/banded rate, explicitly out of scope this round | +| `GatheredScan`/`card` (0.80, 25%, FAIL at boundary) | unchanged since Round 9 | **already covered** — same population as above, pooled by unique instead of acquire | +| `GatheredScan`/`candidates` (0.76, 30%, FAIL) | unchanged since Round 9's partial fix | **already covered, still open** — Round 9 fixed the zero-match fixed-cost mechanism; Round 8's mechanism 2 (card-mode residual-selectivity discount) was never shipped. See "Ranked candidate list" below — this is the one already-diagnosed item worth a fresh look | +| `GatheredScan`/`plane`, `PlanePopcountOrder`/`plane` (0.70/0.77, FAIL) | new cells, not previously called out per-plan | **checked, routing-inert** — pairwise ordering confirms `PlanePopcountOrder` wins the `Plane`-acquire argmin essentially always at 0.00µs regret regardless of these absolute-cost errors (see Pairwise section). Extends the already-known "absolute costing under `Plane` acquire doesn't matter" finding from `PrintingCompose` (Round 12/13) to the other two plans in that branch | +| `PrintingCompose`/`plane` (0.72, 17%, FAIL) | unchanged | **already covered** — Round 12/13, structurally excluded from the `Plane`-acquire argmin, fixing the costing changes nothing | +| `PrintingCompose`/`printing_range_scan` (0.70, 27%, FAIL) | new cell | **checked, low impact** — pairwise ordering shows this pair at 98-99% ordered right, 0.13-0.24µs mean regret, n≈678, negligible real share | + +## Regret matrix, ranked by SHARE + +`bench_regret_matrix.py --seconds 180 --mode realistic --seed 0`, 81,119 multi-plan queries, **total +regret 61.8ms, mean 0.76µs**. + +``` +acquire n miss% mean SHARE +printing_compose 21944 8% 2.40 85% +candidates 43154 1% 0.21 15% +printing_range_scan 315 3% 0.38 0% +plane 14338 0% 0.00 0% +card_range_popcount 1368 0% 0.00 0% + +compose paging branch (every row where a compose cost was priced) +Perm 9832 12% 3.94 63% +Gather 57707 1% 0.18 17% +OrderbyWalk 3682 8% 1.97 12% +Decline 9898 2% 0.55 9% + +picked -> best (only when they differ) +StreamedSelect -> GatheredScan 1597 66% 21.88µs 57% +PrintingCompose -> StreamedSelect 269 98% 31.22µs 14% +GatheredScan -> PrintingCompose 368 85% 19.35µs 12% +PrintingCompose -> GatheredScan 183 99% 33.47µs 10% +StreamedSelect -> PrintingCompose 196 54% 19.34µs 6% +``` + +Ranking: + +1. **`printing_compose` acquire, 85% of all regret (~52.5ms).** Essentially every real routing loss in + the engine funnels here. Decomposes into the paging-branch table: `Perm` 63% share, `OrderbyWalk` + 12%, the rest low-severity default/decline buckets. + - `Perm`'s share (~38.9ms) is **already covered** by the separate, currently-active "Sigma" effort + (see Context above) — not re-examined in depth this round; recommend Round 27 not duplicate it. + - `OrderbyWalk`'s share (~7.4ms) is **already covered** by an existing, fully-designed-but-unshipped + fix — [local-engine-compose-paging-cost-based.md](local-engine-compose-paging-cost-based.md) + ("let compose choose OrderbyWalk vs Gather on cost, not on shape"). This round's feature-accuracy + pass (below) supplies a fresh, previously-missing quantification of part of why: `printings_walked` + is badly wrong specifically for card/artwork mode on this branch. + - `picked -> best` mismatches (`StreamedSelect <-> GatheredScan`, `GatheredScan <-> PrintingCompose`) + are the SAME phenomenon Rounds 1-9 repeatedly confirmed as "unchanged" in confirmation runs — never + the subject of their own investigation, always used as a regression check. **Already covered** in + substance (this is the Round 25-parked compound-existential-plane miscalibration showing up as real + regret); this round's contribution is the precise sizing (57% of ALL regret share alone, + `StreamedSelect -> GatheredScan`, ~34.9ms/mean 21.88µs/miss 66%) confirming it as by far the largest + single item, just one this effort has already tried and shelved four times. +2. **`candidates` acquire, 15% of all regret (~9.3ms).** **Already covered, still open** — Round 8's + mechanism 2 (card-mode residual-selectivity discount never shipped). See ranked candidates below. +3. Everything else (`printing_range_scan`, `plane`, `card_range_popcount`) — 0% share, confirmed + negligible. + +## Pairwise ordering + +`bench_pairwise_ordering.py --seconds 300 --seed 0`, realistic and uniform, by acquire branch. + +``` +pair / acquire ordered right mean regret +GatheredScan vs PrintingCompose [plane] realistic 85% 19.12µs uniform 82% 28.35µs +PrintingCompose vs StreamedSelect [plane] realistic 92% 11.26µs uniform 86% 16.55µs +GatheredScan vs StreamedSelect [printing_compose] realistic 92% 2.67µs uniform 92% 2.58µs +GatheredScan vs PrintingCompose [printing_compose] realistic 90% 3.03µs uniform 87% 5.12µs +PrintingCompose vs StreamedSelect [printing_compose] realistic 95% 1.84µs uniform 96% 1.76µs +GatheredScan vs PlanePopcountOrder [plane] realistic/uniform 100% 0.00-0.01µs +PlanePopcountOrder vs StreamedSelect [plane] realistic/uniform 100% 0.00µs +GatheredScan vs PrintingCompose [printing_range_scan] 99% 0.14-0.24µs n≈678-2077 +``` + +- **`[plane]` pairs are the worst-ordered in the whole engine (82-92%) but confirmed structurally + inert** — same reachability check Round 12/13 already did for `PrintingCompose`: `PlanePopcountOrder` + wins the real argmin under `Plane` acquire 100% of the time against both `GatheredScan` and + `StreamedSelect`, at 0.00-0.01µs regret, in both modes. **Already covered.** +- **`[printing_compose]` pairs are the real, reachable regret** — 87-96% ordered right, 1.76-5.12µs mean + regret, tens of thousands of rows. This is the reachable form of item 1 in the regret ranking above. + **Already covered** (the parked compound-existential-plane fit). +- **`[printing_range_scan]`** — checked for reachability per the task's own instruction (don't repeat + Round 12's mistake): tiny population (n≈678-2077), 98-99% ordered right, negligible mean regret. + **Not a candidate.** + +## Feature accuracy + +`bench_feature_accuracy.py --seconds 180 --mode uniform --seed 0`, 450,772 feature-rows. First use of +this tool this session. Ratio is feature/counter; >1 over-counts. + +``` +feature (pooled) n p10 p50 p90 p90/p10 +compose_scan_printings 1694 0.17 1.47 6.46 37.0 OVER-COUNTS +scan_units 135821 0.09 0.70 1.23 13.3 UNDER-COUNTS +printings_walked 44580 0.09 0.85 2.47 28.0 +matches 136656 0.99 1.00 2.36 2.4 +eval_domain 132021 1.00 1.00 2.16 2.2 + +printings_walked / card 3399 p50=0.23 UNDER-COUNT ~4.3x +printings_walked / artwork 3387 p50=0.22 UNDER-COUNT ~4.5x +printings_walked / card 10324 p50=0.96 +printings_walked / artwork 10441 p50=1.03 +printings_walked / printing 5796 p50=1.05 +``` + +- `scan_units` pooled under-count (median 0.70, wide spread) — **already covered**, the era-correlated + print-position confound for bare existential leaves (Round 17/20/25), plus the printing-varying range + depth work (Rounds 1-9). No new mechanism found here. +- `matches`/`eval_domain` pooled near-1.0 median with a fat right tail (p90 2.16-2.36, p99 10-13, + p100 up to 122) — **already covered**, the residual card-invariant/existential-arith-AND population + Rounds 15-25 partially fixed and Round 25 confirmed the remaining rate-fit still fails. +- **`printings_walked `, card/artwork mode: genuinely under-examined, but not + undiscovered.** Card and artwork mode read median 0.22-0.23 (real walk length ~4.3-4.5x the + predicted feature) while printing mode reads ~1.0-1.05. The ~4.3-4.5x gap matches the corpus's own + `printings_per_card`/`printings_per_artwork` constant almost exactly, and this is **exactly** the gap + [reference-engine-compose-perm-cards-visited-estimator.md](reference-engine-compose-perm-cards-visited-estimator.md) + names in its own "Next" section as still unresolved: "when [the cards_visited rate] does go in, + `OrderbyWalk`'s own accuracy needs re-checking in the same pass, since it shares `COMPOSE_WALK_STEP_NS` + with `Perm` and neither the kernel nor the reconciliation regression says anything about its + `cards_visited` shape (`resolutions`, a different quantity)." This round's number is the first + concrete measurement of that named-but-ungraded gap: **already covered by name, freshly sized here.** +- `compose_scan_printings` OVER-counts badly (median 1.47, p99 34x) under the compose `Gather` paging + branch specifically, small population (n≈1694, 671-831 in the finer slices). Cross-checked against + the regret matrix's compose-paging-branch table: `Gather`'s mean regret is only 0.18µs despite a huge + row count, i.e. low severity — the feature error is real but does not currently translate into much + real regret. **New, minor, not worth ranking above the items below.** + +## Ranked candidate list + +Genuinely open, unaddressed items — ranked by real regret share where measured this round. All of +these are already named somewhere in the doc tree (this repo has 25+ rounds of history); "candidate" +here means "not yet shipped, not yet deliberately parked as unproductive," not "undiscovered." + +1. **`printing_compose` acquire's `Perm`+`OrderbyWalk` paging-branch miscalibration — 63%+12% = 75% of + ALL measured routing regret (~46ms of 61.8ms).** By far the largest number in this whole sweep. Not + a recommendation for the domain-cards round-clock specifically: `Perm`'s share is under active work + by a separate, mature effort (6+ docs, 5 recent commits, "Sigma" decision rule just validated on + real-shaped traffic). `OrderbyWalk`'s share has a fully-designed, unshipped fix sitting in + [local-engine-compose-paging-cost-based.md](local-engine-compose-paging-cost-based.md), and this + round's feature-accuracy data adds the card/artwork-specific sizing that doc's own sibling + ([reference-engine-compose-perm-cards-visited-estimator.md](reference-engine-compose-perm-cards-visited-estimator.md)) + flagged as still needed. **Flagging for cross-effort awareness, not claiming as a domain-cards + finding** — if the round-clock issuing this brief has any flexibility to redirect effort, this is + where the real money still is, by an order of magnitude over anything below. +2. **`GatheredScan`/`candidates` residual-selectivity discount (Round 8's mechanism 2, never shipped) — + 15% of all regret (~9.3ms), cost-model-agreement FAIL (0.76 median, 30% within 25%, unchanged since + Round 9).** Squarely within the domain-cards/GatheredScan effort's own turf (`cost.rs`'s + `GatheredScan` arm, the same file Round 9 already touched for the zero-match fixed cost). Round 9 + fixed one of Round 8's two named mechanisms; this is the other one, still open, still real, and now + has a fresh regret-share number attached. **Top recommendation for Round 27** if it stays inside the + domain-cards effort's own scope, given item 1 is someone else's active turf and the compound- + existential-plane fit (below) is explicitly parked. +3. **The compound-existential-plane `GatheredScan` cost-formula miscalibration** (Round 25's parked + negative result) — sized precisely by this round's regret matrix at 57% of ALL regret share alone + just for the `StreamedSelect -> GatheredScan` mismatch (~34.9ms, mean 21.88µs, miss 66%), plus another + ~28% split across the other three compose-acquire mismatch directions. This is explicitly out of + scope per this round's own brief (four discarded fit attempts; needs a saturating/banded rate, not a + flat linear one) — listed here only so the size of the parked issue is visible against the rest of + this ranking, not as a live recommendation. +4. **`compose_scan_printings` over-count under the compose `Gather` paging branch** — real (median 1.47, + p99 34x) but low measured severity (Gather's mean regret is 0.18µs despite a large row count). Minor, + not worth a dedicated round on its own; worth a one-line note for whoever next touches + `COMPOSE_GATHER_SPAN_PER_MATCH`/`COMPOSE_GATHER_BITTEST_PER_PRINTING_NS`. + +## Explicitly not candidates + +Checked this round, confirmed already covered or immaterial — not to be re-discovered by a future +round: + +- **`GatheredScan`/`printing_compose` and `GatheredScan`/`card` cost-agreement cells** — same population + as candidate 3 above; the parked Round 25 negative result. +- **`GatheredScan`/`plane` and `PlanePopcountOrder`/`plane` cost-agreement FAILs** — confirmed + routing-inert via pairwise ordering (100% ordered right, 0.00-0.01µs regret against both competitors + under `Plane` acquire, both realistic and uniform mode). Extends Round 12/13's "`PrintingCompose`'s + absolute cost under `Plane` acquire doesn't matter" finding to the other two plans sharing that + branch — `PlanePopcountOrder`'s near-free popcount wins regardless of how any competitor is priced. +- **`PrintingCompose`/`plane`** — Round 12/13, structurally excluded from the `Plane`-acquire argmin. +- **`PrintingCompose`/`printing_range_scan`** — checked for reachability (per the task's explicit + instruction not to repeat Round 12's mistake): real but tiny population (n≈678-2077), 98-99% ordered + right, negligible mean regret. Not worth a round. +- **`scan_units` pooled under-count, `matches`/`eval_domain` fat right tail** — both already covered by + the Rounds 1-25 history (printing-varying range depth work, era-correlated print-position confound, + the parked existential-AND rate fit). +- **The `Or`/negation/nested-paren population** — Round 8 flagged this as invisible to + `bench_cost_model_agreement.py`'s flat-conjunction sampler and to every tool this round used (all four + draw from `QuerySampler`, same limitation). Still unmeasured by this round for the same reason; not + re-discovered, not newly sized either. + +## Reproducing + +```bash +maturin build --release --out /wheels # in card_engine/, isolated wheel, never maturin develop +PYTHONPATH=/wheels-extracted .venv/bin/python scripts/bench_cost_model_agreement.py \ + --seconds 300 --seed 0 --corpus benchmarks/bitplanes/corpus.jsonl --shm-path /store +# same pattern for bench_regret_matrix.py --seconds 180 --mode realistic, +# bench_pairwise_ordering.py --seconds 300 --mode realistic|uniform, +# bench_feature_accuracy.py --seconds 180 --mode uniform +``` diff --git a/scripts/costbench.py b/scripts/costbench.py index 3597d4db4..79ccdb3ac 100644 --- a/scripts/costbench.py +++ b/scripts/costbench.py @@ -144,6 +144,11 @@ def load_engine(corpus: pathlib.Path, shm_path: pathlib.Path) -> object: # `page_span * n_cards / matches`, which assumes matches are spread uniformly through the # permutation -- an assumption worth grading rather than trusting. "perm_steps", + # StreamedSelect's small-total branch only (0 for every other plan/exit): printings + # `push_card_matches` re-examined in the second, page-selecting pass over every matching + # card -- the redo `printings_examined` (captured only from the first, counting-only pass) + # structurally cannot see. Round 31 of the printing-varying-leaf depth ledger. + "redo_examined", "ns_setup", "ns_loop", "ns_finish",