diff --git a/.gitattributes b/.gitattributes index f9e8ef81..7d8066c4 100644 --- a/.gitattributes +++ b/.gitattributes @@ -35,3 +35,7 @@ power.csv binary thermals.csv binary gpu-nvml-clock.csv binary *.gz binary + +# Frozen benchmark extracts - SHA-pinned; bytes must survive verbatim +# (python csv.writer emits CRLF; the stamped sha256 covers those bytes). +benchmarks/qwen36-vs-qwen38-27b-2026-08/data/mmbt-frozen-dataset-v2.csv binary diff --git a/KNOWN-LIMITATIONS.md b/KNOWN-LIMITATIONS.md index 1bf428e3..c136a732 100644 --- a/KNOWN-LIMITATIONS.md +++ b/KNOWN-LIMITATIONS.md @@ -51,6 +51,24 @@ while byte-for-byte replay of every model turn requires the external archives. Source bench repo has full receipts + transcripts + workspaces for every attempted run (success and failure). MMBT publishes receipts + transcripts only for the single representative run per entry. The 5 failed runs across the local models (`27b_invest_memo_v3`, `27b_invest_memo_v4`, `coder_invest_memo_v6`, `coder_invest_memo_v7`, `n1_coder_v1`, `n1_coder_v3`, all three 27B PR-audit canonicals' "secondary" runs, and the 35B-A3B failure runs) have receipts and transcripts in the private bench repo but not here. A more rigorous audit of variance would need those. +### The Qwen3.6-vs-3.8 numbers come from a frozen extract, not a live re-scan + +Every headline figure in this entry is computed from a single frozen extract of the run corpus — `mmbt-frozen-dataset-v2.csv` (freeze #2), 802 cells, frozen `2026-08-16T14:23:09Z` — and **not** from a live re-scan of the five run checkouts. That is deliberate. The corpus was still accumulating cells while the freeze-1 analysis was underway, and live re-scans moved headline percentages by 4-9 points between passes purely because more cells had landed. Freeze #2 ends that: every campaign process is stopped, in-flight cells are quarantined out of the extract, and the on-disk verdict now agrees with the frozen verdict for all 733 graded rows (the overlay manifest's `post_freeze_divergence` ledger is empty). + +Two consequences a replicator should plan for. First, the frozen extract is the citable object: model and sampler identity in it were read from each cell's `receipt.json`, never inferred from directory names, and re-deriving identity from names will not reproduce the arm assignments. Second, the freeze boundary still hides real objects: quarantined in-flight cells are not rows in the extract at all — the clearest instance is `p3_doc_qwen38q8-nothink-matched_v2`, a Q8_0 run quarantined mid-flight in a rewrite loop (139 iterations rewriting `brief.md`, context grown to ~228k tokens), which this entry cites only as unscored qualitative evidence and excludes from every rate. A reader who finds such directories on disk has found something real that the extract deliberately does not count. (The freeze-1 version of this entry disclosed 7 unreconciled Q8_0 grades; freeze #2 admits them — the Q8_0 arm now carries 8 graded cells — so that particular discrepancy no longer exists.) + +### Replicate depth is unbalanced across the compared arms + +The arms are not equal-sized, and the imbalance runs the same direction as the headline. Per family, Qwen3.6 no-think at `T0.3/p0.8/pp0` carries 9-19 replicates, Qwen3.8 at the same sampler 7-13; in think mode at that sampler Qwen3.6 carries 9 per family against Qwen3.8's 4-6. At the vendor points the gap is smaller but present (10 per family for 3.6, 8 for 3.8). Aggregate arm sizes are 121 vs 95 (no-think matched), 120 vs 96 (no-think vendor), 108 vs 51 and 120 vs 72 (think). + +This means pooled percentages weight the two models' family mixes differently, and a family where one model happens to have twice the replicates pulls the pooled number toward that family's behavior. Per-family rates with their own denominators are printed in `benchmarks/qwen36-vs-qwen38-27b-2026-08/findings.md` for exactly this reason. **Do not compute a pooled model-level rate from this data without checking whether the family weighting is doing the work.** Confidence intervals are correspondingly wide: several per-family cells rest on 4-6 runs. + +### Replicates share a seed and are not fully independent draws + +Every one of the 802 cells ran at `seed=42`. Sampling temperature breaks strict determinism, so replicates do differ — of the duplicate-length sibling pairs we checked, **none** had byte-identical transcripts (8 of 8 sets checked, all distinct SHA-256). But the replicates are visibly correlated rather than independent. Within same-arm same-family replicate sets, the share of cells landing on an exactly identical total completion-token count as a sibling is 5.4% (17/316) at `T0.3/p0.8/pp0`, 3.6% (10/276) at `T1/p0.95/pp0`, and 0% (0/76) at `T0.7/p0.8/pp1.5`, the one sampler carrying a presence penalty. Concrete example: five `p2_extract` replicates in one Qwen3.6 arm all finished at exactly 1,515 completion tokens. + +The honest reading is narrower than "the runs are near-deterministic": they are not, and the transcripts prove it. The reading that survives is that **effective n is somewhat below nominal n at the low-temperature samplers**, that the correlation is not uniform across samplers, and that this entry does not quantify how much statistical power is lost. Varying the seed across replicates is the obvious fix and was not done here. + ## Methodology caveats ### No formal scoring rubric @@ -109,6 +127,26 @@ marathon reliability. Tasks that reference real public state (DreamServer PRs, SEC filings, market prices) will see that state drift over time. The DreamServer PR audit task pins to a specific baseline commit (`d5154c37...`) but PR comments accumulate, contributors close PRs, the issue tracker moves. The wallstreet task has no such anchor — the company-pick is the agent's decision and the analyzed material may have been updated since extraction. Take time-of-run into account when comparing across replicates. +### No thinking-mode arm exists at Qwen3.8's own vendor sampler + +This is the largest hole in the comparison and it cannot be patched by re-analysis. Qwen3.8's vendor sampling point is `T0.7/p0.8/pp1.5`; the corpus contains **zero** Qwen3.8 think cells there. Its think arms exist only at `T0.3/p0.8/pp0` (51 cells) and `T1/p0.95/pp0` (72 cells), and `T1/p0.95/pp0` is *Qwen3.6's* vendor point. So the only sampler at which both models have think cells is one model's home turf, and no run in this repository shows Qwen3.8 thinking at the settings its own model card recommends. + +Any cross-model thinking comparison drawn from this entry is therefore both-models-at-3.6's-settings, and this repo declines to label it otherwise. The gap compounds with a second asymmetry: every Qwen3.8 think cell carries a `reasoning_effort` value, so those arms are mixtures of `low`/`medium`/`xhigh` (14/24/13 at the matched sampler, 12/12/48 at `T1`), while Qwen3.6's shipped chat template contains no `reasoning_effort` variable at all and its think arm is a single configuration. **A mixture compared against a point is not a like-for-like comparison**, whatever the sampler. Running Qwen3.8 think at `T0.7/p0.8/pp1.5` at a fixed effort level is the single highest-value follow-up to this entry. + +### No phase-3 grader checks fabrication, factual accuracy, or quality — and the hand-rating slots are empty + +Every phase-3 grader in `tooling/graders/` is a keyword-recall and word-count instrument. `phase3_project_mgmt_grade.py` asks whether literal strings appear; `phase3_doc_synthesis_grade.py` and `phase3_business_memo_grade.py` count matched fact keywords against a threshold and check a word ceiling; `phase3_writing_editing_grade.py` checks must-include and must-not-include strings plus a per-audience ceiling; `phase3_market_research_grade.py` emits `STRUCTURAL_PASS`/`STRUCTURAL_FAIL` on counts of products named and URLs cited. **None of them reads the deliverable for whether it is true.** A confidently fabricated status report that uses the right vocabulary and lands under 700 words passes every automated gate in this entry. + +The graders were designed knowing this — they carry `hand_rating_placeholders` blocks for exactly the dimensions the automation cannot reach (fabrication count, faithfulness, citation validity, stance and structure quality). Those blocks are empty corpus-wide. Of the 733 `grade.json` files read (one per frozen graded cell), 441 have no placeholder block at all (the phase-1 and phase-2 graders), 292 carry one, and **every numeric or free-text rating field in all 292 is null**. The only non-null entries anywhere are 46 `p3_market` cells carrying a boilerplate `_HAND_VERIFICATION_REQUIRED_` note that says the structural pass is necessary but not sufficient — a marker that the work is outstanding, not a rating. + +So: read every phase-3 result in this entry as *"produced an artifact with the expected shape and vocabulary."* It is not a quality score, and the corrected pass rates published here inherit that limit in full. The 39 `p3_market` `STRUCTURAL_PASS` verdicts in particular assert structure only and prove no citation. + +### `p3_market` is a live-internet task and its results are not replayable + +The `p3_market` family sends the agent to the live public web. Its inputs are whatever the internet returned at run time, so unlike the other eleven families it has no fixed fixture and no possibility of a byte-identical replay. Cells run on different days saw different pages; cells run against sources that have since changed cannot be re-graded against what the model actually read. + +It is also the family most distorted by the run-length pathologies described in this entry — of its 64 cells, 18 are loop-labelled and only 46 are graded, the worst delivery of any family — which means its aggregate is computed on a smaller and differently-selected subset than its neighbors. Treat `p3_market` as a qualitative probe. **Do not include it in cross-model aggregates without saying so**, and do not read a difference in its pass counts as a difference in market-research ability. + ## Hardware and platform caveats ### Single-workstation hardware specificity @@ -137,6 +175,12 @@ What this means for the data here: The FP8 re-run of the same 12-cell grid is the highest-priority follow-up — see [`ROADMAP.md`](ROADMAP.md). Contributors with FP8-capable hardware are welcome to PR results via the [`tooling/ADDING-A-MODEL.md`](tooling/ADDING-A-MODEL.md) flow (which now explicitly covers the "same model, different quant" contribution path). +### Quantization specificity — the Qwen3.6-vs-3.8 comparison does not settle the quant question + +The head-to-head runs entirely on Unsloth UD-Q4_K_XL GGUFs for both models. The one quantization control in the corpus is a 19-cell Qwen3.8 Q8_0 arm at the matched sampler (2 replicates on the 7 phase-1/phase-2 families, 1 on the 5 phase-3 families), and at freeze #2 it establishes one thing at provisional-rate strength: the identical-call-loop failure shape occurs at Q8_0 at a real, non-negligible rate — 6 of 19 cells trip `looped_freq30` (31.6%, Wilson 95% CI [15.4%, 54.0%], excluding zero), with maximum identical-call runs of 110, 109, 81, 80 and 71. Against 3.8 at UD-Q4_K_XL, same sampler, same 12 families, the loop rate is statistically indistinguishable (29/95, 30.5%, vs 6/19; Fisher p = 1.0). **The loop is not an artifact of UD-Q4_K_XL alone.** A quarantined in-flight Q8_0 cell (`p3_doc_qwen38q8-nothink-matched_v2`, 139 iterations rewriting `brief.md`, ~228k context) additionally shows a rewrite-loop subclass at Q8_0 — unscored qualitative evidence only, excluded from every rate. + +What the control cannot do is still more important than what it can. The graded subset is 8 of 19 cells (5 PASS, 3 FAIL; 62.5%, Wilson [30.6%, 86.3%]) — published as provisional data, and far too thin for any quality verdict: the vs-Q4 graded-only contrast (78.5% vs 62.5%, Fisher p = 0.38) fails this PR's own power screen. And there is **no matched Qwen3.6 Q8_0 arm at all**, so this is not a quant A/B: it can neither attribute the Qwen3.8 delivery regression to quantization nor rule quantization out. Whether Qwen3.8's no-think delivery gap narrows, holds, or disappears at Q8_0, FP8, or BF16 is open, and a Q8_0 arm at N>=5 per family for *both* models — graded — is the experiment that would close it. + ### Cloud-LLM hardware is different Cloud entries (`Opus-4.7/`, `GPT-5.5/`) ran on the providers' inference infrastructure, not Tower2. Cross-comparison should account for that — "the cloud LLM is better" partly reflects "different hardware + different quantization-strategy + different inference engine," not just model differences. diff --git a/benchmarks/findings-index.md b/benchmarks/findings-index.md index 5e5afd53..ab647a4b 100644 --- a/benchmarks/findings-index.md +++ b/benchmarks/findings-index.md @@ -4,6 +4,7 @@ Cross-cutting analyses that synthesize across multiple model entries or benchmar | Doc | Date | One-line summary | |---|---|---| +| [`qwen36-vs-qwen38-27b-2026-08/findings.md`](qwen36-vs-qwen38-27b-2026-08/findings.md) | 2026-08-16 | Qwen3.6-27B vs Qwen3.8-27B head-to-head at UD-Q4_K_XL (802 frozen cells + 19-cell Q8_0 control). The one large effect is a no-think delivery regression (+28-29 pp identical-call-loop rate, p <= 4.9e-09), not a capability gap - corrected pass-given-delivery is within noise at the matched sampler. Three MMBT grader defects found and corrected via non-destructive overlay; five earlier conclusions retracted in `claims.yaml`. | | [`deepseek-v4-flash-0731/DEEPSEEK_V4_FLASH_0731_VERIFIED_RESULTS.md`](deepseek-v4-flash-0731/DEEPSEEK_V4_FLASH_0731_VERIFIED_RESULTS.md) | 2026-08-01 | Fully optimized DeepSeek V4 Flash campaign: canonical 35/36 corrected, complete N=3 single-PR runs, strict finance and deck audits, and 0/3 full-context frozen 75-PR outcomes including an 815,279-token terminal runaway. | | [`microbench-phase-b-2026-05-02/findings.md`](microbench-phase-b-2026-05-02/findings.md) | 2026-05-02 | N=10 expansion + 27B-no-think third arm. Three identical-call-loop subclasses (`scroll-loop`, `word-trim-loop`, `rewrite-loop`). Per-cell ship rates with Wilson CIs, cost-per-shipped-run, "when to use which" updates. | | [`microbench-phase-b-2026-05-02/findings-pairwise-quality-three-model.md`](microbench-phase-b-2026-05-02/findings-pairwise-quality-three-model.md) | 2026-05-03 | Hand-graded deliverable quality study on the both-ship cells (p2_ci, p2_extract, p2_triage). Headline: 27B-thinking and 27B-no-think substantively equivalent on output decisions; Coder-Next has a distinct reasoning style. **Includes a load-bearing correction to the 2026-04-28 study's `p2_ci` regression attribution.** | diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/AUDIT.md b/benchmarks/qwen36-vs-qwen38-27b-2026-08/AUDIT.md new file mode 100644 index 00000000..3d328154 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/AUDIT.md @@ -0,0 +1,235 @@ +> [!IMPORTANT] +> **EXPLORATORY — notice added 2026-08-16 per the corrective protocol, section 9. No number +> or audit statement below has been altered; this notice is prepended only.** +> +> 1. **Arm mislabel (V36 vendor no-think).** The arm this bundle labels "3.6 vendor +> no-think" ran Qwen3.6-27B at its **thinking** sampler (T1.0 / top_p 0.95 / +> presence_penalty 0) with thinking disabled — off-spec per the pinned Qwen3.6-27B card +> (rev `6a9e13bd`, README L637: official non-thinking is T0.7 / top_p 0.80 / top_k 20 / +> min_p 0 / pp 1.5 / rp 1, identical to Qwen3.8-27B). The "vendor vs vendor" no-think +> contrast is mode/sampler-confounded, and the corpus contains **no cell of Qwen3.6 +> non-thinking at its official sampler**. +> 2. **Cell-level Fisher exact values in this bundle are descriptive only.** They treat +> cells as independent and are anti-conservative under family/seed clustering. +> 3. **The 802-cell freeze audited here is exploratory evidence** and feeds no confirmatory +> claim. The confirmatory design, arms, fixed N, and analysis plan are preregistered at +> [`../qwen36-vs-qwen38-corrective-2026-08/PREREGISTRATION.md`](../qwen36-vs-qwen38-corrective-2026-08/PREREGISTRATION.md); +> the pinned-card sampler evidence is recorded at +> [`../qwen36-vs-qwen38-corrective-2026-08/protocol/CARD-EVIDENCE.md`](../qwen36-vs-qwen38-corrective-2026-08/protocol/CARD-EVIDENCE.md). + +# AUDIT — Qwen3.6-27B vs Qwen3.8-27B agentic comparison (2026-08-16) + +This bundle compares two model generations on the 12-family MMBT microbench. The headline is a +**delivery-reliability** finding, not a capability finding, and the distinction is load-bearing: +the two models are statistically indistinguishable on the work they deliver once three verified +grader defects are corrected. This document states exactly what is locked, what varies, what the +comparison can and cannot support, and every bias we know of — including the ones that cut +against our own headline. + +**Frozen snapshot: `2026-08-16T14:23:09Z` (freeze #2).** Every number in this bundle derives +from `data/mmbt-frozen-dataset-v2.csv` (802 cells) and nothing else. All campaign processes were +stopped and in-flight cells quarantined *before* this freeze was taken, so the corpus is static. +An earlier freeze (746 cells, 11:49:14Z) is superseded; during the live-corpus phase of the +investigation, figures drifted 4–9 pp between recomputations — that is why the freeze discipline +exists, and readers should treat any number without this stamp as stale. + +--- + +## What is locked across both models + +- **Task definitions.** Exactly one task sha256 per family across all five checkouts, 12/12 + families. Neither model saw a different brief. +- **Harness.** `harness.py` file_sha256 `b6656a23737ef89e…` on 800 of 813 receipts corpus-wide; + the 13 non-canonical receipts are early bring-up and excluded-arm cells (2 on the previously + disclosed `fdbc1584074f36d9…`, 11 across five minor variants). Same agent loop, same tool + surface, same stuck detector across every headline comparison. +- **Graders.** All 11 grader `.py` files byte-identical across all five checkouts + (e.g. `phase3_business_memo_grade.py` md5 `9b94cfcd09aa7bbed0e03a6e5cc600a3`). All 7 + `ground_truth/*.json` byte-identical. Zero model-conditional or arm-conditional logic, verified + by three independent scans with a positive control proving the files were actually read. + Graders are stdlib-only with hardcoded thresholds and open fixed relative filenames, so they + cannot branch on arm implicitly. +- **Grader provenance at run time.** All post-fix `receipt.json` files record a single + `grade_microbench_sha256` across both models. +- **Engine.** Same llama.cpp container image digest `sha256:0c8dc7c0954f…`, same + `/app/llama-server` entrypoint, same flags (`--n-gpu-layers 999 --ctx-size 262144 + --batch-size 2048 --threads 4 --parallel 1 --flash-attn on --cache-type-k q8_0 + --cache-type-v q8_0 --jinja --reasoning-format none --no-context-shift --metrics`). Only + `--model` and `--alias` differ between lanes. +- **Generation caps.** `max_iters = 10000` and `max_completion_total_default = 1e12` for every + cell of both models. The longest observed cell is 565 tool calls. **Nothing was truncated by a + cap**, so no result here is an artifact of an artificial ceiling. +- **Seed.** `seed = 42` on every cell. +- **Model attribution.** Taken per cell from the live endpoint capture in `receipt.json`, never + from directory names. Zero mismatches against arm label across all in-scope cells. This matters + because one arm named `…-offspec` was found to actually carry the vendor-card sampler. + +--- + +## What varies + +| axis | Qwen3.6 | Qwen3.8 | why / status | +|---|---|---|---| +| **Weights** | `Qwen3.6-27B-UD-Q4_K_XL` | `Qwen3.8-27B-UD-Q4_K_XL` | the intended variable | +| **Quantization grade** | ftype `Q4_K - Medium` | ftype **`Q4_K - Small`** | **NOT MATCHED — see B2** | +| **Vendor sampler** | T1.0 / top_p 0.95 / pp 0.0 | T0.7 / top_p 0.8 / **pp 1.5** | each model's own published point | +| **Matched sampler** | T0.3 / top_p 0.8 / pp 0.0 | identical | one distinct config block across all cells in this regime | +| **Thinking control** | binary `enable_thinking` | `reasoning_effort` ∈ {xhigh (default), medium, low} | genuinely new in 3.8; `high` is an alias for `xhigh` (verified byte-identical at seed 42) | +| **Replicate depth** | 9–19 per family per regime | 7–13 (matched) / 8 (vendor) | **NOT MATCHED — see B4** | +| **Host** | tower3 / tower2-gpu1 | tower1 / tower2-gpu0 | **confounded in the card regime — see B3** | + +--- + +## What this bundle CAN support + +- **A non-thinking-mode delivery-reliability regression in Qwen3.8 relative to Qwen3.6**, at both + each model's own vendor sampler and at a byte-identical matched sampler, driven by + byte-identical tool-call repetition loops concentrated in long-horizon coding families. +- **That the loop is not an artifact of the harness.** The container-death signature + (`wall_s ≤ 0.05 ∧ 130 ≤ result_len ≤ 200`) returns **0 hits** across all flagged cells while + firing 45–502× per cell on a 20-cell positive control of known container deaths. An independent + channel using neither timing nor size agrees: the known-artifact cells repeat trivial probes + (`echo hi`, `sleep 3600` returning 148 bytes in 0.01 s) while flagged cells repeat substantive + work at 0.06–8.63 s. +- **That the loop is not an artifact of quantization — as a provisional rate.** At Q8_0 + (near-lossless) the loop rate is 6/19 = 31.6% (Wilson [15.4, 54.0], excluding zero) against + the Q4 build's 29/95 = 30.5% — Fisher exact p = 1.0, the two builds indistinguishable — with + the loops landing in the same families. See B2 for the residual limits. +- **That, conditional on delivering, the two models are indistinguishable** once D1–D3 are + corrected. +- **A cost ordering within Qwen3.8's reasoning-effort ladder**, including its non-monotonicity. + +## What this bundle CANNOT support + +- **Any model-family generality.** One quant per model, no crossover build, no perplexity/KL + fidelity check. Claims are about **these specific GGUF builds**. +- **Any causal account of the loop.** Tool result **bodies are never stored** — only + `result_len`. Nothing here can establish whether the repeated command was erroring, returning + empty, or succeeding. Any transcript grep for error text is a **null test** that returns + nothing regardless of truth; it fires on 1 of 20 known container deaths. +- **Anything about Qwen3.8 thinking at Qwen3.8's own vendor sampler.** Every 3.8 thinking arm in + this corpus ran at T1.0/0.95/pp0.0 — **Qwen3.6's** vendor point. No such run exists. Any + "vendor default thinking" comparison is unsupported. +- **Output quality in any deep sense.** No phase-3 grader contains a fabrication, correctness or + quality check; `hand_rating_placeholders` is `null` corpus-wide. +- **Generality to other quants, serving stacks, harnesses, or task sets.** +- **Multi-user serving.** Every lane ran `--parallel 1`. + +--- + +## Bias list + +### B1. The operator abort label is not a consistent rule — and we stopped using it +An earlier version of this finding was built on `label.json`, the operator-terminated pathology +marker. It is not consistently applied in either direction: one cell repeated a command **250 +times** and was graded **PASS**, never aborted; another was aborted at a maximum repeat frequency +of **1**. Whether a run lands in the abort bucket partly measures operator attention. +**Mitigation:** the published metric is derived from `transcript.jsonl` alone and never reads +`label.json`. The abort-rate framing is formally retracted in `claims.yaml`. + +### B2. Quantization is not matched between the models — but the Q8_0 control closes the main question +3.8 is `Q4_K - Small`; 3.6 is `Q4_K - Medium`, so quant damage could not be separated from model +behaviour by the Q4 arms alone. The Q8_0 control was run for exactly this reason and, at +freeze #2, it carries **19 cells across 12 families (8 graded: 5 PASS / 3 FAIL)** with a loop +rate of **6/19 = 31.6%** — statistically indistinguishable from the Q4 build (29/95 = 30.5%, +Fisher p = 1.0) and landing in the same families. Near-lossless quantization does not remove the +regression: **it is the model, not the build.** Residual limits, disclosed rather than waved +away: the arm is thin (per-family n ≈ 1–2), its graded-quality comparison is underpowered +(5/8 vs Q4, Fisher p = 0.38), no perplexity/KL fidelity check was run on either build, and both +GGUFs come from a single quantizer (unsloth). One additional Q8_0 cell — a 139-iteration +rewrite-loop that grew its context to ~228k — was quarantined in-flight and is reported as +**unscored** qualitative evidence only. + +### B3. Host is confounded with model in the card regime +All `qwen38-nothink-card` cells ran on tower1; all `qwen36-nothink-card` and `qwen36-think-card` +cells on tower3. Zero crossover inside the scored arms. **Partially broken by out-of-scope +evidence:** an excluded arm at the same sampler ran cells on *both* hosts and flagged on both, +including on the identical GPU UUID that served every 3.6 card cell at zero flags. The effect +follows the weights across machines. The matched-sampler regime is not host-confounded in the +same way. + +### B4. Replicate depth is unbalanced by design +3.6 reached 9–19 replicates per family per regime; 3.8 is at 7–13 (matched) / 8 (vendor), and +the thinking arms are thinner and effort-mixed (see the group inventory). Extra 3.6 replicates +can only tighten 3.6's interval — they cannot manufacture the gap — but every headline is +additionally reported **depth-matched** so a reader does not have to take that on faith. + +### B5. The matched-sampler regime is not symmetric in what it costs each model +Running both models at pp 0.0 strips Qwen3.8 of the `presence_penalty 1.5` its model card +specifies (an anti-repetition measure) while costing Qwen3.6 nothing, since its vendor pp is +already 0.0. This regime is **not neutral**. It does not rescue 3.8 — the regression is present +at 3.8's own vendor sampler with pp 1.5 applied, and the matched pair (pp 0 on both sides) +still shows a ~+28 pp loop delta — but the asymmetry must be disclosed, and the missing 2×2 +cell (3.6 at pp 1.5) has not been run. + +### B6. seed=42 induces replicate correlation — not determinism — and the power loss is unquantified +No two same-arm same-family replicates are byte-identical (checked by SHA-256 on transcripts; +llama.cpp continuous batching breaks determinism), and verdicts flip within replicate groups. +What does hold is correlation: 5.4% of replicated cells at T0.3 (17/316) share an exact +completion-token count with a sibling, 3.6% at T1 (10/276), 0/76 at the 3.8 vendor point. +Effective n is therefore somewhat below nominal n, by an amount this bundle does not quantify; +per-family fractions in the matched regime should be read as texture, not rates. + +### B7. Three grader defects, all corrected non-destructively, all in 3.8's favour when corrected +See `grader-defects.md`. Summary: a word-count gate whose tokenizer disagrees with the counter +both models demonstrably budget against (the correction *invalidates the gate* — recorded as +`gate_invalidated`, not as a verified PASS); a `p3_pm` risk-recall gate whose applied fix is the +repo's own R3 adjacency rule, with the contraction pair as the decisive natural experiment; and +a `p2_triage` ground truth that contradicts its own brief and penalised **64/64 graded cells of +both models**. All corrections are strictly leniency — no cell flips from PASS to FAIL under any +of them. **Disclosure:** the repo already contained an unapplied fix for the second +(`tooling/correct_gemma4_project_mgmt_grades.py`, unit-tested, hardcoded to a different model's +cell names). A reviewer will find it, so we say it here. Post-correction, `p2_triage` (64/64) +and `p3_pm` (61/64) saturate and carry **no discriminating weight** in any headline. + +### B8. An interim arm whitelist dropped cells anti-conservatively — resolved by inclusion +Interim analyses during the investigation hardcoded a set of arm labels and skipped 95 +non-quarantined cells; the exclusion worked *against* the eventual headline (the skipped clean +cells add flagged 3.8 cells and zero flagged 3.6 cells). The published pipeline resolves this: +the group inventory in `results-tables.md` **pools every arm at its receipt-derived sampler +point**, including the formerly skipped ones, and lists the pooled arm labels per group so the +numbers are reproducible from the stated composition. The interim exclusion is disclosed here +because earlier circulated figures were computed under it. + +### B9. `p3_market` is a live-internet task +Arms ran on different days against changing external endpoints. Any `p3_market` result carries an +uncontrolled network confound. One of Qwen3.6's three no-think loops is a `curl` retry loop +against an external pricing endpoint — a network-retry loop, not a reasoning loop. + +### B10. The container-artifact era, and why the time gate was retired +A harness bug removed sandbox containers mid-run and was misread for hours as model behaviour; +it is fixed (cleanup scoped by campaign label). Analysis was initially gated to transcripts newer +than the fix. That gate was **retired** after a retrospective scan applied the real signature to +all 159 surviving pre-fix cells and found **zero** contaminated — the damaged cells had already +been quarantined. Two container deaths were later found to have occurred *after* the fix and were +caught by operator quarantine, so **"post-fix therefore clean" is false as a rule**: cells are +clean because they were tested, not because of their timestamp. + +### B11. Fifteen loop-killed cells left recoverable workspaces — unresolved +SIGTERM skips the final `tar czf` but leaves the host workspace; at freeze #2, 15 of the 64 +operator-labelled cells have one. A probe graded a subset and reported four PASSes, three of +them 3.8 cells currently counted as all-cells failures; recovering them would move 3.8's matched +all-cells rate by several pp. This did **not** survive verification as stated and is **open**. +It is the live objection that attacks the strongest surviving finding most directly, and it is +listed as future work rather than quietly resolved in our favour. + +--- + +## Reproducing + +From this entry directory (`benchmarks/qwen36-vs-qwen38-27b-2026-08/`): + +```bash +python3 tooling/test_stats.py # validate the estimator implementations +python3 tooling/mmbt_results.py # regenerate results.json from data/mmbt-frozen-dataset-v2.csv +python3 tooling/make_md.py # render results-tables.md (byte-identical rebuild) +python3 tooling/apply_grade_corrections.py --dry-run # D1-D3 overlay summary, writes nothing +``` + +Regeneration is deterministic: the committed `results.json` and `results-tables.md` rebuild +byte-identically from the committed CSV + overlay (modulo three absolute-path provenance +strings; see the PR notes). `tooling/freeze_dataset.py` documents how the CSV was derived from +the raw corpus. Raw per-run logs and agent workspaces are **not published** (`/logs/` is +gitignored; the workspace tarballs alone are ~1.5 GB). The frozen dataset carries one row per +cell with the outcome, sampler, repetition metrics and token counts needed to re-derive every +published number. Anyone wanting the raw corpus should open an issue. diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/data/FREEZE2_STAMP.txt b/benchmarks/qwen36-vs-qwen38-27b-2026-08/data/FREEZE2_STAMP.txt new file mode 100644 index 00000000..d44103ea --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/data/FREEZE2_STAMP.txt @@ -0,0 +1 @@ +FREEZE2 STAMP: 2026-08-16T14:23:09Z diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/data/mmbt-frozen-dataset-v2.csv b/benchmarks/qwen36-vs-qwen38-27b-2026-08/data/mmbt-frozen-dataset-v2.csv new file mode 100644 index 00000000..33e9c162 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/data/mmbt-frozen-dataset-v2.csv @@ -0,0 +1,803 @@ +cell,repo,family,arm,replicate,model,quant,mode,effort,regime,temperature,top_p,top_k,presence_penalty,seed,verdict,graded,passed,terminal,looped_freq30,looped_run30,max_freq,max_run,n_tools,distinct_ratio,completion_tokens,elapsed_s,label_primary,container_death_signature,transcript_mtime +p1_bugfix_qwen36-27b-udq4xl-t03-nothink_v1,mmbt-qwen36-compare,p1_bugfix,qwen36-27b-udq4xl-t03-nothink,1,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,4,1,124,0.9274,29520,869.2,,0,1786804304 +p1_bugfix_qwen36-nothink-offspec_v1,mmbt-qwen36-compare,p1_bugfix,qwen36-nothink-offspec,1,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,4,1,137,0.9489,35270,980.9,,0,1786809082 +p1_bugfix_qwen36-27b-udq4xl-t03-nothink_v2,mmbt-qwen36-compare,p1_bugfix,qwen36-27b-udq4xl-t03-nothink,2,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,5,1,123,0.9024,28660,840.0,,0,1786804275 +p1_bugfix_qwen36-nothink-offspec_v2,mmbt-qwen36-compare,p1_bugfix,qwen36-nothink-offspec,2,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,128,0.9609,33429,931.1,,0,1786810015 +p1_bugfix_qwen36-27b-udq4xl-t03-nothink_v3,mmbt-qwen36-compare,p1_bugfix,qwen36-27b-udq4xl-t03-nothink,3,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,5,1,114,0.9123,25358,610.3,,0,1786804917 +p1_bugfix_qwen36-nothink-offspec_v3,mmbt-qwen36-compare,p1_bugfix,qwen36-nothink-offspec,3,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,7,1,137,0.9197,33470,954.9,,0,1786811523 +p1_bugfix_qwen36-27b-udq4xl-t03-nothink_v4,mmbt-qwen36-compare,p1_bugfix,qwen36-27b-udq4xl-t03-nothink,4,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,4,1,126,0.9286,27128,818.0,,0,1786805094 +p1_bugfix_qwen36-nothink-offspec_v4,mmbt-qwen36-compare,p1_bugfix,qwen36-nothink-offspec,4,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,3,1,107,0.9439,25982,784.5,,0,1786812309 +p1_bugfix_qwen36-27b-udq4xl-t03-nothink_v5,mmbt-qwen36-compare,p1_bugfix,qwen36-27b-udq4xl-t03-nothink,5,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,8,1,157,0.879,35714,821.8,,0,1786805741 +p1_bugfix_qwen36-nothink-offspec_v5,mmbt-qwen36-compare,p1_bugfix,qwen36-nothink-offspec,5,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,6,1,127,0.8898,25976,807.7,,0,1786813120 +p1_bugfix_qwen36-27b-udq4xl-t03-nothink_v6,mmbt-qwen36-compare,p1_bugfix,qwen36-27b-udq4xl-t03-nothink,6,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,5,1,172,0.9419,39223,1075.8,,0,1786806172 +p1_bugfix_qwen36-nothink-offspec_v6,mmbt-qwen36-compare,p1_bugfix,qwen36-nothink-offspec,6,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,3,1,153,0.9346,38884,1045.6,,0,1786814166 +p1_bugfix_qwen36-27b-udq4xl-t03-nothink_v7,mmbt-qwen36-compare,p1_bugfix,qwen36-27b-udq4xl-t03-nothink,7,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,5,1,153,0.9216,28805,568.6,,0,1786806312 +p1_bugfix_qwen36-nothink-offspec_v7,mmbt-qwen36-compare,p1_bugfix,qwen36-nothink-offspec,7,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,6,1,121,0.8843,31191,892.9,,0,1786815061 +p1_bugfix_qwen36-27b-udq4xl-t03-nothink_v8,mmbt-qwen36-compare,p1_bugfix,qwen36-27b-udq4xl-t03-nothink,8,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,5,1,126,0.9206,29260,1269.7,,0,1786807699 +p1_bugfix_qwen36-nothink-offspec_v8,mmbt-qwen36-compare,p1_bugfix,qwen36-nothink-offspec,8,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,6,1,142,0.9085,33068,948.0,,0,1786816011 +p1_bugfix_qwen36-27b-udq4xl-t03-nothink_v9,mmbt-qwen36-compare,p1_bugfix,qwen36-27b-udq4xl-t03-nothink,9,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,5,1,138,0.9203,31469,944.8,,0,1786807374 +p1_bugfix_qwen36-nothink-offspec_v9,mmbt-qwen36-compare,p1_bugfix,qwen36-nothink-offspec,9,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,7,1,132,0.9015,31175,895.6,,0,1786869948 +p1_bugfix_qwen36-27b-udq4xl-t03-nothink_v10,mmbt-qwen36-compare,p1_bugfix,qwen36-27b-udq4xl-t03-nothink,10,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,0,0,0,3,1,90,0.7667,,,,0,1786808036 +p1_refactor_qwen36-nothink-offspec_v1,mmbt-qwen36-compare,p1_refactor,qwen36-nothink-offspec,1,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,3,1,96,0.9688,11734,228.6,,0,1786817854 +p1_refactor_qwen36-nothink-offspec_v2,mmbt-qwen36-compare,p1_refactor,qwen36-nothink-offspec,2,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,83,0.988,11335,212.7,,0,1786827853 +p1_refactor_qwen36-nothink-offspec_v3,mmbt-qwen36-compare,p1_refactor,qwen36-nothink-offspec,3,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,96,0.9896,11748,227.1,,0,1786834186 +p1_refactor_qwen36-nothink-offspec_v4,mmbt-qwen36-compare,p1_refactor,qwen36-nothink-offspec,4,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,3,2,79,0.9494,9768,186.2,,0,1786839678 +p1_refactor_qwen36-nothink-offspec_v5,mmbt-qwen36-compare,p1_refactor,qwen36-nothink-offspec,5,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,77,0.9481,9816,181.9,,0,1786846206 +p1_refactor_qwen36-nothink-offspec_v6,mmbt-qwen36-compare,p1_refactor,qwen36-nothink-offspec,6,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,3,1,80,0.9625,9854,188.4,,0,1786852051 +p1_refactor_qwen36-nothink-offspec_v7,mmbt-qwen36-compare,p1_refactor,qwen36-nothink-offspec,7,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,88,0.9545,11033,214.4,,0,1786857826 +p1_refactor_qwen36-nothink-offspec_v8,mmbt-qwen36-compare,p1_refactor,qwen36-nothink-offspec,8,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,81,0.9753,9808,185.6,,0,1786864012 +p1_refactor_qwen36-nothink-offspec_v9,mmbt-qwen36-compare,p1_refactor,qwen36-nothink-offspec,9,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,3,1,88,0.9545,11314,212.9,,0,1786870849 +p1_testwrite_qwen36-nothink-offspec_v1,mmbt-qwen36-compare,p1_testwrite,qwen36-nothink-offspec,1,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,7,1,68,0.8971,37072,645.8,,0,1786817625 +p1_testwrite_qwen36-nothink-offspec_v2,mmbt-qwen36-compare,p1_testwrite,qwen36-nothink-offspec,2,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,8,1,87,0.908,63649,1154.9,,0,1786827639 +p1_testwrite_qwen36-27b-udq4xl-t03-nothink_v3,mmbt-qwen36-compare,p1_testwrite,qwen36-27b-udq4xl-t03-nothink,3,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,4,1,87,0.908,33845,605.6,,0,1786807982 +p1_testwrite_qwen36-nothink-offspec_v3,mmbt-qwen36-compare,p1_testwrite,qwen36-nothink-offspec,3,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,6,1,64,0.9219,42376,740.0,,0,1786833959 +p1_testwrite_qwen36-nothink-offspec_v4,mmbt-qwen36-compare,p1_testwrite,qwen36-nothink-offspec,4,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,2,1,68,0.9706,36608,630.5,,0,1786845367 +p1_testwrite_qwen36-27b-udq4xl-t03-nothink_v5,mmbt-qwen36-compare,p1_testwrite,qwen36-27b-udq4xl-t03-nothink,5,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,0,0,0,2,1,28,0.9643,,,,0,1786808034 +p1_testwrite_qwen36-nothink-offspec_v5,mmbt-qwen36-compare,p1_testwrite,qwen36-nothink-offspec,5,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,5,1,65,0.9385,35847,622.5,,0,1786846024 +p1_testwrite_qwen36-nothink-offspec_v6,mmbt-qwen36-compare,p1_testwrite,qwen36-nothink-offspec,6,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,3,1,68,0.9706,39193,689.7,,0,1786851862 +p1_testwrite_qwen36-nothink-offspec_v7,mmbt-qwen36-compare,p1_testwrite,qwen36-nothink-offspec,7,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,3,1,58,0.931,28681,501.3,,0,1786857611 +p1_testwrite_qwen36-nothink-offspec_v8,mmbt-qwen36-compare,p1_testwrite,qwen36-nothink-offspec,8,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,5,1,75,0.9467,32634,572.4,,0,1786863825 +p1_testwrite_qwen36-nothink-offspec_v9,mmbt-qwen36-compare,p1_testwrite,qwen36-nothink-offspec,9,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,4,1,71,0.9437,39139,685.8,,0,1786870636 +p1_testwrite_qwen36-nothink-offspec_v10,mmbt-qwen36-compare,p1_testwrite,qwen36-nothink-offspec,10,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,0,0,0,1,1,3,1.0,,,,0,1786875777 +p2_ci_qwen36-nothink-offspec_v1,mmbt-qwen36-compare,p2_ci,qwen36-nothink-offspec,1,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,39,0.9744,5520,94.4,,0,1786818005 +p2_ci_qwen36-nothink-offspec_v2,mmbt-qwen36-compare,p2_ci,qwen36-nothink-offspec,2,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,42,0.9762,4650,83.4,,0,1786827983 +p2_ci_qwen36-nothink-offspec_v3,mmbt-qwen36-compare,p2_ci,qwen36-nothink-offspec,3,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,3,1,41,0.9512,5798,101.1,,0,1786834334 +p2_ci_qwen36-nothink-offspec_v4,mmbt-qwen36-compare,p2_ci,qwen36-nothink-offspec,4,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,3,1,38,0.9474,4203,76.9,,0,1786839802 +p2_ci_qwen36-nothink-offspec_v5,mmbt-qwen36-compare,p2_ci,qwen36-nothink-offspec,5,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,3,1,41,0.9512,6236,107.7,,0,1786846361 +p2_ci_qwen36-nothink-offspec_v6,mmbt-qwen36-compare,p2_ci,qwen36-nothink-offspec,6,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,38,0.9737,4538,79.5,,0,1786852187 +p2_ci_qwen36-nothink-offspec_v7,mmbt-qwen36-compare,p2_ci,qwen36-nothink-offspec,7,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,36,0.9722,4376,77.5,,0,1786857950 +p2_ci_qwen36-nothink-offspec_v8,mmbt-qwen36-compare,p2_ci,qwen36-nothink-offspec,8,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,38,0.9737,4498,78.8,,0,1786864146 +p2_ci_qwen36-nothink-offspec_v9,mmbt-qwen36-compare,p2_ci,qwen36-nothink-offspec,9,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,39,0.9744,4600,82.6,,0,1786870990 +p2_extract_qwen36-nothink-offspec_v1,mmbt-qwen36-compare,p2_extract,qwen36-nothink-offspec,1,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,10,1.0,3488,54.6,,0,1786817910 +p2_extract_qwen36-nothink-offspec_v2,mmbt-qwen36-compare,p2_extract,qwen36-nothink-offspec,2,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,8,1.0,2868,45.5,,0,1786827899 +p2_extract_qwen36-nothink-offspec_v3,mmbt-qwen36-compare,p2_extract,qwen36-nothink-offspec,3,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,8,1.0,2871,45.1,,0,1786834232 +p2_extract_qwen36-nothink-offspec_v4,mmbt-qwen36-compare,p2_extract,qwen36-nothink-offspec,4,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,8,1.0,2871,45.4,,0,1786839724 +p2_extract_qwen36-nothink-offspec_v5,mmbt-qwen36-compare,p2_extract,qwen36-nothink-offspec,5,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,8,1.0,2868,45.2,,0,1786846252 +p2_extract_qwen36-nothink-offspec_v6,mmbt-qwen36-compare,p2_extract,qwen36-nothink-offspec,6,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,10,1.0,3488,54.9,,0,1786852106 +p2_extract_qwen36-nothink-offspec_v7,mmbt-qwen36-compare,p2_extract,qwen36-nothink-offspec,7,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,8,1.0,2868,45.2,,0,1786857872 +p2_extract_qwen36-nothink-offspec_v8,mmbt-qwen36-compare,p2_extract,qwen36-nothink-offspec,8,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,10,1.0,3467,54.3,,0,1786864067 +p2_extract_qwen36-nothink-offspec_v9,mmbt-qwen36-compare,p2_extract,qwen36-nothink-offspec,9,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,10,1.0,3589,56.2,,0,1786870906 +p2_hallucination_qwen36-nothink-offspec_v1,mmbt-qwen36-compare,p2_hallucination,qwen36-nothink-offspec,1,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,45,1.0,6092,99.8,,0,1786818106 +p2_hallucination_qwen36-nothink-offspec_v2,mmbt-qwen36-compare,p2_hallucination,qwen36-nothink-offspec,2,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,44,1.0,5101,85.7,,0,1786828070 +p2_hallucination_qwen36-nothink-offspec_v3,mmbt-qwen36-compare,p2_hallucination,qwen36-nothink-offspec,3,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,35,1.0,5122,84.1,,0,1786834419 +p2_hallucination_qwen36-nothink-offspec_v4,mmbt-qwen36-compare,p2_hallucination,qwen36-nothink-offspec,4,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,41,1.0,5412,88.7,,0,1786839891 +p2_hallucination_qwen36-nothink-offspec_v5,mmbt-qwen36-compare,p2_hallucination,qwen36-nothink-offspec,5,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,38,1.0,4661,77.3,,0,1786846439 +p2_hallucination_qwen36-nothink-offspec_v6,mmbt-qwen36-compare,p2_hallucination,qwen36-nothink-offspec,6,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,39,1.0,4797,80.1,,0,1786852267 +p2_hallucination_qwen36-nothink-offspec_v7,mmbt-qwen36-compare,p2_hallucination,qwen36-nothink-offspec,7,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,39,1.0,4930,81.8,,0,1786858033 +p2_hallucination_qwen36-nothink-offspec_v8,mmbt-qwen36-compare,p2_hallucination,qwen36-nothink-offspec,8,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,40,1.0,4947,82.1,,0,1786864229 +p2_hallucination_qwen36-nothink-offspec_v9,mmbt-qwen36-compare,p2_hallucination,qwen36-nothink-offspec,9,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,40,1.0,5111,83.1,,0,1786871074 +p2_triage_qwen36-nothink-offspec_v1,mmbt-qwen36-compare,p2_triage,qwen36-nothink-offspec,1,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,15,1.0,7896,122.0,,0,1786818229 +p2_triage_qwen36-nothink-offspec_v2,mmbt-qwen36-compare,p2_triage,qwen36-nothink-offspec,2,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,17,1.0,7199,111.2,,0,1786828182 +p2_triage_qwen36-nothink-offspec_v3,mmbt-qwen36-compare,p2_triage,qwen36-nothink-offspec,3,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,14,1.0,7743,118.1,,0,1786834538 +p2_triage_qwen36-nothink-offspec_v4,mmbt-qwen36-compare,p2_triage,qwen36-nothink-offspec,4,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,16,1.0,7707,119.1,,0,1786840011 +p2_triage_qwen36-nothink-offspec_v5,mmbt-qwen36-compare,p2_triage,qwen36-nothink-offspec,5,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,16,1.0,7055,109.4,,0,1786846549 +p2_triage_qwen36-nothink-offspec_v6,mmbt-qwen36-compare,p2_triage,qwen36-nothink-offspec,6,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,15,1.0,7599,118.3,,0,1786852387 +p2_triage_qwen36-nothink-offspec_v7,mmbt-qwen36-compare,p2_triage,qwen36-nothink-offspec,7,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,15,1.0,7831,121.1,,0,1786858155 +p2_triage_qwen36-nothink-offspec_v8,mmbt-qwen36-compare,p2_triage,qwen36-nothink-offspec,8,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,15,1.0,7735,120.0,,0,1786864350 +p2_triage_qwen36-nothink-offspec_v9,mmbt-qwen36-compare,p2_triage,qwen36-nothink-offspec,9,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,15,1.0,7209,110.4,,0,1786871185 +p3_business_qwen36-nothink-offspec_v1,mmbt-qwen36-compare,p3_business,qwen36-nothink-offspec,1,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,3,1,19,0.8947,9370,142.3,,0,1786818482 +p3_business_qwen36-nothink-offspec_v2,mmbt-qwen36-compare,p3_business,qwen36-nothink-offspec,2,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,4,1,23,0.8261,10622,162.8,,0,1786828468 +p3_business_qwen36-nothink-offspec_v3,mmbt-qwen36-compare,p3_business,qwen36-nothink-offspec,3,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,5,2,41,0.8049,22718,354.8,,0,1786835003 +p3_business_qwen36-nothink-offspec_v4,mmbt-qwen36-compare,p3_business,qwen36-nothink-offspec,4,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,5,1,26,0.8077,12904,197.0,,0,1786840329 +p3_business_qwen36-nothink-offspec_v5,mmbt-qwen36-compare,p3_business,qwen36-nothink-offspec,5,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,17,0.9412,8188,124.0,,0,1786846801 +p3_business_qwen36-nothink-offspec_v6,mmbt-qwen36-compare,p3_business,qwen36-nothink-offspec,6,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,3,1,19,0.8947,9301,141.8,,0,1786852649 +p3_business_qwen36-nothink-offspec_v7,mmbt-qwen36-compare,p3_business,qwen36-nothink-offspec,7,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,4,1,23,0.8696,9489,146.3,,0,1786858423 +p3_business_qwen36-nothink-offspec_v8,mmbt-qwen36-compare,p3_business,qwen36-nothink-offspec,8,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,4,1,24,0.7917,11411,174.4,,0,1786864650 +p3_business_qwen36-nothink-offspec_v9,mmbt-qwen36-compare,p3_business,qwen36-nothink-offspec,9,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,4,1,24,0.875,9503,145.1,,0,1786871454 +p3_doc_qwen36-nothink-offspec_v1,mmbt-qwen36-compare,p3_doc,qwen36-nothink-offspec,1,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,17,0.9412,7102,109.1,,0,1786818339 +p3_doc_qwen36-nothink-offspec_v2,mmbt-qwen36-compare,p3_doc,qwen36-nothink-offspec,2,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,19,0.9474,7896,121.6,,0,1786828304 +p3_doc_qwen36-nothink-offspec_v3,mmbt-qwen36-compare,p3_doc,qwen36-nothink-offspec,3,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,17,0.9412,7085,108.8,,0,1786834647 +p3_doc_qwen36-nothink-offspec_v4,mmbt-qwen36-compare,p3_doc,qwen36-nothink-offspec,4,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,19,0.9474,7801,119.7,,0,1786840132 +p3_doc_qwen36-nothink-offspec_v5,mmbt-qwen36-compare,p3_doc,qwen36-nothink-offspec,5,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,18,0.9444,8264,125.9,,0,1786846676 +p3_doc_qwen36-nothink-offspec_v6,mmbt-qwen36-compare,p3_doc,qwen36-nothink-offspec,6,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,19,0.9474,7666,119.0,,0,1786852506 +p3_doc_qwen36-nothink-offspec_v7,mmbt-qwen36-compare,p3_doc,qwen36-nothink-offspec,7,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,3,1,21,0.8571,7669,120.1,,0,1786858276 +p3_doc_qwen36-nothink-offspec_v8,mmbt-qwen36-compare,p3_doc,qwen36-nothink-offspec,8,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,19,0.9474,8040,124.0,,0,1786864475 +p3_doc_qwen36-nothink-offspec_v9,mmbt-qwen36-compare,p3_doc,qwen36-nothink-offspec,9,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,17,0.9412,8094,122.6,,0,1786871308 +p3_market_qwen36-nothink-offspec_v1,mmbt-qwen36-compare,p3_market,qwen36-nothink-offspec,1,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,STRUCTURAL_FAIL,1,0,1,0,0,1,1,56,1.0,10006,3789.7,,0,1786823170 +p3_market_qwen36-nothink-offspec_v2,mmbt-qwen36-compare,p3_market,qwen36-nothink-offspec,2,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,138,1.0,44410,953.6,,0,1786829423 +p3_market_qwen36-nothink-offspec_v3,mmbt-qwen36-compare,p3_market,qwen36-nothink-offspec,3,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,45,45,236,0.8008,,,identical-call-loop,0,1786835627 +p3_market_qwen36-nothink-offspec_v4,mmbt-qwen36-compare,p3_market,qwen36-nothink-offspec,4,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,68,1.0,45574,943.4,,0,1786841274 +p3_market_qwen36-nothink-offspec_v5,mmbt-qwen36-compare,p3_market,qwen36-nothink-offspec,5,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,4,4,95,0.9579,40432,799.0,,0,1786847601 +p3_market_qwen36-nothink-offspec_v6,mmbt-qwen36-compare,p3_market,qwen36-nothink-offspec,6,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,31,31,140,0.7857,,,identical-call-loop,0,1786852979 +p3_market_qwen36-nothink-offspec_v7,mmbt-qwen36-compare,p3_market,qwen36-nothink-offspec,7,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,42,38,127,0.6772,,,identical-call-loop,0,1786859039 +p3_market_qwen36-nothink-offspec_v8,mmbt-qwen36-compare,p3_market,qwen36-nothink-offspec,8,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,2,2,71,0.9859,31604,839.0,,0,1786865490 +p3_market_qwen36-nothink-offspec_v9,mmbt-qwen36-compare,p3_market,qwen36-nothink-offspec,9,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,0,0,1,1,104,1.0,,,identical-call-loop,0,1786872063 +p3_pm_qwen36-nothink-offspec_v1,mmbt-qwen36-compare,p3_pm,qwen36-nothink-offspec,1,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,2,1,13,0.9231,4220,66.3,,0,1786818960 +p3_pm_qwen36-nothink-offspec_v2,mmbt-qwen36-compare,p3_pm,qwen36-nothink-offspec,2,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,2,1,13,0.9231,4594,71.7,,0,1786829611 +p3_pm_qwen36-nothink-offspec_v3,mmbt-qwen36-compare,p3_pm,qwen36-nothink-offspec,3,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,2,1,12,0.9167,4468,68.7,,0,1786835807 +p3_pm_qwen36-nothink-offspec_v4,mmbt-qwen36-compare,p3_pm,qwen36-nothink-offspec,4,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,3,1,16,0.875,6176,94.8,,0,1786841467 +p3_pm_qwen36-nothink-offspec_v5,mmbt-qwen36-compare,p3_pm,qwen36-nothink-offspec,5,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,2,1,13,0.9231,4217,65.6,,0,1786847772 +p3_pm_qwen36-nothink-offspec_v6,mmbt-qwen36-compare,p3_pm,qwen36-nothink-offspec,6,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,2,1,13,0.9231,4634,71.5,,0,1786853156 +p3_pm_qwen36-nothink-offspec_v7,mmbt-qwen36-compare,p3_pm,qwen36-nothink-offspec,7,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,2,1,13,0.9231,4657,71.5,,0,1786859196 +p3_pm_qwen36-nothink-offspec_v8,mmbt-qwen36-compare,p3_pm,qwen36-nothink-offspec,8,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,2,1,13,0.9231,4648,72.2,,0,1786865668 +p3_pm_qwen36-nothink-offspec_v9,mmbt-qwen36-compare,p3_pm,qwen36-nothink-offspec,9,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,2,1,13,0.9231,4640,71.3,,0,1786872232 +p3_writing_qwen36-nothink-offspec_v1,mmbt-qwen36-compare,p3_writing,qwen36-nothink-offspec,1,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,21,1.0,6682,105.6,,0,1786818893 +p3_writing_qwen36-nothink-offspec_v2,mmbt-qwen36-compare,p3_writing,qwen36-nothink-offspec,2,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,2,1,24,0.9167,7024,114.0,,0,1786829538 +p3_writing_qwen36-nothink-offspec_v3,mmbt-qwen36-compare,p3_writing,qwen36-nothink-offspec,3,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,21,1.0,6717,106.7,,0,1786835737 +p3_writing_qwen36-nothink-offspec_v4,mmbt-qwen36-compare,p3_writing,qwen36-nothink-offspec,4,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,2,1,21,0.9524,5957,96.3,,0,1786841371 +p3_writing_qwen36-nothink-offspec_v5,mmbt-qwen36-compare,p3_writing,qwen36-nothink-offspec,5,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,21,1.0,6590,104.3,,0,1786847706 +p3_writing_qwen36-nothink-offspec_v6,mmbt-qwen36-compare,p3_writing,qwen36-nothink-offspec,6,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,2,1,22,0.9545,6427,101.4,,0,1786853083 +p3_writing_qwen36-nothink-offspec_v7,mmbt-qwen36-compare,p3_writing,qwen36-nothink-offspec,7,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,20,1.0,5165,83.1,,0,1786859123 +p3_writing_qwen36-nothink-offspec_v8,mmbt-qwen36-compare,p3_writing,qwen36-nothink-offspec,8,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,2,1,26,0.9231,6403,104.0,,0,1786865595 +p3_writing_qwen36-nothink-offspec_v9,mmbt-qwen36-compare,p3_writing,qwen36-nothink-offspec,9,3.6,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,20,1.0,5791,91.2,,0,1786872160 +p1_bugfix_qwen36-nothink-card_v1,mmbt-q36-card,p1_bugfix,qwen36-nothink-card,1,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,6,1,147,0.9048,30446,897.4,,0,1786820268 +p1_bugfix_qwen36-nothink-card_v2,mmbt-q36-card,p1_bugfix,qwen36-nothink-card,2,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,8,1,182,0.9341,34149,985.5,,0,1786812152 +p1_bugfix_qwen36-nothink-card_v3,mmbt-q36-card,p1_bugfix,qwen36-nothink-card,3,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,12,1,177,0.8927,34060,1002.0,,0,1786813156 +p1_bugfix_qwen36-nothink-card_v4,mmbt-q36-card,p1_bugfix,qwen36-nothink-card,4,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,4,1,133,0.9248,33765,972.0,,0,1786814131 +p1_bugfix_qwen36-nothink-card_v5,mmbt-q36-card,p1_bugfix,qwen36-nothink-card,5,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,3,1,141,0.9787,34121,954.2,,0,1786815086 +p1_bugfix_qwen36-nothink-card_v6,mmbt-q36-card,p1_bugfix,qwen36-nothink-card,6,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,5,1,132,0.9242,28643,630.9,,0,1786815719 +p1_bugfix_qwen36-nothink-card_v7,mmbt-q36-card,p1_bugfix,qwen36-nothink-card,7,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,10,1,159,0.9057,37096,1022.5,,0,1786816743 +p1_bugfix_qwen36-nothink-card_v8,mmbt-q36-card,p1_bugfix,qwen36-nothink-card,8,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,7,1,180,0.9111,42288,1148.6,,0,1786863010 +p1_bugfix_qwen36-nothink-card_v9,mmbt-q36-card,p1_bugfix,qwen36-nothink-card,9,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,8,1,165,0.9152,35621,992.4,,0,1786870038 +p1_bugfix_qwen36-nothink-card_v10,mmbt-q36-card,p1_bugfix,qwen36-nothink-card,10,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,5,1,147,0.9388,45886,1519.4,,0,1786877323 +p1_refactor_qwen36-nothink-card_v1,mmbt-q36-card,p1_refactor,qwen36-nothink-card,1,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,3,1,108,0.9815,11020,218.0,,0,1786817512 +p1_refactor_qwen36-nothink-card_v2,mmbt-q36-card,p1_refactor,qwen36-nothink-card,2,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,91,0.956,11380,213.2,,0,1786826714 +p1_refactor_qwen36-nothink-card_v3,mmbt-q36-card,p1_refactor,qwen36-nothink-card,3,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,74,0.9459,8440,156.4,,0,1786833223 +p1_refactor_qwen36-nothink-card_v4,mmbt-q36-card,p1_refactor,qwen36-nothink-card,4,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,80,0.9625,11051,198.5,,0,1786838882 +p1_refactor_qwen36-nothink-card_v5,mmbt-q36-card,p1_refactor,qwen36-nothink-card,5,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,79,0.9494,11507,212.7,,0,1786845068 +p1_refactor_qwen36-nothink-card_v6,mmbt-q36-card,p1_refactor,qwen36-nothink-card,6,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,81,0.9753,12077,214.8,,0,1786850821 +p1_refactor_qwen36-nothink-card_v7,mmbt-q36-card,p1_refactor,qwen36-nothink-card,7,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,75,0.9867,10872,194.3,,0,1786857168 +p1_refactor_qwen36-nothink-card_v8,mmbt-q36-card,p1_refactor,qwen36-nothink-card,8,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,89,0.9775,12481,229.2,,0,1786863690 +p1_refactor_qwen36-nothink-card_v9,mmbt-q36-card,p1_refactor,qwen36-nothink-card,9,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,86,0.9651,11895,221.5,,0,1786870717 +p1_refactor_qwen36-nothink-card_v10,mmbt-q36-card,p1_refactor,qwen36-nothink-card,10,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,3,1,87,0.954,11268,211.6,,0,1786878211 +p1_testwrite_qwen36-nothink-card_v1,mmbt-q36-card,p1_testwrite,qwen36-nothink-card,1,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,5,1,49,0.9184,36260,616.5,,0,1786820886 +p1_testwrite_qwen36-nothink-card_v2,mmbt-q36-card,p1_testwrite,qwen36-nothink-card,2,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,7,1,57,0.8947,41864,729.2,,0,1786826500 +p1_testwrite_qwen36-nothink-card_v3,mmbt-q36-card,p1_testwrite,qwen36-nothink-card,3,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,4,1,56,0.9286,34698,593.6,,0,1786833066 +p1_testwrite_qwen36-nothink-card_v4,mmbt-q36-card,p1_testwrite,qwen36-nothink-card,4,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,5,1,47,0.9149,30678,515.9,,0,1786838682 +p1_testwrite_qwen36-nothink-card_v5,mmbt-q36-card,p1_testwrite,qwen36-nothink-card,5,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,5,1,50,0.92,36317,618.1,,0,1786844855 +p1_testwrite_qwen36-nothink-card_v6,mmbt-q36-card,p1_testwrite,qwen36-nothink-card,6,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,7,1,64,0.9062,27304,475.5,,0,1786850605 +p1_testwrite_qwen36-nothink-card_v7,mmbt-q36-card,p1_testwrite,qwen36-nothink-card,7,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,5,1,47,0.9149,31798,533.6,,0,1786856973 +p1_testwrite_qwen36-nothink-card_v8,mmbt-q36-card,p1_testwrite,qwen36-nothink-card,8,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,4,1,60,0.9333,26122,447.8,,0,1786863460 +p1_testwrite_qwen36-nothink-card_v9,mmbt-q36-card,p1_testwrite,qwen36-nothink-card,9,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,5,1,48,0.8958,26918,453.9,,0,1786870495 +p1_testwrite_qwen36-nothink-card_v10,mmbt-q36-card,p1_testwrite,qwen36-nothink-card,10,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,5,1,62,0.9194,39034,673.6,,0,1786877999 +p2_ci_qwen36-nothink-card_v1,mmbt-q36-card,p2_ci,qwen36-nothink-card,1,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,3,1,34,0.9412,4984,88.0,,0,1786817627 +p2_ci_qwen36-nothink-card_v2,mmbt-q36-card,p2_ci,qwen36-nothink-card,2,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,39,0.9744,5719,98.6,,0,1786826839 +p2_ci_qwen36-nothink-card_v3,mmbt-q36-card,p2_ci,qwen36-nothink-card,3,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,38,1.0,5751,100.0,,0,1786833350 +p2_ci_qwen36-nothink-card_v4,mmbt-q36-card,p2_ci,qwen36-nothink-card,4,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,39,1.0,5814,101.2,,0,1786839011 +p2_ci_qwen36-nothink-card_v5,mmbt-q36-card,p2_ci,qwen36-nothink-card,5,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,44,0.9773,5590,101.5,,0,1786845196 +p2_ci_qwen36-nothink-card_v6,mmbt-q36-card,p2_ci,qwen36-nothink-card,6,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,43,0.9535,5380,97.2,,0,1786850944 +p2_ci_qwen36-nothink-card_v7,mmbt-q36-card,p2_ci,qwen36-nothink-card,7,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,38,0.9474,4921,87.5,,0,1786857281 +p2_ci_qwen36-nothink-card_v8,mmbt-q36-card,p2_ci,qwen36-nothink-card,8,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,3,1,40,0.95,6413,108.7,,0,1786863826 +p2_ci_qwen36-nothink-card_v9,mmbt-q36-card,p2_ci,qwen36-nothink-card,9,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,37,0.973,5319,92.5,,0,1786870837 +p2_ci_qwen36-nothink-card_v10,mmbt-q36-card,p2_ci,qwen36-nothink-card,10,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,38,0.9737,5604,98.3,,0,1786878336 +p2_extract_qwen36-nothink-card_v1,mmbt-q36-card,p2_extract,qwen36-nothink-card,1,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,6,1.0,1515,25.3,,0,1786817538 +p2_extract_qwen36-nothink-card_v2,mmbt-q36-card,p2_extract,qwen36-nothink-card,2,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,6,1.0,1490,24.8,,0,1786826740 +p2_extract_qwen36-nothink-card_v3,mmbt-q36-card,p2_extract,qwen36-nothink-card,3,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,6,1.0,1515,24.9,,0,1786833249 +p2_extract_qwen36-nothink-card_v4,mmbt-q36-card,p2_extract,qwen36-nothink-card,4,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,6,1.0,1515,25.2,,0,1786838909 +p2_extract_qwen36-nothink-card_v5,mmbt-q36-card,p2_extract,qwen36-nothink-card,5,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,6,1.0,1490,24.9,,0,1786845094 +p2_extract_qwen36-nothink-card_v6,mmbt-q36-card,p2_extract,qwen36-nothink-card,6,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,6,1.0,1434,24.0,,0,1786850846 +p2_extract_qwen36-nothink-card_v7,mmbt-q36-card,p2_extract,qwen36-nothink-card,7,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,6,1.0,1434,23.9,,0,1786857193 +p2_extract_qwen36-nothink-card_v8,mmbt-q36-card,p2_extract,qwen36-nothink-card,8,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,6,1.0,1515,25.4,,0,1786863716 +p2_extract_qwen36-nothink-card_v9,mmbt-q36-card,p2_extract,qwen36-nothink-card,9,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,6,1.0,1515,25.4,,0,1786870743 +p2_extract_qwen36-nothink-card_v10,mmbt-q36-card,p2_extract,qwen36-nothink-card,10,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,6,1.0,1490,24.8,,0,1786878237 +p2_hallucination_qwen36-nothink-card_v1,mmbt-q36-card,p2_hallucination,qwen36-nothink-card,1,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,38,1.0,4314,72.3,,0,1786817700 +p2_hallucination_qwen36-nothink-card_v2,mmbt-q36-card,p2_hallucination,qwen36-nothink-card,2,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,47,1.0,5800,98.1,,0,1786826938 +p2_hallucination_qwen36-nothink-card_v3,mmbt-q36-card,p2_hallucination,qwen36-nothink-card,3,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,47,1.0,7737,128.1,,0,1786833479 +p2_hallucination_qwen36-nothink-card_v4,mmbt-q36-card,p2_hallucination,qwen36-nothink-card,4,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,51,1.0,6713,113.4,,0,1786839125 +p2_hallucination_qwen36-nothink-card_v5,mmbt-q36-card,p2_hallucination,qwen36-nothink-card,5,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,47,1.0,6641,109.9,,0,1786845307 +p2_hallucination_qwen36-nothink-card_v6,mmbt-q36-card,p2_hallucination,qwen36-nothink-card,6,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,47,1.0,5853,98.2,,0,1786851043 +p2_hallucination_qwen36-nothink-card_v7,mmbt-q36-card,p2_hallucination,qwen36-nothink-card,7,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,51,1.0,5924,99.8,,0,1786857382 +p2_hallucination_qwen36-nothink-card_v8,mmbt-q36-card,p2_hallucination,qwen36-nothink-card,8,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,41,1.0,5255,87.0,,0,1786863914 +p2_hallucination_qwen36-nothink-card_v9,mmbt-q36-card,p2_hallucination,qwen36-nothink-card,9,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,46,1.0,6836,112.8,,0,1786870950 +p2_hallucination_qwen36-nothink-card_v10,mmbt-q36-card,p2_hallucination,qwen36-nothink-card,10,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,43,1.0,6503,108.4,,0,1786878445 +p2_triage_qwen36-nothink-card_v1,mmbt-q36-card,p2_triage,qwen36-nothink-card,1,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,12,1.0,7038,108.2,,0,1786817809 +p2_triage_qwen36-nothink-card_v2,mmbt-q36-card,p2_triage,qwen36-nothink-card,2,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,11,1.0,6803,104.9,,0,1786827044 +p2_triage_qwen36-nothink-card_v3,mmbt-q36-card,p2_triage,qwen36-nothink-card,3,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,11,1.0,6815,106.2,,0,1786833586 +p2_triage_qwen36-nothink-card_v4,mmbt-q36-card,p2_triage,qwen36-nothink-card,4,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,12,1.0,6708,103.5,,0,1786839230 +p2_triage_qwen36-nothink-card_v5,mmbt-q36-card,p2_triage,qwen36-nothink-card,5,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,14,1.0,8797,134.3,,0,1786845442 +p2_triage_qwen36-nothink-card_v6,mmbt-q36-card,p2_triage,qwen36-nothink-card,6,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,11,1.0,6643,102.1,,0,1786851146 +p2_triage_qwen36-nothink-card_v7,mmbt-q36-card,p2_triage,qwen36-nothink-card,7,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,16,1.0,9703,148.9,,0,1786857531 +p2_triage_qwen36-nothink-card_v8,mmbt-q36-card,p2_triage,qwen36-nothink-card,8,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,12,1.0,7033,108.4,,0,1786864023 +p2_triage_qwen36-nothink-card_v9,mmbt-q36-card,p2_triage,qwen36-nothink-card,9,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,18,1.0,9851,152.3,,0,1786871104 +p2_triage_qwen36-nothink-card_v10,mmbt-q36-card,p2_triage,qwen36-nothink-card,10,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,12,1.0,6988,107.3,,0,1786878553 +p3_business_qwen36-nothink-card_v1,mmbt-q36-card,p3_business,qwen36-nothink-card,1,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,17,0.9412,7708,117.3,,0,1786818059 +p3_business_qwen36-nothink-card_v2,mmbt-q36-card,p3_business,qwen36-nothink-card,2,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,13,0.9231,7345,110.9,,0,1786827268 +p3_business_qwen36-nothink-card_v3,mmbt-q36-card,p3_business,qwen36-nothink-card,3,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,16,0.9375,7533,114.6,,0,1786833823 +p3_business_qwen36-nothink-card_v4,mmbt-q36-card,p3_business,qwen36-nothink-card,4,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,13,0.9231,7387,112.2,,0,1786844223 +p3_business_qwen36-nothink-card_v5,mmbt-q36-card,p3_business,qwen36-nothink-card,5,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,14,0.9286,7295,110.4,,0,1786845680 +p3_business_qwen36-nothink-card_v6,mmbt-q36-card,p3_business,qwen36-nothink-card,6,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,13,1.0,7121,107.8,,0,1786851378 +p3_business_qwen36-nothink-card_v7,mmbt-q36-card,p3_business,qwen36-nothink-card,7,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,14,0.9286,7398,111.7,,0,1786857761 +p3_business_qwen36-nothink-card_v8,mmbt-q36-card,p3_business,qwen36-nothink-card,8,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,4,1,28,0.8571,11913,184.4,,0,1786864324 +p3_business_qwen36-nothink-card_v9,mmbt-q36-card,p3_business,qwen36-nothink-card,9,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,15,0.9333,7503,113.6,,0,1786871351 +p3_business_qwen36-nothink-card_v10,mmbt-q36-card,p3_business,qwen36-nothink-card,10,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,13,0.9231,7411,111.6,,0,1786878797 +p3_doc_qwen36-nothink-card_v1,mmbt-q36-card,p3_doc,qwen36-nothink-card,1,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,16,0.9375,8568,131.1,,0,1786817941 +p3_doc_qwen36-nothink-card_v2,mmbt-q36-card,p3_doc,qwen36-nothink-card,2,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,14,1.0,7320,111.4,,0,1786827156 +p3_doc_qwen36-nothink-card_v3,mmbt-q36-card,p3_doc,qwen36-nothink-card,3,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,15,0.9333,7883,120.7,,0,1786833707 +p3_doc_qwen36-nothink-card_v4,mmbt-q36-card,p3_doc,qwen36-nothink-card,4,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,20,0.95,8370,129.0,,0,1786839360 +p3_doc_qwen36-nothink-card_v5,mmbt-q36-card,p3_doc,qwen36-nothink-card,5,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,16,0.9375,8268,126.4,,0,1786845569 +p3_doc_qwen36-nothink-card_v6,mmbt-q36-card,p3_doc,qwen36-nothink-card,6,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,16,0.9375,7926,122.0,,0,1786851269 +p3_doc_qwen36-nothink-card_v7,mmbt-q36-card,p3_doc,qwen36-nothink-card,7,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,16,1.0,7562,116.4,,0,1786857648 +p3_doc_qwen36-nothink-card_v8,mmbt-q36-card,p3_doc,qwen36-nothink-card,8,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,15,1.0,7470,115.0,,0,1786864139 +p3_doc_qwen36-nothink-card_v9,mmbt-q36-card,p3_doc,qwen36-nothink-card,9,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,21,0.9524,8551,132.1,,0,1786871237 +p3_doc_qwen36-nothink-card_v10,mmbt-q36-card,p3_doc,qwen36-nothink-card,10,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,18,0.9444,8468,130.1,,0,1786878684 +p3_market_qwen36-nothink-card_v1,mmbt-q36-card,p3_market,qwen36-nothink-card,1,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,2,1,110,0.9909,32074,719.3,,0,1786821606 +p3_market_qwen36-nothink-card_v2,mmbt-q36-card,p3_market,qwen36-nothink-card,2,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,83,1.0,38520,850.8,,0,1786828120 +p3_market_qwen36-nothink-card_v3,mmbt-q36-card,p3_market,qwen36-nothink-card,3,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,123,1.0,34081,711.4,,0,1786834535 +p3_market_qwen36-nothink-card_v4,mmbt-q36-card,p3_market,qwen36-nothink-card,4,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,98,1.0,33381,1192.9,,0,1786840685 +p3_market_qwen36-nothink-card_v5,mmbt-q36-card,p3_market,qwen36-nothink-card,5,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,124,1.0,32475,651.0,,0,1786846332 +p3_market_qwen36-nothink-card_v6,mmbt-q36-card,p3_market,qwen36-nothink-card,6,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,69,1.0,30760,653.9,,0,1786852033 +p3_market_qwen36-nothink-card_v7,mmbt-q36-card,p3_market,qwen36-nothink-card,7,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,114,1.0,32226,641.4,,0,1786858403 +p3_market_qwen36-nothink-card_v8,mmbt-q36-card,p3_market,qwen36-nothink-card,8,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,107,1.0,38261,850.9,,0,1786865176 +p3_market_qwen36-nothink-card_v9,mmbt-q36-card,p3_market,qwen36-nothink-card,9,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,99,1.0,44948,967.3,,0,1786872320 +p3_market_qwen36-nothink-card_v10,mmbt-q36-card,p3_market,qwen36-nothink-card,10,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,55,1.0,32115,948.3,,0,1786879746 +p3_pm_qwen36-nothink-card_v1,mmbt-q36-card,p3_pm,qwen36-nothink-card,1,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,11,1.0,4150,65.7,,0,1786821673 +p3_pm_qwen36-nothink-card_v2,mmbt-q36-card,p3_pm,qwen36-nothink-card,2,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,12,1.0,4176,64.8,,0,1786828270 +p3_pm_qwen36-nothink-card_v3,mmbt-q36-card,p3_pm,qwen36-nothink-card,3,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,11,1.0,4187,64.6,,0,1786834722 +p3_pm_qwen36-nothink-card_v4,mmbt-q36-card,p3_pm,qwen36-nothink-card,4,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,11,1.0,4157,64.1,,0,1786840862 +p3_pm_qwen36-nothink-card_v5,mmbt-q36-card,p3_pm,qwen36-nothink-card,5,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,11,1.0,3898,60.8,,0,1786846509 +p3_pm_qwen36-nothink-card_v6,mmbt-q36-card,p3_pm,qwen36-nothink-card,6,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,12,1.0,4264,66.2,,0,1786852217 +p3_pm_qwen36-nothink-card_v7,mmbt-q36-card,p3_pm,qwen36-nothink-card,7,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,12,0.9167,4735,73.5,,0,1786858621 +p3_pm_qwen36-nothink-card_v8,mmbt-q36-card,p3_pm,qwen36-nothink-card,8,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,12,1.0,4239,65.5,,0,1786865349 +p3_pm_qwen36-nothink-card_v9,mmbt-q36-card,p3_pm,qwen36-nothink-card,9,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,12,0.9167,4700,72.3,,0,1786872511 +p3_pm_qwen36-nothink-card_v10,mmbt-q36-card,p3_pm,qwen36-nothink-card,10,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,11,1.0,4084,63.1,,0,1786879924 +p3_writing_qwen36-nothink-card_v1,mmbt-q36-card,p3_writing,qwen36-nothink-card,1,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,21,0.9524,7045,111.7,,0,1786819307 +p3_writing_qwen36-nothink-card_v2,mmbt-q36-card,p3_writing,qwen36-nothink-card,2,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,25,1.0,5096,84.3,,0,1786828205 +p3_writing_qwen36-nothink-card_v3,mmbt-q36-card,p3_writing,qwen36-nothink-card,3,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,28,0.8929,7486,121.3,,0,1786834657 +p3_writing_qwen36-nothink-card_v4,mmbt-q36-card,p3_writing,qwen36-nothink-card,4,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,26,0.9231,6918,111.8,,0,1786840797 +p3_writing_qwen36-nothink-card_v5,mmbt-q36-card,p3_writing,qwen36-nothink-card,5,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,23,0.9565,7162,114.2,,0,1786846447 +p3_writing_qwen36-nothink-card_v6,mmbt-q36-card,p3_writing,qwen36-nothink-card,6,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,22,0.9545,7320,116.6,,0,1786852150 +p3_writing_qwen36-nothink-card_v7,mmbt-q36-card,p3_writing,qwen36-nothink-card,7,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,35,0.9143,8780,142.4,,0,1786858547 +p3_writing_qwen36-nothink-card_v8,mmbt-q36-card,p3_writing,qwen36-nothink-card,8,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,26,0.9231,6494,106.1,,0,1786865283 +p3_writing_qwen36-nothink-card_v9,mmbt-q36-card,p3_writing,qwen36-nothink-card,9,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,28,0.9286,7235,117.7,,0,1786872438 +p3_writing_qwen36-nothink-card_v10,mmbt-q36-card,p3_writing,qwen36-nothink-card,10,3.6,UD-Q4_K_XL,no-think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,28,0.9286,6886,112.7,,0,1786879860 +p1_bugfix_qwen36-think-offspec_v1,mmbt-qwen36-compare,p1_bugfix,qwen36-think-offspec,1,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,5,1,98,0.9388,25991,794.2,,0,1786823971 +p1_bugfix_qwen36-think-offspec_v2,mmbt-qwen36-compare,p1_bugfix,qwen36-think-offspec,2,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,7,1,113,0.9027,32044,899.6,,0,1786830551 +p1_bugfix_qwen36-think-offspec_v3,mmbt-qwen36-compare,p1_bugfix,qwen36-think-offspec,3,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,7,1,161,0.9255,32596,934.8,,0,1786836776 +p1_bugfix_qwen36-think-offspec_v4,mmbt-qwen36-compare,p1_bugfix,qwen36-think-offspec,4,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,7,1,114,0.9035,36437,967.9,,0,1786842444 +p1_bugfix_qwen36-think-offspec_v5,mmbt-qwen36-compare,p1_bugfix,qwen36-think-offspec,5,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,4,1,137,0.9124,37161,993.6,,0,1786848810 +p1_bugfix_qwen36-think-offspec_v6,mmbt-qwen36-compare,p1_bugfix,qwen36-think-offspec,6,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,6,1,104,0.8942,31111,869.1,,0,1786854033 +p1_bugfix_qwen36-think-offspec_v7,mmbt-qwen36-compare,p1_bugfix,qwen36-think-offspec,7,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,5,1,127,0.9291,32699,920.1,,0,1786860172 +p1_bugfix_qwen36-think-offspec_v8,mmbt-qwen36-compare,p1_bugfix,qwen36-think-offspec,8,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,4,1,125,0.928,30197,854.4,,0,1786866583 +p1_bugfix_qwen36-think-offspec_v9,mmbt-qwen36-compare,p1_bugfix,qwen36-think-offspec,9,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,6,1,123,0.8943,28846,840.6,,0,1786873135 +p1_refactor_qwen36-think-offspec_v1,mmbt-qwen36-compare,p1_refactor,qwen36-think-offspec,1,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,82,0.9634,12071,219.0,,0,1786824987 +p1_refactor_qwen36-think-offspec_v2,mmbt-qwen36-compare,p1_refactor,qwen36-think-offspec,2,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,92,0.913,14253,258.2,,0,1786831501 +p1_refactor_qwen36-think-offspec_v3,mmbt-qwen36-compare,p1_refactor,qwen36-think-offspec,3,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,84,0.9881,14523,258.1,,0,1786838031 +p1_refactor_qwen36-think-offspec_v4,mmbt-qwen36-compare,p1_refactor,qwen36-think-offspec,4,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,2,1,64,0.9844,11009,194.0,,0,1786843372 +p1_refactor_qwen36-think-offspec_v5,mmbt-qwen36-compare,p1_refactor,qwen36-think-offspec,5,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,77,0.9481,12387,220.2,,0,1786849703 +p1_refactor_qwen36-think-offspec_v6,mmbt-qwen36-compare,p1_refactor,qwen36-think-offspec,6,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,3,1,73,0.9726,10982,197.7,,0,1786854951 +p1_refactor_qwen36-think-offspec_v7,mmbt-qwen36-compare,p1_refactor,qwen36-think-offspec,7,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,2,1,85,0.9647,12619,231.2,,0,1786861519 +p1_refactor_qwen36-think-offspec_v8,mmbt-qwen36-compare,p1_refactor,qwen36-think-offspec,8,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,2,1,76,0.9868,13590,244.0,,0,1786867733 +p1_refactor_qwen36-think-offspec_v9,mmbt-qwen36-compare,p1_refactor,qwen36-think-offspec,9,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,3,1,83,0.9518,11948,218.5,,0,1786873964 +p1_testwrite_qwen36-think-offspec_v1,mmbt-qwen36-compare,p1_testwrite,qwen36-think-offspec,1,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,5,1,75,0.9467,45945,793.6,,0,1786824767 +p1_testwrite_qwen36-think-offspec_v2,mmbt-qwen36-compare,p1_testwrite,qwen36-think-offspec,2,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,6,1,75,0.9333,40177,688.7,,0,1786831242 +p1_testwrite_qwen36-think-offspec_v3,mmbt-qwen36-compare,p1_testwrite,qwen36-think-offspec,3,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,84,0.9762,56249,993.1,,0,1786837772 +p1_testwrite_qwen36-think-offspec_v4,mmbt-qwen36-compare,p1_testwrite,qwen36-think-offspec,4,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,6,1,69,0.9275,42849,730.9,,0,1786843177 +p1_testwrite_qwen36-think-offspec_v5,mmbt-qwen36-compare,p1_testwrite,qwen36-think-offspec,5,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,6,1,68,0.9265,39234,669.5,,0,1786849482 +p1_testwrite_qwen36-think-offspec_v6,mmbt-qwen36-compare,p1_testwrite,qwen36-think-offspec,6,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,6,1,61,0.918,41903,716.9,,0,1786854753 +p1_testwrite_qwen36-think-offspec_v7,mmbt-qwen36-compare,p1_testwrite,qwen36-think-offspec,7,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,7,1,79,0.9241,62090,1112.7,,0,1786861287 +p1_testwrite_qwen36-think-offspec_v8,mmbt-qwen36-compare,p1_testwrite,qwen36-think-offspec,8,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,7,1,73,0.9041,51814,902.4,,0,1786867488 +p1_testwrite_qwen36-think-offspec_v9,mmbt-qwen36-compare,p1_testwrite,qwen36-think-offspec,9,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,5,1,63,0.9365,35953,607.1,,0,1786873745 +p2_ci_qwen36-think-offspec_v1,mmbt-qwen36-compare,p2_ci,qwen36-think-offspec,1,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,3,1,39,0.9487,6900,116.5,,0,1786825166 +p2_ci_qwen36-think-offspec_v2,mmbt-qwen36-compare,p2_ci,qwen36-think-offspec,2,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,41,0.9512,7061,119.8,,0,1786831676 +p2_ci_qwen36-think-offspec_v3,mmbt-qwen36-compare,p2_ci,qwen36-think-offspec,3,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,39,0.9487,6670,110.2,,0,1786838203 +p2_ci_qwen36-think-offspec_v4,mmbt-qwen36-compare,p2_ci,qwen36-think-offspec,4,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,36,1.0,6147,102.2,,0,1786843541 +p2_ci_qwen36-think-offspec_v5,mmbt-qwen36-compare,p2_ci,qwen36-think-offspec,5,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,41,0.9512,6675,112.6,,0,1786849877 +p2_ci_qwen36-think-offspec_v6,mmbt-qwen36-compare,p2_ci,qwen36-think-offspec,6,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,43,0.9535,6751,117.1,,0,1786855131 +p2_ci_qwen36-think-offspec_v7,mmbt-qwen36-compare,p2_ci,qwen36-think-offspec,7,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,41,0.9512,7711,129.1,,0,1786861712 +p2_ci_qwen36-think-offspec_v8,mmbt-qwen36-compare,p2_ci,qwen36-think-offspec,8,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,40,0.975,6246,106.8,,0,1786867904 +p2_ci_qwen36-think-offspec_v9,mmbt-qwen36-compare,p2_ci,qwen36-think-offspec,9,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,42,0.9524,6629,115.6,,0,1786874134 +p2_extract_qwen36-think-offspec_v1,mmbt-qwen36-compare,p2_extract,qwen36-think-offspec,1,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,10,1.0,3911,61.0,,0,1786825049 +p2_extract_qwen36-think-offspec_v2,mmbt-qwen36-compare,p2_extract,qwen36-think-offspec,2,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,8,1.0,3392,53.2,,0,1786831555 +p2_extract_qwen36-think-offspec_v3,mmbt-qwen36-compare,p2_extract,qwen36-think-offspec,3,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,10,1.0,3854,59.8,,0,1786838092 +p2_extract_qwen36-think-offspec_v4,mmbt-qwen36-compare,p2_extract,qwen36-think-offspec,4,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,11,1.0,4216,65.4,,0,1786843438 +p2_extract_qwen36-think-offspec_v5,mmbt-qwen36-compare,p2_extract,qwen36-think-offspec,5,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,10,1.0,3865,60.1,,0,1786849764 +p2_extract_qwen36-think-offspec_v6,mmbt-qwen36-compare,p2_extract,qwen36-think-offspec,6,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,10,1.0,3860,60.7,,0,1786855013 +p2_extract_qwen36-think-offspec_v7,mmbt-qwen36-compare,p2_extract,qwen36-think-offspec,7,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,9,1.0,4025,62.4,,0,1786861582 +p2_extract_qwen36-think-offspec_v8,mmbt-qwen36-compare,p2_extract,qwen36-think-offspec,8,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,10,1.0,4065,63.0,,0,1786867797 +p2_extract_qwen36-think-offspec_v9,mmbt-qwen36-compare,p2_extract,qwen36-think-offspec,9,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,8,1.0,3392,53.3,,0,1786874018 +p2_hallucination_qwen36-think-offspec_v1,mmbt-qwen36-compare,p2_hallucination,qwen36-think-offspec,1,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,44,1.0,6674,108.9,,0,1786825276 +p2_hallucination_qwen36-think-offspec_v2,mmbt-qwen36-compare,p2_hallucination,qwen36-think-offspec,2,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,43,1.0,7234,117.8,,0,1786831795 +p2_hallucination_qwen36-think-offspec_v3,mmbt-qwen36-compare,p2_hallucination,qwen36-think-offspec,3,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,40,1.0,7620,123.1,,0,1786838327 +p2_hallucination_qwen36-think-offspec_v4,mmbt-qwen36-compare,p2_hallucination,qwen36-think-offspec,4,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,44,1.0,6867,111.1,,0,1786843654 +p2_hallucination_qwen36-think-offspec_v5,mmbt-qwen36-compare,p2_hallucination,qwen36-think-offspec,5,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,42,1.0,7427,120.1,,0,1786849998 +p2_hallucination_qwen36-think-offspec_v6,mmbt-qwen36-compare,p2_hallucination,qwen36-think-offspec,6,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,41,1.0,6726,109.7,,0,1786855242 +p2_hallucination_qwen36-think-offspec_v7,mmbt-qwen36-compare,p2_hallucination,qwen36-think-offspec,7,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,29,1.0,7685,124.0,,0,1786861837 +p2_hallucination_qwen36-think-offspec_v8,mmbt-qwen36-compare,p2_hallucination,qwen36-think-offspec,8,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,41,1.0,7220,116.2,,0,1786868021 +p2_hallucination_qwen36-think-offspec_v9,mmbt-qwen36-compare,p2_hallucination,qwen36-think-offspec,9,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,41,1.0,7239,118.4,,0,1786874254 +p2_triage_qwen36-think-offspec_v1,mmbt-qwen36-compare,p2_triage,qwen36-think-offspec,1,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,11,1.0,7681,116.9,,0,1786825394 +p2_triage_qwen36-think-offspec_v2,mmbt-qwen36-compare,p2_triage,qwen36-think-offspec,2,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,11,1.0,7681,116.8,,0,1786831912 +p2_triage_qwen36-think-offspec_v3,mmbt-qwen36-compare,p2_triage,qwen36-think-offspec,3,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,11,1.0,7677,116.4,,0,1786838444 +p2_triage_qwen36-think-offspec_v4,mmbt-qwen36-compare,p2_triage,qwen36-think-offspec,4,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,11,1.0,7681,116.8,,0,1786843771 +p2_triage_qwen36-think-offspec_v5,mmbt-qwen36-compare,p2_triage,qwen36-think-offspec,5,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,11,1.0,7681,117.6,,0,1786850116 +p2_triage_qwen36-think-offspec_v6,mmbt-qwen36-compare,p2_triage,qwen36-think-offspec,6,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,11,1.0,8070,123.1,,0,1786855366 +p2_triage_qwen36-think-offspec_v7,mmbt-qwen36-compare,p2_triage,qwen36-think-offspec,7,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,11,1.0,8054,122.3,,0,1786861960 +p2_triage_qwen36-think-offspec_v8,mmbt-qwen36-compare,p2_triage,qwen36-think-offspec,8,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,11,1.0,7680,116.1,,0,1786868138 +p2_triage_qwen36-think-offspec_v9,mmbt-qwen36-compare,p2_triage,qwen36-think-offspec,9,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,11,1.0,7694,117.0,,0,1786874371 +p3_business_qwen36-think-offspec_v1,mmbt-qwen36-compare,p3_business,qwen36-think-offspec,1,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,13,0.9231,8073,120.6,,0,1786825631 +p3_business_qwen36-think-offspec_v2,mmbt-qwen36-compare,p3_business,qwen36-think-offspec,2,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,14,0.9286,8137,122.5,,0,1786832147 +p3_business_qwen36-think-offspec_v3,mmbt-qwen36-compare,p3_business,qwen36-think-offspec,3,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,15,0.9333,8167,122.8,,0,1786838683 +p3_business_qwen36-think-offspec_v4,mmbt-qwen36-compare,p3_business,qwen36-think-offspec,4,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,14,0.9286,8137,121.9,,0,1786844007 +p3_business_qwen36-think-offspec_v5,mmbt-qwen36-compare,p3_business,qwen36-think-offspec,5,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,6,1,30,0.7,20274,312.5,,0,1786850542 +p3_business_qwen36-think-offspec_v6,mmbt-qwen36-compare,p3_business,qwen36-think-offspec,6,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,14,0.9286,8109,122.9,,0,1786855686 +p3_business_qwen36-think-offspec_v7,mmbt-qwen36-compare,p3_business,qwen36-think-offspec,7,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,14,1,59,0.6441,26452,417.7,,0,1786862491 +p3_business_qwen36-think-offspec_v8,mmbt-qwen36-compare,p3_business,qwen36-think-offspec,8,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,13,0.9231,8072,120.5,,0,1786868370 +p3_business_qwen36-think-offspec_v9,mmbt-qwen36-compare,p3_business,qwen36-think-offspec,9,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,14,0.9286,8132,122.4,,0,1786874662 +p3_doc_qwen36-think-offspec_v1,mmbt-qwen36-compare,p3_doc,qwen36-think-offspec,1,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,16,1.0,7495,114.5,,0,1786825510 +p3_doc_qwen36-think-offspec_v2,mmbt-qwen36-compare,p3_doc,qwen36-think-offspec,2,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,15,1.0,7231,110.4,,0,1786832024 +p3_doc_qwen36-think-offspec_v3,mmbt-qwen36-compare,p3_doc,qwen36-think-offspec,3,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,15,1.0,7501,114.1,,0,1786838559 +p3_doc_qwen36-think-offspec_v4,mmbt-qwen36-compare,p3_doc,qwen36-think-offspec,4,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,16,1.0,7304,111.9,,0,1786843884 +p3_doc_qwen36-think-offspec_v5,mmbt-qwen36-compare,p3_doc,qwen36-think-offspec,5,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,15,1.0,7251,111.4,,0,1786850229 +p3_doc_qwen36-think-offspec_v6,mmbt-qwen36-compare,p3_doc,qwen36-think-offspec,6,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,4,1,29,0.8276,12470,194.7,,0,1786855562 +p3_doc_qwen36-think-offspec_v7,mmbt-qwen36-compare,p3_doc,qwen36-think-offspec,7,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,15,1.0,7245,111.0,,0,1786862072 +p3_doc_qwen36-think-offspec_v8,mmbt-qwen36-compare,p3_doc,qwen36-think-offspec,8,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,14,1.0,7212,109.1,,0,1786868248 +p3_doc_qwen36-think-offspec_v9,mmbt-qwen36-compare,p3_doc,qwen36-think-offspec,9,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,4,1,24,0.8333,10832,166.6,,0,1786874539 +p3_market_qwen36-think-offspec_v1,mmbt-qwen36-compare,p3_market,qwen36-think-offspec,1,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,62,1.0,27798,621.1,,0,1786826253 +p3_market_qwen36-think-offspec_v2,mmbt-qwen36-compare,p3_market,qwen36-think-offspec,2,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,48,42,116,0.4483,,,identical-call-loop,0,1786833059 +p3_market_qwen36-think-offspec_v3,mmbt-qwen36-compare,p3_market,qwen36-think-offspec,3,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,139,1.0,27940,559.4,,0,1786839243 +p3_market_qwen36-think-offspec_v4,mmbt-qwen36-compare,p3_market,qwen36-think-offspec,4,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,91,1.0,24701,539.3,,0,1786844548 +p3_market_qwen36-think-offspec_v5,mmbt-qwen36-compare,p3_market,qwen36-think-offspec,5,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,68,1.0,25595,468.6,,0,1786851011 +p3_market_qwen36-think-offspec_v6,mmbt-qwen36-compare,p3_market,qwen36-think-offspec,6,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,13,13,57,0.7193,34225,1159.4,,0,1786856846 +p3_market_qwen36-think-offspec_v7,mmbt-qwen36-compare,p3_market,qwen36-think-offspec,7,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,8,8,94,0.9255,25825,500.0,,0,1786862992 +p3_market_qwen36-think-offspec_v8,mmbt-qwen36-compare,p3_market,qwen36-think-offspec,8,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,68,1.0,22840,454.2,,0,1786868825 +p3_market_qwen36-think-offspec_v9,mmbt-qwen36-compare,p3_market,qwen36-think-offspec,9,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,58,1.0,27207,520.2,,0,1786875183 +p3_pm_qwen36-think-offspec_v1,mmbt-qwen36-compare,p3_pm,qwen36-think-offspec,1,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,10,1.0,3885,59.9,,0,1786826415 +p3_pm_qwen36-think-offspec_v2,mmbt-qwen36-compare,p3_pm,qwen36-think-offspec,2,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,10,1.0,3711,57.3,,0,1786833205 +p3_pm_qwen36-think-offspec_v3,mmbt-qwen36-compare,p3_pm,qwen36-think-offspec,3,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,11,1.0,3771,58.4,,0,1786839429 +p3_pm_qwen36-think-offspec_v4,mmbt-qwen36-compare,p3_pm,qwen36-think-offspec,4,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,10,1.0,3712,57.2,,0,1786844733 +p3_pm_qwen36-think-offspec_v5,mmbt-qwen36-compare,p3_pm,qwen36-think-offspec,5,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,10,1.0,3640,56.2,,0,1786851162 +p3_pm_qwen36-think-offspec_v6,mmbt-qwen36-compare,p3_pm,qwen36-think-offspec,6,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,13,1.0,5066,78.6,,0,1786857038 +p3_pm_qwen36-think-offspec_v7,mmbt-qwen36-compare,p3_pm,qwen36-think-offspec,7,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,11,1.0,3729,58.0,,0,1786863189 +p3_pm_qwen36-think-offspec_v8,mmbt-qwen36-compare,p3_pm,qwen36-think-offspec,8,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,2,1,19,0.9474,6375,98.8,,0,1786869019 +p3_pm_qwen36-think-offspec_v9,mmbt-qwen36-compare,p3_pm,qwen36-think-offspec,9,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,10,1.0,3650,56.5,,0,1786875337 +p3_writing_qwen36-think-offspec_v1,mmbt-qwen36-compare,p3_writing,qwen36-think-offspec,1,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,20,1.0,6315,99.7,,0,1786826354 +p3_writing_qwen36-think-offspec_v2,mmbt-qwen36-compare,p3_writing,qwen36-think-offspec,2,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,15,1.0,5537,86.7,,0,1786833147 +p3_writing_qwen36-think-offspec_v3,mmbt-qwen36-compare,p3_writing,qwen36-think-offspec,3,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,22,1.0,8016,124.7,,0,1786839369 +p3_writing_qwen36-think-offspec_v4,mmbt-qwen36-compare,p3_writing,qwen36-think-offspec,4,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,25,1.0,8056,125.6,,0,1786844675 +p3_writing_qwen36-think-offspec_v5,mmbt-qwen36-compare,p3_writing,qwen36-think-offspec,5,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,18,1.0,5919,92.2,,0,1786851105 +p3_writing_qwen36-think-offspec_v6,mmbt-qwen36-compare,p3_writing,qwen36-think-offspec,6,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,18,1.0,7025,110.4,,0,1786856958 +p3_writing_qwen36-think-offspec_v7,mmbt-qwen36-compare,p3_writing,qwen36-think-offspec,7,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,26,1.0,8774,136.4,,0,1786863130 +p3_writing_qwen36-think-offspec_v8,mmbt-qwen36-compare,p3_writing,qwen36-think-offspec,8,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,18,1.0,5961,92.8,,0,1786868919 +p3_writing_qwen36-think-offspec_v9,mmbt-qwen36-compare,p3_writing,qwen36-think-offspec,9,3.6,UD-Q4_K_XL,think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,18,1.0,6097,95.2,,0,1786875279 +p1_bugfix_qwen36-think-card_v1,mmbt-q36-card,p1_bugfix,qwen36-think-card,1,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,3,1,107,0.9439,33252,913.1,,0,1786822605 +p1_bugfix_qwen36-think-card_v2,mmbt-q36-card,p1_bugfix,qwen36-think-card,2,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,6,1,155,0.9032,38529,1354.0,,0,1786829661 +p1_bugfix_qwen36-think-card_v3,mmbt-q36-card,p1_bugfix,qwen36-think-card,3,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,5,1,143,0.9231,36636,974.5,,0,1786835711 +p1_bugfix_qwen36-think-card_v4,mmbt-q36-card,p1_bugfix,qwen36-think-card,4,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,6,1,101,0.8911,28523,837.9,,0,1786841749 +p1_bugfix_qwen36-think-card_v5,mmbt-q36-card,p1_bugfix,qwen36-think-card,5,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,5,1,114,0.9298,32408,908.5,,0,1786847441 +p1_bugfix_qwen36-think-card_v6,mmbt-q36-card,p1_bugfix,qwen36-think-card,6,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,5,1,119,0.9412,48063,1204.1,,0,1786853449 +p1_bugfix_qwen36-think-card_v7,mmbt-q36-card,p1_bugfix,qwen36-think-card,7,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,7,1,132,0.9091,34365,764.4,,0,1786859434 +p1_bugfix_qwen36-think-card_v8,mmbt-q36-card,p1_bugfix,qwen36-think-card,8,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,8,1,142,0.9085,35182,971.3,,0,1786866389 +p1_bugfix_qwen36-think-card_v9,mmbt-q36-card,p1_bugfix,qwen36-think-card,9,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,5,1,109,0.9358,34903,708.2,,0,1786873274 +p1_bugfix_qwen36-think-card_v10,mmbt-q36-card,p1_bugfix,qwen36-think-card,10,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,5,1,120,0.9,31438,893.6,,0,1786880879 +p1_refactor_qwen36-think-card_v1,mmbt-q36-card,p1_refactor,qwen36-think-card,1,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,85,0.9765,12903,234.4,,0,1786823383 +p1_refactor_qwen36-think-card_v2,mmbt-q36-card,p1_refactor,qwen36-think-card,2,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,78,1.0,15450,270.9,,0,1786830818 +p1_refactor_qwen36-think-card_v3,mmbt-q36-card,p1_refactor,qwen36-think-card,3,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,58,0.9828,11060,192.8,,0,1786836597 +p1_refactor_qwen36-think-card_v4,mmbt-q36-card,p1_refactor,qwen36-think-card,4,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,4,1,79,0.9367,12719,229.0,,0,1786842502 +p1_refactor_qwen36-think-card_v5,mmbt-q36-card,p1_refactor,qwen36-think-card,5,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,3,1,80,0.975,12756,233.7,,0,1786848358 +p1_refactor_qwen36-think-card_v6,mmbt-q36-card,p1_refactor,qwen36-think-card,6,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,3,1,110,0.9091,16018,294.1,,0,1786854537 +p1_refactor_qwen36-think-card_v7,mmbt-q36-card,p1_refactor,qwen36-think-card,7,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,3,1,89,0.9663,15296,273.9,,0,1786860180 +p1_refactor_qwen36-think-card_v8,mmbt-q36-card,p1_refactor,qwen36-think-card,8,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,76,0.9868,12666,224.7,,0,1786867308 +p1_refactor_qwen36-think-card_v9,mmbt-q36-card,p1_refactor,qwen36-think-card,9,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,5,1,90,0.8778,13576,549.6,,0,1786874272 +p1_refactor_qwen36-think-card_v10,mmbt-q36-card,p1_refactor,qwen36-think-card,10,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,81,0.963,12327,222.4,,0,1786881738 +p1_testwrite_qwen36-think-card_v1,mmbt-q36-card,p1_testwrite,qwen36-think-card,1,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,6,1,60,0.9167,31435,539.9,,0,1786823148 +p1_testwrite_qwen36-think-card_v2,mmbt-q36-card,p1_testwrite,qwen36-think-card,2,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,8,1,75,0.9067,49731,884.2,,0,1786830546 +p1_testwrite_qwen36-think-card_v3,mmbt-q36-card,p1_testwrite,qwen36-think-card,3,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,70,0.9857,40026,689.3,,0,1786836403 +p1_testwrite_qwen36-think-card_v4,mmbt-q36-card,p1_testwrite,qwen36-think-card,4,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,5,1,60,0.9333,30967,521.4,,0,1786842272 +p1_testwrite_qwen36-think-card_v5,mmbt-q36-card,p1_testwrite,qwen36-think-card,5,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,5,1,57,0.9298,40064,681.0,,0,1786848123 +p1_testwrite_qwen36-think-card_v6,mmbt-q36-card,p1_testwrite,qwen36-think-card,6,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,86,0.9767,45349,791.1,,0,1786854242 +p1_testwrite_qwen36-think-card_v7,mmbt-q36-card,p1_testwrite,qwen36-think-card,7,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,5,1,61,0.918,27957,467.9,,0,1786859905 +p1_testwrite_qwen36-think-card_v8,mmbt-q36-card,p1_testwrite,qwen36-think-card,8,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,6,1,61,0.918,40482,691.2,,0,1786867082 +p1_testwrite_qwen36-think-card_v9,mmbt-q36-card,p1_testwrite,qwen36-think-card,9,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,67,1.0,26282,445.8,,0,1786873721 +p1_testwrite_qwen36-think-card_v10,mmbt-q36-card,p1_testwrite,qwen36-think-card,10,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,5,1,63,0.9365,37174,633.0,,0,1786881515 +p2_ci_qwen36-think-card_v1,mmbt-q36-card,p2_ci,qwen36-think-card,1,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,43,0.9767,6815,118.8,,0,1786823579 +p2_ci_qwen36-think-card_v2,mmbt-q36-card,p2_ci,qwen36-think-card,2,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,40,1.0,6023,102.7,,0,1786830990 +p2_ci_qwen36-think-card_v3,mmbt-q36-card,p2_ci,qwen36-think-card,3,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,3,1,39,0.8974,7837,127.1,,0,1786836798 +p2_ci_qwen36-think-card_v4,mmbt-q36-card,p2_ci,qwen36-think-card,4,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,43,0.9767,7070,122.4,,0,1786842701 +p2_ci_qwen36-think-card_v5,mmbt-q36-card,p2_ci,qwen36-think-card,5,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,39,0.9487,5883,103.6,,0,1786848536 +p2_ci_qwen36-think-card_v6,mmbt-q36-card,p2_ci,qwen36-think-card,6,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,39,0.9744,6370,108.2,,0,1786854723 +p2_ci_qwen36-think-card_v7,mmbt-q36-card,p2_ci,qwen36-think-card,7,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,3,1,45,0.9111,9704,161.1,,0,1786860416 +p2_ci_qwen36-think-card_v8,mmbt-q36-card,p2_ci,qwen36-think-card,8,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,32,1.0,6014,100.2,,0,1786867488 +p2_ci_qwen36-think-card_v9,mmbt-q36-card,p2_ci,qwen36-think-card,9,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,3,1,51,0.8824,6784,120.6,,0,1786874463 +p2_ci_qwen36-think-card_v10,mmbt-q36-card,p2_ci,qwen36-think-card,10,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,4,1,47,0.8723,7145,124.6,,0,1786881931 +p2_extract_qwen36-think-card_v1,mmbt-q36-card,p2_extract,qwen36-think-card,1,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,12,1.0,4782,75.1,,0,1786823459 +p2_extract_qwen36-think-card_v2,mmbt-q36-card,p2_extract,qwen36-think-card,2,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,11,1.0,4451,68.4,,0,1786830887 +p2_extract_qwen36-think-card_v3,mmbt-q36-card,p2_extract,qwen36-think-card,3,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,12,1.0,4683,72.8,,0,1786836670 +p2_extract_qwen36-think-card_v4,mmbt-q36-card,p2_extract,qwen36-think-card,4,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,12,1.0,4738,74.4,,0,1786842578 +p2_extract_qwen36-think-card_v5,mmbt-q36-card,p2_extract,qwen36-think-card,5,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,12,1.0,4706,73.4,,0,1786848432 +p2_extract_qwen36-think-card_v6,mmbt-q36-card,p2_extract,qwen36-think-card,6,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,12,1.0,4804,75.0,,0,1786854613 +p2_extract_qwen36-think-card_v7,mmbt-q36-card,p2_extract,qwen36-think-card,7,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,12,1.0,4743,74.1,,0,1786860255 +p2_extract_qwen36-think-card_v8,mmbt-q36-card,p2_extract,qwen36-think-card,8,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,13,1.0,4987,77.8,,0,1786867387 +p2_extract_qwen36-think-card_v9,mmbt-q36-card,p2_extract,qwen36-think-card,9,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,11,1.0,4444,68.0,,0,1786874342 +p2_extract_qwen36-think-card_v10,mmbt-q36-card,p2_extract,qwen36-think-card,10,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,10,1.0,4346,66.7,,0,1786881806 +p2_hallucination_qwen36-think-card_v1,mmbt-q36-card,p2_hallucination,qwen36-think-card,1,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,37,1.0,7196,117.6,,0,1786823697 +p2_hallucination_qwen36-think-card_v2,mmbt-q36-card,p2_hallucination,qwen36-think-card,2,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,40,1.0,7113,116.0,,0,1786831107 +p2_hallucination_qwen36-think-card_v3,mmbt-q36-card,p2_hallucination,qwen36-think-card,3,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,49,1.0,8083,132.6,,0,1786836932 +p2_hallucination_qwen36-think-card_v4,mmbt-q36-card,p2_hallucination,qwen36-think-card,4,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,48,1.0,9208,149.1,,0,1786842851 +p2_hallucination_qwen36-think-card_v5,mmbt-q36-card,p2_hallucination,qwen36-think-card,5,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,43,1.0,7699,129.1,,0,1786848666 +p2_hallucination_qwen36-think-card_v6,mmbt-q36-card,p2_hallucination,qwen36-think-card,6,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,44,1.0,6381,106.0,,0,1786854830 +p2_hallucination_qwen36-think-card_v7,mmbt-q36-card,p2_hallucination,qwen36-think-card,7,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,41,1.0,6700,109.6,,0,1786860527 +p2_hallucination_qwen36-think-card_v8,mmbt-q36-card,p2_hallucination,qwen36-think-card,8,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,47,1.0,7443,122.9,,0,1786867611 +p2_hallucination_qwen36-think-card_v9,mmbt-q36-card,p2_hallucination,qwen36-think-card,9,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,44,0.9773,6174,103.7,,0,1786874568 +p2_hallucination_qwen36-think-card_v10,mmbt-q36-card,p2_hallucination,qwen36-think-card,10,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,46,1.0,7934,129.8,,0,1786882061 +p2_triage_qwen36-think-card_v1,mmbt-q36-card,p2_triage,qwen36-think-card,1,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,12,1.0,9282,140.9,,0,1786823839 +p2_triage_qwen36-think-card_v2,mmbt-q36-card,p2_triage,qwen36-think-card,2,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,12,1.0,9615,146.9,,0,1786831255 +p2_triage_qwen36-think-card_v3,mmbt-q36-card,p2_triage,qwen36-think-card,3,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,12,1.0,9188,138.1,,0,1786837071 +p2_triage_qwen36-think-card_v4,mmbt-q36-card,p2_triage,qwen36-think-card,4,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,12,1.0,9313,141.8,,0,1786842994 +p2_triage_qwen36-think-card_v5,mmbt-q36-card,p2_triage,qwen36-think-card,5,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,12,1.0,9288,141.1,,0,1786848808 +p2_triage_qwen36-think-card_v6,mmbt-q36-card,p2_triage,qwen36-think-card,6,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,12,1.0,9150,138.4,,0,1786854969 +p2_triage_qwen36-think-card_v7,mmbt-q36-card,p2_triage,qwen36-think-card,7,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,12,1.0,9209,139.1,,0,1786860667 +p2_triage_qwen36-think-card_v8,mmbt-q36-card,p2_triage,qwen36-think-card,8,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,13,1.0,10094,154.6,,0,1786867767 +p2_triage_qwen36-think-card_v9,mmbt-q36-card,p2_triage,qwen36-think-card,9,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,12,1.0,9337,140.5,,0,1786874709 +p2_triage_qwen36-think-card_v10,mmbt-q36-card,p2_triage,qwen36-think-card,10,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,12,1.0,9215,139.1,,0,1786882201 +p3_business_qwen36-think-card_v1,mmbt-q36-card,p3_business,qwen36-think-card,1,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,19,0.9474,12157,184.6,,0,1786824213 +p3_business_qwen36-think-card_v2,mmbt-q36-card,p3_business,qwen36-think-card,2,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,19,0.9474,12145,184.5,,0,1786831623 +p3_business_qwen36-think-card_v3,mmbt-q36-card,p3_business,qwen36-think-card,3,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,17,0.9412,11934,180.3,,0,1786837413 +p3_business_qwen36-think-card_v4,mmbt-q36-card,p3_business,qwen36-think-card,4,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,15,0.9333,11408,172.1,,0,1786843328 +p3_business_qwen36-think-card_v5,mmbt-q36-card,p3_business,qwen36-think-card,5,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,17,0.9412,11516,175.6,,0,1786849147 +p3_business_qwen36-think-card_v6,mmbt-q36-card,p3_business,qwen36-think-card,6,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,19,1.0,12841,194.7,,0,1786855277 +p3_business_qwen36-think-card_v7,mmbt-q36-card,p3_business,qwen36-think-card,7,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,16,0.9375,11662,175.6,,0,1786861030 +p3_business_qwen36-think-card_v8,mmbt-q36-card,p3_business,qwen36-think-card,8,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,18,0.9444,12399,188.2,,0,1786868069 +p3_business_qwen36-think-card_v9,mmbt-q36-card,p3_business,qwen36-think-card,9,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,17,0.9412,12414,188.0,,0,1786875063 +p3_business_qwen36-think-card_v10,mmbt-q36-card,p3_business,qwen36-think-card,10,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,16,0.9375,12059,181.4,,0,1786882504 +p3_doc_qwen36-think-card_v1,mmbt-q36-card,p3_doc,qwen36-think-card,1,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,3,1,21,0.9048,12228,187.6,,0,1786824028 +p3_doc_qwen36-think-card_v2,mmbt-q36-card,p3_doc,qwen36-think-card,2,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,3,1,26,0.8462,11701,181.3,,0,1786831437 +p3_doc_qwen36-think-card_v3,mmbt-q36-card,p3_doc,qwen36-think-card,3,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,18,0.9444,10543,160.4,,0,1786837232 +p3_doc_qwen36-think-card_v4,mmbt-q36-card,p3_doc,qwen36-think-card,4,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,3,1,18,0.8889,10481,160.1,,0,1786843155 +p3_doc_qwen36-think-card_v5,mmbt-q36-card,p3_doc,qwen36-think-card,5,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,18,0.9444,10541,161.4,,0,1786848970 +p3_doc_qwen36-think-card_v6,mmbt-q36-card,p3_doc,qwen36-think-card,6,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,15,0.9333,7254,111.0,,0,1786855081 +p3_doc_qwen36-think-card_v7,mmbt-q36-card,p3_doc,qwen36-think-card,7,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,3,1,19,0.8947,12108,185.7,,0,1786860854 +p3_doc_qwen36-think-card_v8,mmbt-q36-card,p3_doc,qwen36-think-card,8,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,16,0.9375,7342,112.6,,0,1786867880 +p3_doc_qwen36-think-card_v9,mmbt-q36-card,p3_doc,qwen36-think-card,9,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,19,0.9474,10710,164.1,,0,1786874874 +p3_doc_qwen36-think-card_v10,mmbt-q36-card,p3_doc,qwen36-think-card,10,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,17,0.9412,7744,119.8,,0,1786882322 +p3_market_qwen36-think-card_v1,mmbt-q36-card,p3_market,qwen36-think-card,1,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,75,1.0,30721,1349.0,,0,1786825563 +p3_market_qwen36-think-card_v2,mmbt-q36-card,p3_market,qwen36-think-card,2,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,90,1.0,32770,592.9,,0,1786832216 +p3_market_qwen36-think-card_v3,mmbt-q36-card,p3_market,qwen36-think-card,3,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,55,1.0,21519,477.5,,0,1786837892 +p3_market_qwen36-think-card_v4,mmbt-q36-card,p3_market,qwen36-think-card,4,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,71,1.0,26318,518.7,,0,1786843848 +p3_market_qwen36-think-card_v5,mmbt-q36-card,p3_market,qwen36-think-card,5,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,49,1.0,39102,727.0,,0,1786849875 +p3_market_qwen36-think-card_v6,mmbt-q36-card,p3_market,qwen36-think-card,6,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,68,1.0,25398,891.2,,0,1786856169 +p3_market_qwen36-think-card_v7,mmbt-q36-card,p3_market,qwen36-think-card,7,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,77,1.0,30161,582.4,,0,1786861613 +p3_market_qwen36-think-card_v8,mmbt-q36-card,p3_market,qwen36-think-card,8,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,105,1.0,34651,724.6,,0,1786868795 +p3_market_qwen36-think-card_v9,mmbt-q36-card,p3_market,qwen36-think-card,9,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,50,1.0,25364,478.0,,0,1786875542 +p3_market_qwen36-think-card_v10,mmbt-q36-card,p3_market,qwen36-think-card,10,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,84,1.0,37281,675.2,,0,1786883180 +p3_pm_qwen36-think-card_v1,mmbt-q36-card,p3_pm,qwen36-think-card,1,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,14,1.0,4813,75.0,,0,1786825758 +p3_pm_qwen36-think-card_v2,mmbt-q36-card,p3_pm,qwen36-think-card,2,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,14,0.9286,5094,78.9,,0,1786832442 +p3_pm_qwen36-think-card_v3,mmbt-q36-card,p3_pm,qwen36-think-card,3,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,14,1.0,5013,78.6,,0,1786838102 +p3_pm_qwen36-think-card_v4,mmbt-q36-card,p3_pm,qwen36-think-card,4,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,14,0.9286,5181,80.1,,0,1786844098 +p3_pm_qwen36-think-card_v5,mmbt-q36-card,p3_pm,qwen36-think-card,5,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,15,0.9333,4979,77.7,,0,1786850104 +p3_pm_qwen36-think-card_v6,mmbt-q36-card,p3_pm,qwen36-think-card,6,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,15,0.9333,5073,79.0,,0,1786856374 +p3_pm_qwen36-think-card_v7,mmbt-q36-card,p3_pm,qwen36-think-card,7,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,15,0.9333,5027,78.2,,0,1786861783 +p3_pm_qwen36-think-card_v8,mmbt-q36-card,p3_pm,qwen36-think-card,8,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,13,1.0,3992,62.5,,0,1786868985 +p3_pm_qwen36-think-card_v9,mmbt-q36-card,p3_pm,qwen36-think-card,9,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,16,0.9375,5798,90.0,,0,1786875759 +p3_pm_qwen36-think-card_v10,mmbt-q36-card,p3_pm,qwen36-think-card,10,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,13,1.0,4733,73.5,,0,1786883388 +p3_writing_qwen36-think-card_v1,mmbt-q36-card,p3_writing,qwen36-think-card,1,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,23,0.9565,7427,118.2,,0,1786825682 +p3_writing_qwen36-think-card_v2,mmbt-q36-card,p3_writing,qwen36-think-card,2,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,27,0.963,9148,144.8,,0,1786832363 +p3_writing_qwen36-think-card_v3,mmbt-q36-card,p3_writing,qwen36-think-card,3,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,28,0.9643,8107,129.6,,0,1786838022 +p3_writing_qwen36-think-card_v4,mmbt-q36-card,p3_writing,qwen36-think-card,4,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,35,0.9429,10487,167.8,,0,1786844017 +p3_writing_qwen36-think-card_v5,mmbt-q36-card,p3_writing,qwen36-think-card,5,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,30,0.9,9395,149.4,,0,1786850025 +p3_writing_qwen36-think-card_v6,mmbt-q36-card,p3_writing,qwen36-think-card,6,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,19,0.9474,7880,123.5,,0,1786856294 +p3_writing_qwen36-think-card_v7,mmbt-q36-card,p3_writing,qwen36-think-card,7,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,17,1.0,5760,89.8,,0,1786861704 +p3_writing_qwen36-think-card_v8,mmbt-q36-card,p3_writing,qwen36-think-card,8,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,21,1.0,7940,125.4,,0,1786868921 +p3_writing_qwen36-think-card_v9,mmbt-q36-card,p3_writing,qwen36-think-card,9,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,3,1,25,0.88,7953,125.5,,0,1786875668 +p3_writing_qwen36-think-card_v10,mmbt-q36-card,p3_writing,qwen36-think-card,10,3.6,UD-Q4_K_XL,think,,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,24,0.875,8535,133.4,,0,1786883314 +p1_bugfix_qwen38-nothink-offspec_v1,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-nothink-offspec,1,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,0,0,1,1,55,1.0,,,identical-call-loop,0,1786821211 +p1_bugfix_qwen38q8-nothink-matched_v1,mmbt-q38-q8,p1_bugfix,qwen38q8-nothink-matched,1,3.8,Q8_0,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,81,81,97,0.1753,,,identical-call-loop,0,1786877163 +p1_bugfix_qwen38-nothink-offspec_v2,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-nothink-offspec,2,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,45,45,65,0.3077,,,identical-call-loop,0,1786831918 +p1_bugfix_qwen38q8-nothink-matched_v2,mmbt-q38-q8,p1_bugfix,qwen38q8-nothink-matched,2,3.8,Q8_0,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,0,0,1,1,148,1.0,,,identical-call-loop,0,1786882446 +p1_bugfix_qwen38-nothink-offspec_v3,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-nothink-offspec,3,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,129,129,155,0.1742,,,identical-call-loop,0,1786811461 +p1_bugfix_qwen38-27b-udq4xl-t03-nothink_v4,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-27b-udq4xl-t03-nothink,4,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,85,85,126,0.3333,,,identical-call-loop,0,1786804239 +p1_bugfix_qwen38-nothink-offspec_v4,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-nothink-offspec,4,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,51,51,84,0.4048,,,identical-call-loop,0,1786871379 +p1_bugfix_qwen38-27b-udq4xl-t03-nothink_v5,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-27b-udq4xl-t03-nothink,5,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,120,119,132,0.0985,,,identical-call-loop,0,1786804299 +p1_bugfix_qwen38-nothink-offspec_v5,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-nothink-offspec,5,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,88,88,140,0.3786,,,identical-call-loop,0,1786877874 +p1_bugfix_qwen38-27b-udq4xl-t03-nothink_v6,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-27b-udq4xl-t03-nothink,6,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,137,137,155,0.1226,,,identical-call-loop,0,1786804570 +p1_bugfix_qwen38-nothink-offspec_v6,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-nothink-offspec,6,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,1,0,319,5,515,0.0388,102609,4860.3,,0,1786885221 +p1_bugfix_qwen38-27b-udq4xl-t03-nothink_v7,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-27b-udq4xl-t03-nothink,7,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,128,128,144,0.1181,,,identical-call-loop,0,1786805233 +p1_bugfix_qwen38-nothink-offspec_v7,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-nothink-offspec,7,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,93,91,118,0.1949,,,identical-call-loop,0,1786888322 +p1_bugfix_qwen38-27b-udq4xl-t03-nothink_v8,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-27b-udq4xl-t03-nothink,8,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,0,0,28,28,68,0.6029,,,identical-call-loop,0,1786806407 +p1_refactor_qwen38-nothink-offspec_v1,mmbt-qwen38-eaaa8ca,p1_refactor,qwen38-nothink-offspec,1,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,47,1.0,11468,191.0,,0,1786817435 +p1_refactor_qwen38q8-nothink-matched_v1,mmbt-q38-q8,p1_refactor,qwen38q8-nothink-matched,1,3.8,Q8_0,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,41,1.0,11435,259.2,,0,1786877755 +p1_refactor_qwen38-nothink-offspec_v2,mmbt-qwen38-eaaa8ca,p1_refactor,qwen38-nothink-offspec,2,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,0,0,24,1,84,0.7143,,,identical-call-loop,0,1786832586 +p1_refactor_qwen38q8-nothink-matched_v2,mmbt-q38-q8,p1_refactor,qwen38q8-nothink-matched,2,3.8,Q8_0,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,110,110,130,0.1615,,,identical-call-loop,0,1786883352 +p1_refactor_qwen38-nothink-offspec_v3,mmbt-qwen38-eaaa8ca,p1_refactor,qwen38-nothink-offspec,3,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,49,1.0,37773,633.4,,0,1786864266 +p1_refactor_qwen38-nothink-offspec_v4,mmbt-qwen38-eaaa8ca,p1_refactor,qwen38-nothink-offspec,4,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,51,51,91,0.4505,,,identical-call-loop,0,1786872346 +p1_refactor_qwen38-nothink-offspec_v5,mmbt-qwen38-eaaa8ca,p1_refactor,qwen38-nothink-offspec,5,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,42,1.0,8140,138.1,,0,1786878347 +p1_refactor_qwen38-nothink-offspec_v6,mmbt-qwen38-eaaa8ca,p1_refactor,qwen38-nothink-offspec,6,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,55,1.0,11988,207.0,,0,1786885752 +p1_refactor_qwen38-nothink-offspec_v7,mmbt-qwen38-eaaa8ca,p1_refactor,qwen38-nothink-offspec,7,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,48,1.0,11255,195.0,,0,1786889038 +p1_testwrite_qwen38-27b-udq4xl-t03-nothink_v1,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-27b-udq4xl-t03-nothink,1,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,70,70,87,0.2069,,,identical-call-loop,0,1786807249 +p1_testwrite_qwen38-nothink-offspec_v1,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-nothink-offspec,1,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,118,118,132,0.1136,,,identical-call-loop,0,1786821542 +p1_testwrite_qwen38q8-nothink-matched_v1,mmbt-q38-q8,p1_testwrite,qwen38q8-nothink-matched,1,3.8,Q8_0,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,71,71,87,0.1954,,,identical-call-loop,0,1786877494 +p1_testwrite_qwen38-27b-udq4xl-t03-nothink_v2,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-27b-udq4xl-t03-nothink,2,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,1,0,119,5,251,0.0637,42244,851.2,,0,1786807592 +p1_testwrite_qwen38-nothink-offspec_v2,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-nothink-offspec,2,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,149,149,162,0.0864,,,identical-call-loop,0,1786832254 +p1_testwrite_qwen38q8-nothink-matched_v2,mmbt-q38-q8,p1_testwrite,qwen38q8-nothink-matched,2,3.8,Q8_0,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,0,121,1,253,0.0474,16665,597.3,,0,1786883046 +p1_testwrite_qwen38-27b-udq4xl-t03-nothink_v3,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-27b-udq4xl-t03-nothink,3,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,80,80,99,0.1919,,,identical-call-loop,0,1786807583 +p1_testwrite_qwen38-nothink-offspec_v3,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-nothink-offspec,3,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,0,0,7,1,115,0.5739,,,identical-call-loop,0,1786863631 +p1_testwrite_qwen38-27b-udq4xl-t03-nothink_v4,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-27b-udq4xl-t03-nothink,4,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,0,1,0,73,2,211,0.3033,,,,0,1786808037 +p1_testwrite_qwen38-nothink-offspec_v4,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-nothink-offspec,4,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,102,102,130,0.2154,,,identical-call-loop,0,1786871716 +p1_testwrite_qwen38-27b-udq4xl-t03-nothink_v5,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-27b-udq4xl-t03-nothink,5,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,0,30,1,113,0.4956,,,identical-call-loop,0,1786807914 +p1_testwrite_qwen38-nothink-offspec_v5,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-nothink-offspec,5,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,43,43,70,0.4,,,identical-call-loop,0,1786878206 +p1_testwrite_qwen38-nothink-offspec_v6,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-nothink-offspec,6,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,79,79,100,0.21,,,identical-call-loop,0,1786885541 +p1_testwrite_qwen38-27b-udq4xl-t03-nothink_v7,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-27b-udq4xl-t03-nothink,7,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,0,0,0,2,1,34,0.9412,,,,0,1786808035 +p1_testwrite_qwen38-nothink-offspec_v7,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-nothink-offspec,7,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,254,1.0,18947,516.9,,0,1786888842 +p2_ci_qwen38-nothink-offspec_v1,mmbt-qwen38-eaaa8ca,p2_ci,qwen38-nothink-offspec,1,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,15,1.0,3584,67.4,,0,1786817547 +p2_ci_qwen38q8-nothink-matched_v1,mmbt-q38-q8,p2_ci,qwen38q8-nothink-matched,1,3.8,Q8_0,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,19,1.0,4136,96.2,,0,1786877906 +p2_ci_qwen38-nothink-offspec_v2,mmbt-qwen38-eaaa8ca,p2_ci,qwen38-nothink-offspec,2,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,18,1.0,4099,75.9,,0,1786832707 +p2_ci_qwen38q8-nothink-matched_v2,mmbt-q38-q8,p2_ci,qwen38q8-nothink-matched,2,3.8,Q8_0,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,0,0,1,1,20,1.0,4160,98.7,,0,1786883507 +p2_ci_qwen38-nothink-offspec_v3,mmbt-qwen38-eaaa8ca,p2_ci,qwen38-nothink-offspec,3,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,16,1.0,3598,68.9,,0,1786864379 +p2_ci_qwen38-nothink-offspec_v4,mmbt-qwen38-eaaa8ca,p2_ci,qwen38-nothink-offspec,4,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,19,1.0,3748,71.7,,0,1786872468 +p2_ci_qwen38-nothink-offspec_v5,mmbt-qwen38-eaaa8ca,p2_ci,qwen38-nothink-offspec,5,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,17,1.0,3382,65.0,,0,1786878456 +p2_ci_qwen38-nothink-offspec_v6,mmbt-qwen38-eaaa8ca,p2_ci,qwen38-nothink-offspec,6,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,18,1.0,3954,73.9,,0,1786885870 +p2_ci_qwen38-nothink-offspec_v7,mmbt-qwen38-eaaa8ca,p2_ci,qwen38-nothink-offspec,7,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,20,1.0,3947,75.6,,0,1786889157 +p2_extract_qwen38-nothink-offspec_v1,mmbt-qwen38-eaaa8ca,p2_extract,qwen38-nothink-offspec,1,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,10,1.0,2731,42.9,,0,1786817479 +p2_extract_qwen38q8-nothink-matched_v1,mmbt-q38-q8,p2_extract,qwen38q8-nothink-matched,1,3.8,Q8_0,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,10,1.0,2422,53.0,,0,1786877809 +p2_extract_qwen38-nothink-offspec_v2,mmbt-qwen38-eaaa8ca,p2_extract,qwen38-nothink-offspec,2,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,10,1.0,2664,42.7,,0,1786832631 +p2_extract_qwen38q8-nothink-matched_v2,mmbt-q38-q8,p2_extract,qwen38q8-nothink-matched,2,3.8,Q8_0,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,0,0,1,1,10,1.0,2435,54.5,,0,1786883407 +p2_extract_qwen38-nothink-offspec_v3,mmbt-qwen38-eaaa8ca,p2_extract,qwen38-nothink-offspec,3,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,10,1.0,2647,42.8,,0,1786864309 +p2_extract_qwen38-nothink-offspec_v4,mmbt-qwen38-eaaa8ca,p2_extract,qwen38-nothink-offspec,4,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,10,1.0,2675,43.9,,0,1786872396 +p2_extract_qwen38-nothink-offspec_v5,mmbt-qwen38-eaaa8ca,p2_extract,qwen38-nothink-offspec,5,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,10,1.0,2733,42.8,,0,1786878390 +p2_extract_qwen38-nothink-offspec_v6,mmbt-qwen38-eaaa8ca,p2_extract,qwen38-nothink-offspec,6,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,10,1.0,2666,42.1,,0,1786885795 +p2_extract_qwen38-nothink-offspec_v7,mmbt-qwen38-eaaa8ca,p2_extract,qwen38-nothink-offspec,7,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,10,1.0,2618,42.2,,0,1786889081 +p2_hallucination_qwen38-nothink-offspec_v1,mmbt-qwen38-eaaa8ca,p2_hallucination,qwen38-nothink-offspec,1,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,18,1.0,5075,83.9,,0,1786817632 +p2_hallucination_qwen38q8-nothink-matched_v1,mmbt-q38-q8,p2_hallucination,qwen38q8-nothink-matched,1,3.8,Q8_0,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,16,1.0,4316,97.0,,0,1786878003 +p2_hallucination_qwen38-nothink-offspec_v2,mmbt-qwen38-eaaa8ca,p2_hallucination,qwen38-nothink-offspec,2,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,20,1.0,5057,84.7,,0,1786832793 +p2_hallucination_qwen38q8-nothink-matched_v2,mmbt-q38-q8,p2_hallucination,qwen38q8-nothink-matched,2,3.8,Q8_0,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,0,0,1,1,17,1.0,4446,101.5,,0,1786883609 +p2_hallucination_qwen38-nothink-offspec_v3,mmbt-qwen38-eaaa8ca,p2_hallucination,qwen38-nothink-offspec,3,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,14,1.0,4377,72.4,,0,1786864453 +p2_hallucination_qwen38-nothink-offspec_v4,mmbt-qwen38-eaaa8ca,p2_hallucination,qwen38-nothink-offspec,4,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,13,1.0,4434,72.9,,0,1786872542 +p2_hallucination_qwen38-nothink-offspec_v5,mmbt-qwen38-eaaa8ca,p2_hallucination,qwen38-nothink-offspec,5,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,14,1.0,4216,69.5,,0,1786878526 +p2_hallucination_qwen38-nothink-offspec_v6,mmbt-qwen38-eaaa8ca,p2_hallucination,qwen38-nothink-offspec,6,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,14,1.0,4332,71.3,,0,1786885942 +p2_hallucination_qwen38-nothink-offspec_v7,mmbt-qwen38-eaaa8ca,p2_hallucination,qwen38-nothink-offspec,7,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,13,1.0,3588,60.7,,0,1786889219 +p2_triage_qwen38-nothink-offspec_v1,mmbt-qwen38-eaaa8ca,p2_triage,qwen38-nothink-offspec,1,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,15,1.0,6068,93.7,,0,1786817726 +p2_triage_qwen38q8-nothink-matched_v1,mmbt-q38-q8,p2_triage,qwen38q8-nothink-matched,1,3.8,Q8_0,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,16,1.0,6771,144.5,,0,1786878149 +p2_triage_qwen38-nothink-offspec_v2,mmbt-qwen38-eaaa8ca,p2_triage,qwen38-nothink-offspec,2,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,16,1.0,6575,101.3,,0,1786832895 +p2_triage_qwen38q8-nothink-matched_v2,mmbt-q38-q8,p2_triage,qwen38q8-nothink-matched,2,3.8,Q8_0,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,0,0,1,1,19,1.0,7434,159.1,,0,1786883769 +p2_triage_qwen38-nothink-offspec_v3,mmbt-qwen38-eaaa8ca,p2_triage,qwen38-nothink-offspec,3,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,15,1.0,6156,95.9,,0,1786864549 +p2_triage_qwen38-nothink-offspec_v4,mmbt-qwen38-eaaa8ca,p2_triage,qwen38-nothink-offspec,4,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,14,1.0,5832,90.6,,0,1786872633 +p2_triage_qwen38-nothink-offspec_v5,mmbt-qwen38-eaaa8ca,p2_triage,qwen38-nothink-offspec,5,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,19,1.0,7418,114.4,,0,1786878642 +p2_triage_qwen38-nothink-offspec_v6,mmbt-qwen38-eaaa8ca,p2_triage,qwen38-nothink-offspec,6,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,16,1.0,6426,98.9,,0,1786886042 +p2_triage_qwen38-nothink-offspec_v7,mmbt-qwen38-eaaa8ca,p2_triage,qwen38-nothink-offspec,7,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,16,1.0,6289,97.0,,0,1786889316 +p3_business_qwen38-nothink-offspec_v1,mmbt-qwen38-eaaa8ca,p3_business,qwen38-nothink-offspec,1,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,2,1,25,0.96,10248,157.5,,0,1786818002 +p3_business_qwen38q8-nothink-matched_v1,mmbt-q38-q8,p3_business,qwen38q8-nothink-matched,1,3.8,Q8_0,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,17,1.0,8334,176.6,,0,1786878654 +p3_business_qwen38-nothink-offspec_v2,mmbt-qwen38-eaaa8ca,p3_business,qwen38-nothink-offspec,2,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,19,1.0,10441,158.4,,0,1786833161 +p3_business_qwen38-nothink-offspec_v3,mmbt-qwen38-eaaa8ca,p3_business,qwen38-nothink-offspec,3,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,30,30,48,0.25,,,identical-call-loop,0,1786865557 +p3_business_qwen38-nothink-offspec_v4,mmbt-qwen38-eaaa8ca,p3_business,qwen38-nothink-offspec,4,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,16,1.0,7374,112.3,,0,1786872856 +p3_business_qwen38-nothink-offspec_v5,mmbt-qwen38-eaaa8ca,p3_business,qwen38-nothink-offspec,5,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,20,1.0,12415,188.4,,0,1786878940 +p3_business_qwen38-nothink-offspec_v6,mmbt-qwen38-eaaa8ca,p3_business,qwen38-nothink-offspec,6,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,2,1,24,0.9583,13740,211.4,,0,1786887174 +p3_business_qwen38-nothink-offspec_v7,mmbt-qwen38-eaaa8ca,p3_business,qwen38-nothink-offspec,7,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,21,1.0,10078,153.1,,0,1786889580 +p3_doc_qwen38-nothink-offspec_v1,mmbt-qwen38-eaaa8ca,p3_doc,qwen38-nothink-offspec,1,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,16,1.0,7593,116.5,,0,1786817843 +p3_doc_qwen38q8-nothink-matched_v1,mmbt-q38-q8,p3_doc,qwen38q8-nothink-matched,1,3.8,Q8_0,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,6,1,28,0.8214,15304,326.8,,0,1786878476 +p3_doc_qwen38-nothink-offspec_v2,mmbt-qwen38-eaaa8ca,p3_doc,qwen38-nothink-offspec,2,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,14,1.0,6920,104.8,,0,1786833001 +p3_doc_qwen38-nothink-offspec_v3,mmbt-qwen38-eaaa8ca,p3_doc,qwen38-nothink-offspec,3,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,16,1.0,6498,99.7,,0,1786864650 +p3_doc_qwen38-nothink-offspec_v4,mmbt-qwen38-eaaa8ca,p3_doc,qwen38-nothink-offspec,4,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,15,1.0,7111,108.9,,0,1786872743 +p3_doc_qwen38-nothink-offspec_v5,mmbt-qwen38-eaaa8ca,p3_doc,qwen38-nothink-offspec,5,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,15,1.0,7105,108.1,,0,1786878751 +p3_doc_qwen38-nothink-offspec_v6,mmbt-qwen38-eaaa8ca,p3_doc,qwen38-nothink-offspec,6,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,32,31,52,0.2885,,,identical-call-loop,0,1786886945 +p3_doc_qwen38-nothink-offspec_v7,mmbt-qwen38-eaaa8ca,p3_doc,qwen38-nothink-offspec,7,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,16,1.0,7138,109.5,,0,1786889427 +p3_market_qwen38-nothink-offspec_v1,mmbt-qwen38-eaaa8ca,p3_market,qwen38-nothink-offspec,1,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,206,206,233,0.1202,,,identical-call-loop,0,1786818331 +p3_market_qwen38q8-nothink-matched_v1,mmbt-q38-q8,p3_market,qwen38q8-nothink-matched,1,3.8,Q8_0,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,80,80,222,0.6441,,,identical-call-loop,0,1786880779 +p3_market_qwen38-nothink-offspec_v2,mmbt-qwen38-eaaa8ca,p3_market,qwen38-nothink-offspec,2,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,STRUCTURAL_FAIL,1,0,1,0,0,5,5,502,0.8247,103162,2914.9,,0,1786836076 +p3_market_qwen38-nothink-offspec_v3,mmbt-qwen38-eaaa8ca,p3_market,qwen38-nothink-offspec,3,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,STRUCTURAL_FAIL,1,0,1,1,0,150,1,501,0.4052,127818,3236.0,,0,1786868799 +p3_market_qwen38-nothink-offspec_v4,mmbt-qwen38-eaaa8ca,p3_market,qwen38-nothink-offspec,4,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,74,74,110,0.3364,,,identical-call-loop,0,1786873163 +p3_market_qwen38-nothink-offspec_v5,mmbt-qwen38-eaaa8ca,p3_market,qwen38-nothink-offspec,5,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,STRUCTURAL_FAIL,1,0,1,1,0,205,3,507,0.1933,53659,1261.9,,0,1786880203 +p3_market_qwen38-nothink-offspec_v6,mmbt-qwen38-eaaa8ca,p3_market,qwen38-nothink-offspec,6,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,150,149,275,0.4582,,,identical-call-loop,0,1786887805 +p3_market_qwen38-nothink-offspec_v7,mmbt-qwen38-eaaa8ca,p3_market,qwen38-nothink-offspec,7,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,43,43,99,0.5354,,,identical-call-loop,0,1786889890 +p3_pm_qwen38-nothink-offspec_v1,mmbt-qwen38-eaaa8ca,p3_pm,qwen38-nothink-offspec,1,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,12,1.0,4024,62.2,,0,1786818477 +p3_pm_qwen38q8-nothink-matched_v1,mmbt-q38-q8,p3_pm,qwen38q8-nothink-matched,1,3.8,Q8_0,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,12,1.0,2935,65.2,,0,1786881178 +p3_pm_qwen38-nothink-offspec_v2,mmbt-qwen38-eaaa8ca,p3_pm,qwen38-nothink-offspec,2,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,9,1.0,3265,50.6,,0,1786836205 +p3_pm_qwen38-nothink-offspec_v3,mmbt-qwen38-eaaa8ca,p3_pm,qwen38-nothink-offspec,3,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,10,1.0,3412,52.6,,0,1786868934 +p3_pm_qwen38-nothink-offspec_v4,mmbt-qwen38-eaaa8ca,p3_pm,qwen38-nothink-offspec,4,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,9,1.0,3308,50.6,,0,1786873294 +p3_pm_qwen38-nothink-offspec_v5,mmbt-qwen38-eaaa8ca,p3_pm,qwen38-nothink-offspec,5,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,10,1.0,3151,49.3,,0,1786880344 +p3_pm_qwen38-nothink-offspec_v6,mmbt-qwen38-eaaa8ca,p3_pm,qwen38-nothink-offspec,6,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,9,1.0,3257,50.7,,0,1786887937 +p3_pm_qwen38-nothink-offspec_v7,mmbt-qwen38-eaaa8ca,p3_pm,qwen38-nothink-offspec,7,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,9,1.0,3316,50.9,,0,1786890020 +p3_writing_qwen38-nothink-offspec_v1,mmbt-qwen38-eaaa8ca,p3_writing,qwen38-nothink-offspec,1,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,17,1.0,5264,81.6,,0,1786818414 +p3_writing_qwen38q8-nothink-matched_v1,mmbt-q38-q8,p3_writing,qwen38q8-nothink-matched,1,3.8,Q8_0,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,110,109,140,0.1714,,,identical-call-loop,0,1786881112 +p3_writing_qwen38-nothink-offspec_v2,mmbt-qwen38-eaaa8ca,p3_writing,qwen38-nothink-offspec,2,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,17,1.0,4927,76.6,,0,1786836154 +p3_writing_qwen38-nothink-offspec_v3,mmbt-qwen38-eaaa8ca,p3_writing,qwen38-nothink-offspec,3,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,17,1.0,5278,80.7,,0,1786868881 +p3_writing_qwen38-nothink-offspec_v4,mmbt-qwen38-eaaa8ca,p3_writing,qwen38-nothink-offspec,4,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,17,1.0,4917,76.8,,0,1786873243 +p3_writing_qwen38-nothink-offspec_v5,mmbt-qwen38-eaaa8ca,p3_writing,qwen38-nothink-offspec,5,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,20,1.0,5583,90.2,,0,1786880294 +p3_writing_qwen38-nothink-offspec_v6,mmbt-qwen38-eaaa8ca,p3_writing,qwen38-nothink-offspec,6,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,16,1.0,4837,77.4,,0,1786887885 +p3_writing_qwen38-nothink-offspec_v7,mmbt-qwen38-eaaa8ca,p3_writing,qwen38-nothink-offspec,7,3.8,UD-Q4_K_XL,no-think,,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,16,1.0,4837,75.7,,0,1786889969 +p1_bugfix_qwen38-nothink-card_v1,mmbt-q38-card,p1_bugfix,qwen38-nothink-card,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,1,0,250,6,565,0.1451,105433,4755.0,,0,1786824120 +p1_bugfix_qwen38-27b-udq4xl-nothink_v1,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-27b-udq4xl-nothink,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,,0,0,1,1,1,60,60,170,0.6412,,,identical-call-loop,0,1786754006 +p1_bugfix_qwen38-nothink-card_v2,mmbt-q38-card,p1_bugfix,qwen38-nothink-card,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,1,0,332,6,525,0.0552,63968,3989.5,,0,1786860239 +p1_bugfix_qwen38-27b-udq4xl-nothink_v2,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-27b-udq4xl-nothink,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,,0,0,1,0,0,18,2,81,0.3827,,,identical-call-loop,0,1786790878 +p1_bugfix_qwen38-nothink-card_v3,mmbt-q38-card,p1_bugfix,qwen38-nothink-card,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,2,2,82,0.9878,31678,1858.1,,0,1786865159 +p1_bugfix_qwen38-27b-udq4xl-nothink_v3,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-27b-udq4xl-nothink,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,2,2,152,0.9934,50137,4263.5,success-shipped,0,1786758272 +p1_bugfix_qwen38-nothink-card_v4,mmbt-q38-card,p1_bugfix,qwen38-nothink-card,4,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,,0,0,1,1,1,32,31,124,0.75,,,identical-call-loop,0,1786875972 +p1_bugfix_qwen38-nothink-card_v5,mmbt-q38-card,p1_bugfix,qwen38-nothink-card,5,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,,0,0,1,1,1,71,71,143,0.5105,,,identical-call-loop,0,1786884288 +p1_refactor_qwen38-nothink-card_v1,mmbt-q38-card,p1_refactor,qwen38-nothink-card,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,32,1.0,7748,140.1,,0,1786817337 +p1_refactor_qwen38-27b-udq4xl-nothink_v1,mmbt-qwen38-eaaa8ca,p1_refactor,qwen38-27b-udq4xl-nothink,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,48,1.0,11032,199.4,,0,1786759699 +p1_refactor_qwen38-nothink-card_v2,mmbt-q38-card,p1_refactor,qwen38-nothink-card,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,29,1.0,9157,164.5,,0,1786860901 +p1_refactor_qwen38-27b-udq4xl-nothink_v2,mmbt-qwen38-eaaa8ca,p1_refactor,qwen38-27b-udq4xl-nothink,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,1,0,89,3,515,0.101,42198,1213.5,,0,1786759049 +p1_refactor_qwen38-nothink-card_v3,mmbt-q38-card,p1_refactor,qwen38-nothink-card,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,1,0,109,3,509,0.1454,42360,1239.3,,0,1786871596 +p1_refactor_qwen38-27b-udq4xl-nothink_v3,mmbt-qwen38-eaaa8ca,p1_refactor,qwen38-27b-udq4xl-nothink,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,1,0,164,5,517,0.265,62281,1679.4,,0,1786761379 +p1_refactor_qwen38-nothink-card_v4,mmbt-q38-card,p1_refactor,qwen38-nothink-card,4,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,40,1.0,9904,182.3,,0,1786876930 +p1_refactor_qwen38-nothink-card_v5,mmbt-q38-card,p1_refactor,qwen38-nothink-card,5,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,0,0,6,2,66,0.8939,15545,283.9,,0,1786885106 +p1_testwrite_qwen38-nothink-card_v1,mmbt-q38-card,p1_testwrite,qwen38-nothink-card,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,1,0,41,4,251,0.3665,30632,931.4,,0,1786825054 +p1_testwrite_qwen38-27b-udq4xl-nothink_v1,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-27b-udq4xl-nothink,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,1,0,32,3,251,0.2829,20873,539.0,,0,1786791421 +p1_testwrite_qwen38-nothink-card_v2,mmbt-q38-card,p1_testwrite,qwen38-nothink-card,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,0,0,20,2,251,0.2908,18666,494.3,,0,1786860736 +p1_testwrite_qwen38-27b-udq4xl-nothink_v2,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-27b-udq4xl-nothink,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,1,0,31,3,277,0.361,25923,893.4,,0,1786791442 +p1_testwrite_qwen38-nothink-card_v3,mmbt-q38-card,p1_testwrite,qwen38-nothink-card,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,1,0,36,5,179,0.648,200652,5194.0,,0,1786870356 +p1_testwrite_qwen38-27b-udq4xl-nothink_v3,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-27b-udq4xl-nothink,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,,0,0,1,1,1,46,46,60,0.25,,,identical-call-loop,0,1786791758 +p1_testwrite_qwen38-nothink-card_v4,mmbt-q38-card,p1_testwrite,qwen38-nothink-card,4,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,1,0,39,2,315,0.2476,29879,772.8,,0,1786876747 +p1_testwrite_qwen38-nothink-card_v5,mmbt-q38-card,p1_testwrite,qwen38-nothink-card,5,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,1,0,44,3,251,0.3147,18074,530.5,,0,1786884821 +p2_ci_qwen38-nothink-card_v1,mmbt-q38-card,p2_ci,qwen38-nothink-card,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,18,1.0,3573,67.5,,0,1786817454 +p2_ci_qwen38-27b-udq4xl-nothink_v1,mmbt-qwen38-eaaa8ca,p2_ci,qwen38-27b-udq4xl-nothink,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,20,1.0,4316,79.4,,0,1786761509 +p2_ci_qwen38-nothink-card_v2,mmbt-q38-card,p2_ci,qwen38-nothink-card,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,19,1.0,4136,77.6,,0,1786861032 +p2_ci_qwen38-27b-udq4xl-nothink_v2,mmbt-qwen38-eaaa8ca,p2_ci,qwen38-27b-udq4xl-nothink,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,18,1.0,4069,74.4,,0,1786759224 +p2_ci_qwen38-nothink-card_v3,mmbt-q38-card,p2_ci,qwen38-nothink-card,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,21,1.0,4010,76.3,,0,1786871724 +p2_ci_qwen38-27b-udq4xl-nothink_v3,mmbt-qwen38-eaaa8ca,p2_ci,qwen38-27b-udq4xl-nothink,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,22,1.0,4899,89.5,,0,1786761599 +p2_ci_qwen38-nothink-card_v4,mmbt-q38-card,p2_ci,qwen38-nothink-card,4,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,19,1.0,4112,78.1,,0,1786877060 +p2_ci_qwen38-nothink-card_v5,mmbt-q38-card,p2_ci,qwen38-nothink-card,5,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,18,1.0,3578,66.7,,0,1786885225 +p2_extract_qwen38-nothink-card_v1,mmbt-q38-card,p2_extract,qwen38-nothink-card,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,11,1.0,2854,48.3,,0,1786817386 +p2_extract_qwen38-27b-udq4xl-nothink_v1,mmbt-qwen38-eaaa8ca,p2_extract,qwen38-27b-udq4xl-nothink,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,8,1.0,2934,50.7,,0,1786759100 +p2_extract_qwen38-nothink-card_v2,mmbt-q38-card,p2_extract,qwen38-nothink-card,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,8,1.0,3011,51.3,,0,1786860953 +p2_extract_qwen38-27b-udq4xl-nothink_v2,mmbt-qwen38-eaaa8ca,p2_extract,qwen38-27b-udq4xl-nothink,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,8,1.0,3015,48.6,,0,1786761428 +p2_extract_qwen38-nothink-card_v3,mmbt-q38-card,p2_extract,qwen38-nothink-card,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,8,1.0,3039,49.6,,0,1786871647 +p2_extract_qwen38-27b-udq4xl-nothink_v3,mmbt-qwen38-eaaa8ca,p2_extract,qwen38-27b-udq4xl-nothink,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,11,1.0,2869,47.7,,0,1786759149 +p2_extract_qwen38-nothink-card_v4,mmbt-q38-card,p2_extract,qwen38-nothink-card,4,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,8,1.0,2977,49.7,,0,1786876981 +p2_extract_qwen38-nothink-card_v5,mmbt-q38-card,p2_extract,qwen38-nothink-card,5,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,8,1.0,2983,50.5,,0,1786885157 +p2_hallucination_qwen38-nothink-card_v1,mmbt-q38-card,p2_hallucination,qwen38-nothink-card,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,21,1.0,5584,97.1,,0,1786817552 +p2_hallucination_qwen38-27b-udq4xl-nothink_v1,mmbt-qwen38-eaaa8ca,p2_hallucination,qwen38-27b-udq4xl-nothink,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,20,1.0,4638,82.1,,0,1786759307 +p2_hallucination_qwen38-nothink-card_v2,mmbt-q38-card,p2_hallucination,qwen38-nothink-card,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,15,1.0,3504,63.1,,0,1786861095 +p2_hallucination_qwen38-27b-udq4xl-nothink_v2,mmbt-qwen38-eaaa8ca,p2_hallucination,qwen38-27b-udq4xl-nothink,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,18,1.0,4432,79.6,,0,1786761680 +p2_hallucination_qwen38-nothink-card_v3,mmbt-q38-card,p2_hallucination,qwen38-nothink-card,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,14,1.0,3972,70.9,,0,1786871796 +p2_hallucination_qwen38-27b-udq4xl-nothink_v3,mmbt-qwen38-eaaa8ca,p2_hallucination,qwen38-27b-udq4xl-nothink,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,19,1.0,4688,84.1,,0,1786759392 +p2_hallucination_qwen38-nothink-card_v4,mmbt-q38-card,p2_hallucination,qwen38-nothink-card,4,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,14,1.0,4049,74.6,,0,1786877135 +p2_hallucination_qwen38-nothink-card_v5,mmbt-q38-card,p2_hallucination,qwen38-nothink-card,5,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,15,1.0,3355,61.5,,0,1786885287 +p2_triage_qwen38-nothink-card_v1,mmbt-q38-card,p2_triage,qwen38-nothink-card,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,14,1.0,5993,99.0,,0,1786817652 +p2_triage_qwen38-27b-udq4xl-nothink_v1,mmbt-qwen38-eaaa8ca,p2_triage,qwen38-27b-udq4xl-nothink,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,0,0,2,1,20,0.95,8237,137.5,,0,1786761818 +p2_triage_qwen38-nothink-card_v2,mmbt-q38-card,p2_triage,qwen38-nothink-card,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,0,0,1,1,14,1.0,6189,105.8,,0,1786861202 +p2_triage_qwen38-27b-udq4xl-nothink_v2,mmbt-qwen38-eaaa8ca,p2_triage,qwen38-27b-udq4xl-nothink,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,0,0,1,1,14,1.0,5913,96.8,,0,1786759490 +p2_triage_qwen38-nothink-card_v3,mmbt-q38-card,p2_triage,qwen38-nothink-card,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,13,1.0,5736,94.7,,0,1786871892 +p2_triage_qwen38-27b-udq4xl-nothink_v3,mmbt-qwen38-eaaa8ca,p2_triage,qwen38-27b-udq4xl-nothink,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,18,1.0,6546,111.1,,0,1786761930 +p2_triage_qwen38-nothink-card_v4,mmbt-q38-card,p2_triage,qwen38-nothink-card,4,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,14,1.0,5754,97.1,,0,1786877233 +p2_triage_qwen38-nothink-card_v5,mmbt-q38-card,p2_triage,qwen38-nothink-card,5,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,14,1.0,6002,99.5,,0,1786885387 +p3_business_qwen38-nothink-card_v1,mmbt-q38-card,p3_business,qwen38-nothink-card,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,0,0,1,1,14,1.0,7401,122.6,,0,1786825177 +p3_business_qwen38-27b-udq4xl-nothink_v1,mmbt-qwen38-eaaa8ca,p3_business,qwen38-27b-udq4xl-nothink,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,0,0,1,1,26,1.0,14870,244.5,,0,1786762344 +p3_business_qwen38-nothink-card_v2,mmbt-q38-card,p3_business,qwen38-nothink-card,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,0,0,3,2,26,0.9231,11561,193.3,,0,1786861587 +p3_business_qwen38-27b-udq4xl-nothink_v2,mmbt-qwen38-eaaa8ca,p3_business,qwen38-27b-udq4xl-nothink,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,,0,0,1,1,1,233,233,239,0.0293,,,identical-call-loop,0,1786760168 +p3_business_qwen38-nothink-card_v3,mmbt-q38-card,p3_business,qwen38-nothink-card,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,16,1.0,7029,115.1,,0,1786872181 +p3_business_qwen38-27b-udq4xl-nothink_v3,mmbt-qwen38-eaaa8ca,p3_business,qwen38-27b-udq4xl-nothink,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,15,1.0,7441,123.4,,0,1786762468 +p3_business_qwen38-nothink-card_v4,mmbt-q38-card,p3_business,qwen38-nothink-card,4,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,MISSING_OUTPUT,1,0,1,1,0,290,6,500,0.018,168466,4287.9,,0,1786881709 +p3_business_qwen38-nothink-card_v5,mmbt-q38-card,p3_business,qwen38-nothink-card,5,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,,0,0,1,1,1,254,254,259,0.0232,,,identical-call-loop,0,1786885855 +p3_doc_qwen38-nothink-card_v1,mmbt-q38-card,p3_doc,qwen38-nothink-card,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,0,0,1,1,24,1.0,11510,191.5,,0,1786817844 +p3_doc_qwen38-27b-udq4xl-nothink_v1,mmbt-qwen38-eaaa8ca,p3_doc,qwen38-27b-udq4xl-nothink,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,0,0,1,1,21,1.0,11269,182.9,,0,1786759674 +p3_doc_qwen38-nothink-card_v2,mmbt-q38-card,p3_doc,qwen38-nothink-card,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,0,0,1,1,24,1.0,11359,190.3,,0,1786861393 +p3_doc_qwen38-27b-udq4xl-nothink_v2,mmbt-qwen38-eaaa8ca,p3_doc,qwen38-27b-udq4xl-nothink,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,0,0,2,1,20,0.95,10260,167.1,,0,1786762098 +p3_doc_qwen38-nothink-card_v3,mmbt-q38-card,p3_doc,qwen38-nothink-card,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,0,0,1,1,20,1.0,10390,172.6,,0,1786872065 +p3_doc_qwen38-27b-udq4xl-nothink_v3,mmbt-qwen38-eaaa8ca,p3_doc,qwen38-27b-udq4xl-nothink,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,20,1.0,9935,164.4,,0,1786759839 +p3_doc_qwen38-nothink-card_v4,mmbt-q38-card,p3_doc,qwen38-nothink-card,4,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,0,0,1,1,22,1.0,11334,187.1,,0,1786877421 +p3_doc_qwen38-nothink-card_v5,mmbt-q38-card,p3_doc,qwen38-nothink-card,5,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,0,0,1,1,27,1.0,9413,157.6,,0,1786885546 +p3_market_qwen38-nothink-card_v1,mmbt-q38-card,p3_market,qwen38-nothink-card,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,,0,0,1,0,0,13,3,91,0.6813,,,identical-call-loop,0,1786825490 +p3_market_qwen38-27b-udq4xl-nothink_v1,mmbt-qwen38-eaaa8ca,p3_market,qwen38-27b-udq4xl-nothink,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,STRUCTURAL_FAIL,1,0,1,1,0,333,5,549,0.1056,62918,1689.7,,0,1786761859 +p3_market_qwen38-nothink-card_v2,mmbt-q38-card,p3_market,qwen38-nothink-card,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,,0,0,1,1,1,36,36,256,0.8555,,,identical-call-loop,0,1786862809 +p3_market_qwen38-27b-udq4xl-nothink_v2,mmbt-qwen38-eaaa8ca,p3_market,qwen38-27b-udq4xl-nothink,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,,0,0,1,1,1,121,121,235,0.4809,,,identical-call-loop,0,1786763105 +p3_market_qwen38-nothink-card_v3,mmbt-q38-card,p3_market,qwen38-nothink-card,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,,0,0,1,0,0,1,1,132,1.0,,,identical-call-loop,0,1786872805 +p3_market_qwen38-27b-udq4xl-nothink_v3,mmbt-qwen38-eaaa8ca,p3_market,qwen38-27b-udq4xl-nothink,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,,0,0,1,0,0,1,1,114,1.0,,,identical-call-loop,0,1786762166 +p3_market_qwen38-nothink-card_v4,mmbt-q38-card,p3_market,qwen38-nothink-card,4,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,,0,0,1,1,1,54,54,121,0.5455,,,identical-call-loop,0,1786882022 +p3_market_qwen38-nothink-card_v5,mmbt-q38-card,p3_market,qwen38-nothink-card,5,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,STRUCTURAL_FAIL,1,0,1,1,0,56,1,517,0.1393,42834,1292.9,,0,1786887149 +p3_pm_qwen38-nothink-card_v1,mmbt-q38-card,p3_pm,qwen38-nothink-card,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,11,1.0,5081,85.6,,0,1786826210 +p3_pm_qwen38-27b-udq4xl-nothink_v1,mmbt-qwen38-eaaa8ca,p3_pm,qwen38-27b-udq4xl-nothink,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,11,1.0,5080,82.4,,0,1786762885 +p3_pm_qwen38-nothink-card_v2,mmbt-q38-card,p3_pm,qwen38-nothink-card,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,9,1.0,3919,67.2,,0,1786863127 +p3_pm_qwen38-27b-udq4xl-nothink_v2,mmbt-qwen38-eaaa8ca,p3_pm,qwen38-27b-udq4xl-nothink,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,9,1.0,3979,66.2,,0,1786769598 +p3_pm_qwen38-nothink-card_v3,mmbt-q38-card,p3_pm,qwen38-nothink-card,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,9,1.0,3943,66.3,,0,1786873507 +p3_pm_qwen38-27b-udq4xl-nothink_v3,mmbt-qwen38-eaaa8ca,p3_pm,qwen38-27b-udq4xl-nothink,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,9,1.0,3935,63.1,,0,1786762950 +p3_pm_qwen38-nothink-card_v4,mmbt-q38-card,p3_pm,qwen38-nothink-card,4,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,12,1.0,5080,82.6,,0,1786882738 +p3_pm_qwen38-nothink-card_v5,mmbt-q38-card,p3_pm,qwen38-nothink-card,5,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,PASS,1,1,1,0,0,1,1,11,1.0,5115,84.2,,0,1786887495 +p3_writing_qwen38-nothink-card_v1,mmbt-q38-card,p3_writing,qwen38-nothink-card,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,,0,0,1,1,1,62,62,79,0.1772,,,identical-call-loop,0,1786826117 +p3_writing_qwen38-27b-udq4xl-nothink_v1,mmbt-qwen38-eaaa8ca,p3_writing,qwen38-27b-udq4xl-nothink,1,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,0,0,4,3,34,0.8529,12652,215.0,,0,1786763322 +p3_writing_qwen38-nothink-card_v2,mmbt-q38-card,p3_writing,qwen38-nothink-card,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,0,0,4,3,38,0.8421,14603,248.8,,0,1786863059 +p3_writing_qwen38-27b-udq4xl-nothink_v2,mmbt-qwen38-eaaa8ca,p3_writing,qwen38-27b-udq4xl-nothink,2,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,,0,0,1,1,1,63,63,80,0.175,,,identical-call-loop,0,1786762801 +p3_writing_qwen38-nothink-card_v3,mmbt-q38-card,p3_writing,qwen38-nothink-card,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,,0,0,1,1,1,62,62,78,0.1795,,,identical-call-loop,0,1786873434 +p3_writing_qwen38-27b-udq4xl-nothink_v3,mmbt-qwen38-eaaa8ca,p3_writing,qwen38-27b-udq4xl-nothink,3,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,1,0,366,14,491,0.0367,238124,6208.5,,0,1786769531 +p3_writing_qwen38-nothink-card_v4,mmbt-q38-card,p3_writing,qwen38-nothink-card,4,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,,0,0,1,1,1,55,55,76,0.1974,,,identical-call-loop,0,1786882645 +p3_writing_qwen38-nothink-card_v5,mmbt-q38-card,p3_writing,qwen38-nothink-card,5,3.8,UD-Q4_K_XL,no-think,,T0.7/p0.8/pp1.5,0.7,0.8,20,1.5,42,FAIL,1,0,1,0,0,3,2,41,0.9268,15259,259.7,,0,1786887410 +p1_bugfix_qwen38-think-low-offspec_v1,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-think-low-offspec,1,3.8,UD-Q4_K_XL,think,low,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,148,1.0,86053,2378.7,,0,1786846045 +p1_bugfix_qwen38-think-medium-offspec_v1,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-think-medium-offspec,1,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,4,1,125,0.976,61733,1841.9,,0,1786851575 +p1_bugfix_qwen38-think-xhigh-offspec_v1,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-think-xhigh-offspec,1,3.8,UD-Q4_K_XL,think,xhigh,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,9,1,173,0.9538,116350,2638.9,,0,1786824185 +p1_bugfix_qwen38-think-low-offspec_v2,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-think-low-offspec,2,3.8,UD-Q4_K_XL,think,low,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,9,1,168,0.9286,111588,2702.2,,0,1786862699 +p1_bugfix_qwen38-think-medium-offspec_v2,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-think-medium-offspec,2,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,3,1,119,0.9832,72868,2009.0,,0,1786853633 +p1_bugfix_qwen38-think-xhigh-offspec_v2,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-think-xhigh-offspec,2,3.8,UD-Q4_K_XL,think,xhigh,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,0,0,0,1,1,171,1.0,,,,0,1786839481 +p1_refactor_qwen38-think-low-offspec_v1,mmbt-qwen38-eaaa8ca,p1_refactor,qwen38-think-low-offspec,1,3.8,UD-Q4_K_XL,think,low,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,46,1.0,35761,597.4,,0,1786847183 +p1_refactor_qwen38-think-medium-offspec_v1,mmbt-qwen38-eaaa8ca,p1_refactor,qwen38-think-medium-offspec,1,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,41,1.0,19383,791.4,,0,1786841353 +p1_refactor_qwen38-think-xhigh-offspec_v1,mmbt-qwen38-eaaa8ca,p1_refactor,qwen38-think-xhigh-offspec,1,3.8,UD-Q4_K_XL,think,xhigh,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,67,1.0,43124,734.9,,0,1786826062 +p1_refactor_qwen38-think-medium-offspec_v2,mmbt-qwen38-eaaa8ca,p1_refactor,qwen38-think-medium-offspec,2,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,42,1.0,20177,336.2,,0,1786857609 +p1_testwrite_qwen38-think-low-offspec_v1,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-think-low-offspec,1,3.8,UD-Q4_K_XL,think,low,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,4,1,61,0.9508,31491,537.1,,0,1786846585 +p1_testwrite_qwen38-think-medium-offspec_v1,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-think-medium-offspec,1,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,47,1.0,30283,940.5,,0,1786840560 +p1_testwrite_qwen38-think-xhigh-offspec_v1,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-think-xhigh-offspec,1,3.8,UD-Q4_K_XL,think,xhigh,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,58,1.0,65201,1139.5,,0,1786825326 +p1_testwrite_qwen38-think-low-offspec_v2,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-think-low-offspec,2,3.8,UD-Q4_K_XL,think,low,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,6,1,65,0.9231,32211,545.7,,0,1786863247 +p1_testwrite_qwen38-think-medium-offspec_v2,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-think-medium-offspec,2,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,11,1.0,1946,3638.4,,0,1786857272 +p2_ci_qwen38-think-low-offspec_v1,mmbt-qwen38-eaaa8ca,p2_ci,qwen38-think-low-offspec,1,3.8,UD-Q4_K_XL,think,low,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,43,0.9535,8214,135.7,,0,1786847395 +p2_ci_qwen38-think-medium-offspec_v1,mmbt-qwen38-eaaa8ca,p2_ci,qwen38-think-medium-offspec,1,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,17,1.0,4992,83.5,,0,1786841631 +p2_ci_qwen38-think-xhigh-offspec_v1,mmbt-qwen38-eaaa8ca,p2_ci,qwen38-think-xhigh-offspec,1,3.8,UD-Q4_K_XL,think,xhigh,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,22,1.0,9216,148.4,,0,1786826360 +p2_ci_qwen38-think-medium-offspec_v2,mmbt-qwen38-eaaa8ca,p2_ci,qwen38-think-medium-offspec,2,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,15,1.0,5548,92.0,,0,1786857780 +p2_extract_qwen38-think-low-offspec_v1,mmbt-qwen38-eaaa8ca,p2_extract,qwen38-think-low-offspec,1,3.8,UD-Q4_K_XL,think,low,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,14,1.0,4760,74.5,,0,1786847259 +p2_extract_qwen38-think-medium-offspec_v1,mmbt-qwen38-eaaa8ca,p2_extract,qwen38-think-medium-offspec,1,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,13,1.0,4697,192.9,,0,1786841547 +p2_extract_qwen38-think-xhigh-offspec_v1,mmbt-qwen38-eaaa8ca,p2_extract,qwen38-think-xhigh-offspec,1,3.8,UD-Q4_K_XL,think,xhigh,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,21,1.0,9605,147.5,,0,1786826210 +p2_extract_qwen38-think-medium-offspec_v2,mmbt-qwen38-eaaa8ca,p2_extract,qwen38-think-medium-offspec,2,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,13,1.0,4979,77.5,,0,1786857688 +p2_hallucination_qwen38-think-low-offspec_v1,mmbt-qwen38-eaaa8ca,p2_hallucination,qwen38-think-low-offspec,1,3.8,UD-Q4_K_XL,think,low,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,29,1.0,9255,145.6,,0,1786847542 +p2_hallucination_qwen38-think-medium-offspec_v1,mmbt-qwen38-eaaa8ca,p2_hallucination,qwen38-think-medium-offspec,1,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,12,1.0,7987,125.9,,0,1786841757 +p2_hallucination_qwen38-think-xhigh-offspec_v1,mmbt-qwen38-eaaa8ca,p2_hallucination,qwen38-think-xhigh-offspec,1,3.8,UD-Q4_K_XL,think,xhigh,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,16,1.0,13221,210.7,,0,1786826571 +p2_hallucination_qwen38-think-medium-offspec_v2,mmbt-qwen38-eaaa8ca,p2_hallucination,qwen38-think-medium-offspec,2,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,13,1.0,7820,124.1,,0,1786857905 +p2_triage_qwen38-think-low-offspec_v1,mmbt-qwen38-eaaa8ca,p2_triage,qwen38-think-low-offspec,1,3.8,UD-Q4_K_XL,think,low,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,11,1.0,12964,195.9,,0,1786847739 +p2_triage_qwen38-think-medium-offspec_v1,mmbt-qwen38-eaaa8ca,p2_triage,qwen38-think-medium-offspec,1,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,10,1.0,6850,103.9,,0,1786841862 +p2_triage_qwen38-think-xhigh-offspec_v1,mmbt-qwen38-eaaa8ca,p2_triage,qwen38-think-xhigh-offspec,1,3.8,UD-Q4_K_XL,think,xhigh,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,14,1.0,15246,233.7,,0,1786826806 +p2_triage_qwen38-think-medium-offspec_v2,mmbt-qwen38-eaaa8ca,p2_triage,qwen38-think-medium-offspec,2,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,10,1.0,6766,102.8,,0,1786858009 +p3_business_qwen38-think-low-offspec_v1,mmbt-qwen38-eaaa8ca,p3_business,qwen38-think-low-offspec,1,3.8,UD-Q4_K_XL,think,low,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,2,1,17,0.9412,17246,262.6,,0,1786848240 +p3_business_qwen38-think-medium-offspec_v1,mmbt-qwen38-eaaa8ca,p3_business,qwen38-think-medium-offspec,1,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,15,1.0,13809,207.0,,0,1786842637 +p3_business_qwen38-think-xhigh-offspec_v1,mmbt-qwen38-eaaa8ca,p3_business,qwen38-think-xhigh-offspec,1,3.8,UD-Q4_K_XL,think,xhigh,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,2,1,22,0.9545,29744,463.6,,0,1786828078 +p3_business_qwen38-think-medium-offspec_v2,mmbt-qwen38-eaaa8ca,p3_business,qwen38-think-medium-offspec,2,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,20,1.0,18214,276.2,,0,1786859253 +p3_doc_qwen38-think-low-offspec_v1,mmbt-qwen38-eaaa8ca,p3_doc,qwen38-think-low-offspec,1,3.8,UD-Q4_K_XL,think,low,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,16,1.0,15533,237.0,,0,1786847977 +p3_doc_qwen38-think-medium-offspec_v1,mmbt-qwen38-eaaa8ca,p3_doc,qwen38-think-medium-offspec,1,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,3,1,26,0.9231,36026,566.7,,0,1786842430 +p3_doc_qwen38-think-xhigh-offspec_v1,mmbt-qwen38-eaaa8ca,p3_doc,qwen38-think-xhigh-offspec,1,3.8,UD-Q4_K_XL,think,xhigh,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,3,1,29,0.8966,49433,806.9,,0,1786827614 +p3_doc_qwen38-think-medium-offspec_v2,mmbt-qwen38-eaaa8ca,p3_doc,qwen38-think-medium-offspec,2,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,5,1,17,0.7647,59657,965.9,,0,1786858976 +p3_market_qwen38-think-low-offspec_v1,mmbt-qwen38-eaaa8ca,p3_market,qwen38-think-low-offspec,1,3.8,UD-Q4_K_XL,think,low,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,STRUCTURAL_FAIL,1,0,1,0,0,1,1,151,1.0,54790,996.2,,0,1786849237 +p3_market_qwen38-think-medium-offspec_v1,mmbt-qwen38-eaaa8ca,p3_market,qwen38-think-medium-offspec,1,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,116,116,174,0.3391,,,identical-call-loop,0,1786843272 +p3_market_qwen38-think-xhigh-offspec_v1,mmbt-qwen38-eaaa8ca,p3_market,qwen38-think-xhigh-offspec,1,3.8,UD-Q4_K_XL,think,xhigh,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,201,1.0,102258,2563.4,,0,1786830642 +p3_market_qwen38-think-medium-offspec_v2,mmbt-qwen38-eaaa8ca,p3_market,qwen38-think-medium-offspec,2,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,,0,0,1,1,1,34,34,65,0.4923,,,identical-call-loop,0,1786859588 +p3_pm_qwen38-think-low-offspec_v1,mmbt-qwen38-eaaa8ca,p3_pm,qwen38-think-low-offspec,1,3.8,UD-Q4_K_XL,think,low,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,20,1.0,14161,216.1,,0,1786849723 +p3_pm_qwen38-think-medium-offspec_v1,mmbt-qwen38-eaaa8ca,p3_pm,qwen38-think-medium-offspec,1,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,12,1.0,5947,89.8,,0,1786843606 +p3_pm_qwen38-think-xhigh-offspec_v1,mmbt-qwen38-eaaa8ca,p3_pm,qwen38-think-xhigh-offspec,1,3.8,UD-Q4_K_XL,think,xhigh,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,20,1.0,24907,383.0,,0,1786831567 +p3_pm_qwen38-think-medium-offspec_v2,mmbt-qwen38-eaaa8ca,p3_pm,qwen38-think-medium-offspec,2,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,10,1.0,6243,93.7,,0,1786859947 +p3_writing_qwen38-think-low-offspec_v1,mmbt-qwen38-eaaa8ca,p3_writing,qwen38-think-low-offspec,1,3.8,UD-Q4_K_XL,think,low,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,PASS,1,1,1,0,0,1,1,22,1.0,17487,268.4,,0,1786849506 +p3_writing_qwen38-think-medium-offspec_v1,mmbt-qwen38-eaaa8ca,p3_writing,qwen38-think-medium-offspec,1,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,25,1.0,15650,240.5,,0,1786843515 +p3_writing_qwen38-think-xhigh-offspec_v1,mmbt-qwen38-eaaa8ca,p3_writing,qwen38-think-xhigh-offspec,1,3.8,UD-Q4_K_XL,think,xhigh,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,1,1,27,1.0,34445,540.3,,0,1786831184 +p3_writing_qwen38-think-medium-offspec_v2,mmbt-qwen38-eaaa8ca,p3_writing,qwen38-think-medium-offspec,2,3.8,UD-Q4_K_XL,think,medium,T0.3/p0.8/pp0,0.3,0.8,20,0.0,42,FAIL,1,0,1,0,0,3,1,26,0.9231,16968,262.2,,0,1786859852 +p1_bugfix_qwen38-think-low-card_v1,mmbt-q38-card,p1_bugfix,qwen38-think-low-card,1,3.8,UD-Q4_K_XL,think,low,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,4,1,145,0.9379,41617,1404.7,,0,1786846439 +p1_bugfix_qwen38-think-medium-card_v1,mmbt-q38-card,p1_bugfix,qwen38-think-medium-card,1,3.8,UD-Q4_K_XL,think,medium,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,6,1,135,0.963,77080,1872.3,,0,1786856207 +p1_bugfix_qwen38-think-xhigh-card_v1,mmbt-q38-card,p1_bugfix,qwen38-think-xhigh-card,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,4,1,199,0.9497,150391,4682.0,,0,1786830946 +p1_bugfix_qwen38-27b-udq4xl-think-xhigh_v1,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-27b-udq4xl-think-xhigh,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,168,1.0,112822,3336.0,,0,1786772960 +p1_bugfix_qwen38-27b-udq4xl-think-xhigh_v2,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-27b-udq4xl-think-xhigh,2,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,5,1,186,0.9785,125759,2811.8,,0,1786794620 +p1_bugfix_qwen38-27b-udq4xl-think-xhigh_v3,mmbt-qwen38-eaaa8ca,p1_bugfix,qwen38-27b-udq4xl-think-xhigh,3,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,248,1.0,159066,4573.2,,0,1786777536 +p1_refactor_qwen38-think-low-card_v1,mmbt-q38-card,p1_refactor,qwen38-think-low-card,1,3.8,UD-Q4_K_XL,think,low,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,2,1,56,0.9821,16005,275.2,,0,1786847735 +p1_refactor_qwen38-think-medium-card_v1,mmbt-q38-card,p1_refactor,qwen38-think-medium-card,1,3.8,UD-Q4_K_XL,think,medium,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,45,1.0,19011,491.1,,0,1786841614 +p1_refactor_qwen38-think-xhigh-card_v1,mmbt-q38-card,p1_refactor,qwen38-think-xhigh-card,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,56,1.0,46909,838.2,,0,1786833358 +p1_refactor_qwen38-27b-udq4xl-think-xhigh_v1,mmbt-qwen38-eaaa8ca,p1_refactor,qwen38-27b-udq4xl-think-xhigh,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,64,1.0,55858,992.2,,0,1786780216 +p1_refactor_qwen38-27b-udq4xl-think-xhigh_v2,mmbt-qwen38-eaaa8ca,p1_refactor,qwen38-27b-udq4xl-think-xhigh,2,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,61,1.0,46369,818.5,,0,1786774665 +p1_refactor_qwen38-27b-udq4xl-think-xhigh_v3,mmbt-qwen38-eaaa8ca,p1_refactor,qwen38-27b-udq4xl-think-xhigh,3,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,80,1.0,57439,1048.0,,0,1786781264 +p1_testwrite_qwen38-think-low-card_v1,mmbt-q38-card,p1_testwrite,qwen38-think-low-card,1,3.8,UD-Q4_K_XL,think,low,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,63,1.0,57253,1017.7,,0,1786847459 +p1_testwrite_qwen38-think-medium-card_v1,mmbt-q38-card,p1_testwrite,qwen38-think-medium-card,1,3.8,UD-Q4_K_XL,think,medium,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,51,1.0,49750,1403.0,,0,1786841122 +p1_testwrite_qwen38-think-xhigh-card_v1,mmbt-q38-card,p1_testwrite,qwen38-think-xhigh-card,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,92,1.0,83345,1570.9,,0,1786832519 +p1_testwrite_qwen38-27b-udq4xl-think-xhigh_v1,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-27b-udq4xl-think-xhigh,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,69,1.0,66958,1195.7,,0,1786772273 +p1_testwrite_qwen38-27b-udq4xl-think-xhigh_v2,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-27b-udq4xl-think-xhigh,2,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,89,1.0,88795,1684.5,,0,1786779222 +p1_testwrite_qwen38-27b-udq4xl-think-xhigh_v3,mmbt-qwen38-eaaa8ca,p1_testwrite,qwen38-27b-udq4xl-think-xhigh,3,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,86,1.0,84407,1571.3,,0,1786773845 +p2_ci_qwen38-think-low-card_v1,mmbt-q38-card,p2_ci,qwen38-think-low-card,1,3.8,UD-Q4_K_XL,think,low,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,3,1,44,0.9545,9294,157.0,,0,1786847965 +p2_ci_qwen38-think-medium-card_v1,mmbt-q38-card,p2_ci,qwen38-think-medium-card,1,3.8,UD-Q4_K_XL,think,medium,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,19,1.0,5946,101.0,,0,1786841795 +p2_ci_qwen38-think-xhigh-card_v1,mmbt-q38-card,p2_ci,qwen38-think-xhigh-card,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,26,1.0,18431,306.5,,0,1786833856 +p2_ci_qwen38-27b-udq4xl-think-xhigh_v1,mmbt-qwen38-eaaa8ca,p2_ci,qwen38-27b-udq4xl-think-xhigh,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,32,1.0,20732,341.6,,0,1786781789 +p2_ci_qwen38-27b-udq4xl-think-xhigh_v2,mmbt-qwen38-eaaa8ca,p2_ci,qwen38-27b-udq4xl-think-xhigh,2,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,36,1.0,19262,318.1,,0,1786775324 +p2_ci_qwen38-27b-udq4xl-think-xhigh_v3,mmbt-qwen38-eaaa8ca,p2_ci,qwen38-27b-udq4xl-think-xhigh,3,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,22,1.0,15426,248.4,,0,1786782039 +p2_extract_qwen38-think-low-card_v1,mmbt-q38-card,p2_extract,qwen38-think-low-card,1,3.8,UD-Q4_K_XL,think,low,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,14,1.0,4410,70.7,,0,1786847807 +p2_extract_qwen38-think-medium-card_v1,mmbt-q38-card,p2_extract,qwen38-think-medium-card,1,3.8,UD-Q4_K_XL,think,medium,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,12,1.0,5028,79.3,,0,1786841694 +p2_extract_qwen38-think-xhigh-card_v1,mmbt-q38-card,p2_extract,qwen38-think-xhigh-card,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,18,1.0,12080,190.2,,0,1786833549 +p2_extract_qwen38-27b-udq4xl-think-xhigh_v1,mmbt-qwen38-eaaa8ca,p2_extract,qwen38-27b-udq4xl-think-xhigh,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,12,1.0,9394,143.4,,0,1786774809 +p2_extract_qwen38-27b-udq4xl-think-xhigh_v2,mmbt-qwen38-eaaa8ca,p2_extract,qwen38-27b-udq4xl-think-xhigh,2,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,19,1.0,11613,181.5,,0,1786781447 +p2_extract_qwen38-27b-udq4xl-think-xhigh_v3,mmbt-qwen38-eaaa8ca,p2_extract,qwen38-27b-udq4xl-think-xhigh,3,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,17,1.0,12647,195.4,,0,1786775005 +p2_hallucination_qwen38-think-low-card_v1,mmbt-q38-card,p2_hallucination,qwen38-think-low-card,1,3.8,UD-Q4_K_XL,think,low,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,23,1.0,9600,153.1,,0,1786848118 +p2_hallucination_qwen38-think-medium-card_v1,mmbt-q38-card,p2_hallucination,qwen38-think-medium-card,1,3.8,UD-Q4_K_XL,think,medium,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,10,1.0,7410,119.3,,0,1786841915 +p2_hallucination_qwen38-think-xhigh-card_v1,mmbt-q38-card,p2_hallucination,qwen38-think-xhigh-card,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,30,1.0,16124,258.6,,0,1786834115 +p2_hallucination_qwen38-27b-udq4xl-think-xhigh_v1,mmbt-qwen38-eaaa8ca,p2_hallucination,qwen38-27b-udq4xl-think-xhigh,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,29,1.0,16310,259.3,,0,1786775584 +p2_hallucination_qwen38-27b-udq4xl-think-xhigh_v2,mmbt-qwen38-eaaa8ca,p2_hallucination,qwen38-27b-udq4xl-think-xhigh,2,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,34,1.0,20128,328.9,,0,1786782368 +p2_hallucination_qwen38-27b-udq4xl-think-xhigh_v3,mmbt-qwen38-eaaa8ca,p2_hallucination,qwen38-27b-udq4xl-think-xhigh,3,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,31,1.0,17220,272.9,,0,1786775858 +p2_triage_qwen38-think-low-card_v1,mmbt-q38-card,p2_triage,qwen38-think-low-card,1,3.8,UD-Q4_K_XL,think,low,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,11,1.0,11673,181.7,,0,1786848301 +p2_triage_qwen38-think-medium-card_v1,mmbt-q38-card,p2_triage,qwen38-think-medium-card,1,3.8,UD-Q4_K_XL,think,medium,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,9,1.0,7613,118.6,,0,1786842035 +p2_triage_qwen38-think-xhigh-card_v1,mmbt-q38-card,p2_triage,qwen38-think-xhigh-card,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,13,1.0,20129,316.6,,0,1786834433 +p2_triage_qwen38-27b-udq4xl-think-xhigh_v1,mmbt-qwen38-eaaa8ca,p2_triage,qwen38-27b-udq4xl-think-xhigh,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,13,1.0,19432,302.9,,0,1786782672 +p2_triage_qwen38-27b-udq4xl-think-xhigh_v2,mmbt-qwen38-eaaa8ca,p2_triage,qwen38-27b-udq4xl-think-xhigh,2,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,11,1.0,18552,285.9,,0,1786776144 +p2_triage_qwen38-27b-udq4xl-think-xhigh_v3,mmbt-qwen38-eaaa8ca,p2_triage,qwen38-27b-udq4xl-think-xhigh,3,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,11,1.0,18437,283.9,,0,1786782957 +p3_business_qwen38-think-low-card_v1,mmbt-q38-card,p3_business,qwen38-think-low-card,1,3.8,UD-Q4_K_XL,think,low,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,18,0.9444,16661,260.8,,0,1786848766 +p3_business_qwen38-think-medium-card_v1,mmbt-q38-card,p3_business,qwen38-think-medium-card,1,3.8,UD-Q4_K_XL,think,medium,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,21,1.0,16022,249.4,,0,1786842554 +p3_business_qwen38-think-xhigh-card_v1,mmbt-q38-card,p3_business,qwen38-think-xhigh-card,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,3,1,44,0.9318,58308,998.9,,0,1786836248 +p3_business_qwen38-27b-udq4xl-think-xhigh_v1,mmbt-qwen38-eaaa8ca,p3_business,qwen38-27b-udq4xl-think-xhigh,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,21,0.9524,25298,401.2,,0,1786784453 +p3_business_qwen38-27b-udq4xl-think-xhigh_v2,mmbt-qwen38-eaaa8ca,p3_business,qwen38-27b-udq4xl-think-xhigh,2,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,20,0.95,33381,529.0,,0,1786778049 +p3_business_qwen38-27b-udq4xl-think-xhigh_v3,mmbt-qwen38-eaaa8ca,p3_business,qwen38-27b-udq4xl-think-xhigh,3,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,30,1.0,45586,746.9,,0,1786785200 +p3_doc_qwen38-think-low-card_v1,mmbt-q38-card,p3_doc,qwen38-think-low-card,1,3.8,UD-Q4_K_XL,think,low,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,18,1.0,13024,203.1,,0,1786848505 +p3_doc_qwen38-think-medium-card_v1,mmbt-q38-card,p3_doc,qwen38-think-medium-card,1,3.8,UD-Q4_K_XL,think,medium,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,19,0.9474,16967,268.0,,0,1786842304 +p3_doc_qwen38-think-xhigh-card_v1,mmbt-q38-card,p3_doc,qwen38-think-xhigh-card,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,26,0.9231,48907,814.8,,0,1786835248 +p3_doc_qwen38-27b-udq4xl-think-xhigh_v1,mmbt-qwen38-eaaa8ca,p3_doc,qwen38-27b-udq4xl-think-xhigh,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,19,1.0,38515,627.1,,0,1786776772 +p3_doc_qwen38-27b-udq4xl-think-xhigh_v2,mmbt-qwen38-eaaa8ca,p3_doc,qwen38-27b-udq4xl-think-xhigh,2,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,35,0.9714,64062,1092.3,,0,1786784050 +p3_doc_qwen38-27b-udq4xl-think-xhigh_v3,mmbt-qwen38-eaaa8ca,p3_doc,qwen38-27b-udq4xl-think-xhigh,3,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,25,1.0,46138,745.5,,0,1786777519 +p3_market_qwen38-think-low-card_v1,mmbt-q38-card,p3_market,qwen38-think-low-card,1,3.8,UD-Q4_K_XL,think,low,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,81,1.0,38677,704.3,,0,1786849471 +p3_market_qwen38-think-medium-card_v1,mmbt-q38-card,p3_market,qwen38-think-medium-card,1,3.8,UD-Q4_K_XL,think,medium,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,133,1.0,67114,2172.2,,0,1786844727 +p3_market_qwen38-think-xhigh-card_v1,mmbt-q38-card,p3_market,qwen38-think-xhigh-card,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,176,1.0,109811,3252.0,,0,1786853192 +p3_market_qwen38-27b-udq4xl-think-xhigh_v1,mmbt-qwen38-eaaa8ca,p3_market,qwen38-27b-udq4xl-think-xhigh,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,254,1.0,132034,3838.8,,0,1786781888 +p3_market_qwen38-27b-udq4xl-think-xhigh_v2,mmbt-qwen38-eaaa8ca,p3_market,qwen38-27b-udq4xl-think-xhigh,2,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,264,1.0,133720,3660.3,,0,1786788862 +p3_market_qwen38-27b-udq4xl-think-xhigh_v3,mmbt-qwen38-eaaa8ca,p3_market,qwen38-27b-udq4xl-think-xhigh,3,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,STRUCTURAL_PASS,1,1,1,0,0,1,1,192,1.0,124036,3335.8,,0,1786785226 +p3_pm_qwen38-think-low-card_v1,mmbt-q38-card,p3_pm,qwen38-think-low-card,1,3.8,UD-Q4_K_XL,think,low,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,15,1.0,9952,155.3,,0,1786849921 +p3_pm_qwen38-think-medium-card_v1,mmbt-q38-card,p3_pm,qwen38-think-medium-card,1,3.8,UD-Q4_K_XL,think,medium,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,12,1.0,6366,99.9,,0,1786845031 +p3_pm_qwen38-think-xhigh-card_v1,mmbt-q38-card,p3_pm,qwen38-think-xhigh-card,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,27,1.0,30239,484.0,,0,1786854319 +p3_pm_qwen38-27b-udq4xl-think-xhigh_v1,mmbt-qwen38-eaaa8ca,p3_pm,qwen38-27b-udq4xl-think-xhigh,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,19,1.0,19762,307.9,,0,1786786169 +p3_pm_qwen38-27b-udq4xl-think-xhigh_v2,mmbt-qwen38-eaaa8ca,p3_pm,qwen38-27b-udq4xl-think-xhigh,2,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,20,1.0,23293,371.1,,0,1786790421 +p3_pm_qwen38-27b-udq4xl-think-xhigh_v3,mmbt-qwen38-eaaa8ca,p3_pm,qwen38-27b-udq4xl-think-xhigh,3,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,PASS,1,1,1,0,0,1,1,17,1.0,22658,351.2,,0,1786786521 +p3_writing_qwen38-think-low-card_v1,mmbt-q38-card,p3_writing,qwen38-think-low-card,1,3.8,UD-Q4_K_XL,think,low,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,24,1.0,18543,293.3,,0,1786849765 +p3_writing_qwen38-think-medium-card_v1,mmbt-q38-card,p3_writing,qwen38-think-medium-card,1,3.8,UD-Q4_K_XL,think,medium,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,23,0.9565,12694,201.8,,0,1786844930 +p3_writing_qwen38-think-xhigh-card_v1,mmbt-q38-card,p3_writing,qwen38-think-xhigh-card,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,26,1.0,39291,640.8,,0,1786853834 +p3_writing_qwen38-27b-udq4xl-think-xhigh_v1,mmbt-qwen38-eaaa8ca,p3_writing,qwen38-27b-udq4xl-think-xhigh,1,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,26,1.0,34973,565.7,,0,1786789429 +p3_writing_qwen38-27b-udq4xl-think-xhigh_v2,mmbt-qwen38-eaaa8ca,p3_writing,qwen38-27b-udq4xl-think-xhigh,2,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,1,1,31,1.0,39042,632.1,,0,1786785860 +p3_writing_qwen38-27b-udq4xl-think-xhigh_v3,mmbt-qwen38-eaaa8ca,p3_writing,qwen38-27b-udq4xl-think-xhigh,3,3.8,UD-Q4_K_XL,think,xhigh,T1/p0.95/pp0,1.0,0.95,20,0.0,42,FAIL,1,0,1,0,0,2,1,35,0.9714,38498,619.5,,0,1786790049 diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/findings.md b/benchmarks/qwen36-vs-qwen38-27b-2026-08/findings.md new file mode 100644 index 00000000..64ed05c3 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/findings.md @@ -0,0 +1,435 @@ +> [!IMPORTANT] +> **EXPLORATORY — notice added 2026-08-16 per the corrective protocol, section 9. No number +> below has been altered; this notice is prepended only.** +> +> 1. **Arm mislabel (V36 vendor no-think).** The arm labelled "3.6 vendor no-think" ran +> Qwen3.6-27B at its **thinking** sampler (T1.0 / top_p 0.95 / presence_penalty 0) with +> thinking disabled — off-spec per the pinned Qwen3.6-27B card (rev `6a9e13bd`, README +> L637: official non-thinking is T0.7 / top_p 0.80 / top_k 20 / min_p 0 / pp 1.5 / rp 1, +> identical to Qwen3.8-27B). The "vendor vs vendor" no-think contrast in this entry is +> therefore mode/sampler-confounded, and this corpus contains **no cell of Qwen3.6 +> non-thinking at its official sampler**. +> 2. **Cell-level Fisher exact values in this entry are descriptive only.** They treat cells +> as independent and are anti-conservative under family/seed clustering; they are not +> confirmatory inference. +> 3. **This 802-cell freeze is exploratory evidence.** It feeds no confirmatory claim. The +> confirmatory design, arms, fixed N, and analysis plan are preregistered at +> [`../qwen36-vs-qwen38-corrective-2026-08/PREREGISTRATION.md`](../qwen36-vs-qwen38-corrective-2026-08/PREREGISTRATION.md); +> the pinned-card sampler evidence is recorded at +> [`../qwen36-vs-qwen38-corrective-2026-08/protocol/CARD-EVIDENCE.md`](../qwen36-vs-qwen38-corrective-2026-08/protocol/CARD-EVIDENCE.md). + +# 2026-08 — Qwen3.6-27B vs Qwen3.8-27B (UD-Q4_K_XL) across 12 task families + +> Head-to-head of Qwen3.6-27B and Qwen3.8-27B, both at Unsloth UD-Q4_K_XL, on the 12-family +> MMBT suite: 802 frozen cells over 9 sampler/mode arms, plus a 19-cell Q8_0 control. +> The campaign also audited MMBT's own graders and found three defects that materially +> distort the raw numbers; corrections ship as a non-destructive overlay, and five earlier +> conclusions from this investigation are formally retracted in [`claims.yaml`](../../claims.yaml). +> +> **Top-line:** +> +> 1. **The one large, unambiguous effect is a delivery regression in no-think mode, not a +> capability regression.** 3.8's identical-call-loop rate is +28.0 pp over 3.6 at the +> matched sampler (Fisher p = 4.9e-09) and +29.2 pp at each model's own vendor sampler +> (p = 1.1e-11). This is measured upstream of grading — no scoring choice and none of the +> three grader defects can touch it. +> 2. **The regression disappears in thinking mode.** 3.8's loop rate falls to 2/51 (T0.3) +> and 0/72 (T1), neither distinguishable from 3.6. +> 3. **Conditional on delivering, and after the grader corrections, the models are within +> noise of each other at the matched sampler** (86.0% vs 86.2% no-think; 92.5% vs 89.6% +> think). "3.8 is worse at the task" is not a supportable summary of this corpus; +> "3.8 fails more often by not finishing" is. The vendor-point conditional gap (~15 pp) +> survives correction and is unresolved. +> 4. **Thinking costs 3.8 roughly 2.2–3.0× more tokens with no pass-rate gain**, and its +> `reasoning_effort` ladder buys tokens rather than quality (no step is significant). +> 5. **Three grader defects are verified and corrected by overlay** — a word-count tokenizer +> mismatch (62 counter-dependent verdicts), a `p3_pm` keyword literalism (36 verdict +> flips), and a `p2_triage` brief/ground-truth contradiction (24 flips). Two of the three +> corrected families saturate and stop discriminating. See [`grader-defects.md`](grader-defects.md). +> 6. **The identical-call loop reproduces at Q8_0** (6/19, Wilson 95% [15.4%, 54.0%], +> excluding zero) — the no-think loop is not an artifact of the 4-bit quantization. + +--- + +## Provenance + +| Item | Value | +| --- | --- | +| Frozen dataset | [`data/mmbt-frozen-dataset-v2.csv`](data/mmbt-frozen-dataset-v2.csv) — freeze #2, 802 cells (733 graded), frozen `2026-08-16T14:23:09Z` | +| Frozen dataset sha256 | `d2ed0beca5b68e9ca63788e452235f23a299af06639893f558bef19f784cf018` | +| Freeze stamp | [`data/FREEZE2_STAMP.txt`](data/FREEZE2_STAMP.txt) | +| Supersedes | freeze #1 (746 cells, `2026-08-16T11:49:14Z`), which was cut while campaign processes were still writing cells | +| Correction overlay | [`overlay/`](overlay/) — 165 per-cell records + manifest, digest `e332cd2c78ad94fe264aed7d31e6c64f5273cd9598f889dc1a8fac539e971351` | +| Full statistical tables | [`results-tables.md`](results-tables.md) (rendered from [`results.json`](results.json)) | +| Grader-defect report | [`grader-defects.md`](grader-defects.md) | +| Methodology audit (locked/varies, bias list B1-B11) | [`AUDIT.md`](AUDIT.md) | + +Every number in this entry derives from the frozen CSV and the immutable run artifacts — +never from a live re-scan. Model, quantization and sampler identity come from each run's +`receipt.json`, never from directory names (one arm named `-offspec` was found actually +carrying the vendor-card sampler; filename inference is unsafe). The corpus is static: +every campaign process is stopped, in-flight cells are quarantined out of the extract, and +the overlay manifest's `post_freeze_divergence` ledger is empty — the on-disk verdict +agrees with the frozen verdict for all 733 graded rows. Raw per-run logs and workspaces +stay outside the repo per `REPO-SPACE.md` (the workspace tarballs alone are ~1.5 GB); the +frozen CSV, the overlay, and the generators in [`tooling/`](tooling/) are the publishable, +auditable derivation chain. + +All 802 runs used seed 42. Replicates are repeat runs at a fixed seed, not a seed sweep — +see the seed-correlation caveat added to [`KNOWN-LIMITATIONS.md`](../../KNOWN-LIMITATIONS.md). + +## What was tested + +| Group | Definition | Cells | Graded | Notes | +|---|---|---:|---:|---| +| `A36` | 3.6 no-think T0.3/p0.8/pp0 | 121 | 114 | off-spec for 3.6; sampler-matched to A38 | +| `A38` | 3.8 no-think T0.3/p0.8/pp0 | 95 | 65 | off-spec for 3.8; sampler-matched to A36 | +| `V36` | 3.6 no-think T1/p0.95/pp0 | 120 | 120 | 3.6 vendor point | +| `V38` | 3.8 no-think T0.7/p0.8/pp1.5 | 96 | 79 | 3.8 vendor point | +| `B36` | 3.6 think T0.3/p0.8/pp0 | 108 | 107 | sampler-matched to B38 | +| `B38` | 3.8 think T0.3/p0.8/pp0 | 51 | 48 | mixed effort (14 low / 24 medium / 13 xhigh) | +| `C36` | 3.6 think T1/p0.95/pp0 | 120 | 120 | 3.6 vendor point | +| `C38` | 3.8 think T1/p0.95/pp0 | 72 | 72 | 3.6's vendor sampler, NOT 3.8's; mixed effort (12/12/48) | +| `Q8` | 3.8 **Q8_0** no-think T0.3/p0.8/pp0 | 19 | 8 | quantization control | + +Four paired contrasts are reported throughout: **P1** (no-think, sampler-matched A36/A38), +**P2** (no-think, vendor-matched V36/V38), **P3** (think, sampler-matched B36/B38), and +**P4** (think at T1 — which is 3.6's vendor point and not 3.8's, so P4 is biased in 3.6's +favour by construction). Group composition, arm pooling, and the missing-data structure are +in [`results-tables.md`](results-tables.md) §0. + +## Delivery reliability + +Delivery means the run produced a gradeable artifact (`graded == 1` in the frozen CSV). +Loop detection uses two mechanical detectors recorded per cell: `looped_freq30` (≥30 +occurrences of one digit-stripped tool template) and the stricter `looped_run30` (≥30 +consecutive). The loop metric is upstream of every grader and every grader defect. + +| Contrast | Metric | 3.6 | 3.8 | Δ (3.8−3.6) | Fisher p | +|---|---|---|---|---:|---:| +| P1 matched no-think | Delivered | 114/121 94.2% | 65/95 68.4% | −25.8 pp | 8.2e-07 | +| P1 matched no-think | Loop (freq30) | 3/121 2.5% | 29/95 30.5% | **+28.0 pp** | **4.9e-09** | +| P2 vendor no-think | Delivered | 120/120 100% | 79/96 82.3% | −17.7 pp | 4.4e-07 | +| P2 vendor no-think | Loop (freq30) | 0/120 0.0% | 28/96 29.2% | **+29.2 pp** | **1.1e-11** | +| P3 matched think | Loop (freq30) | 1/108 0.9% | 2/51 3.9% | +3.0 pp | 0.2412 | +| P4 think at T1 | Loop (freq30) | 0/120 0.0% | 0/72 0.0% | +0.0 pp | 1 | + +Structure of the regression: + +- **It is concentrated, not uniform.** At the matched sampler 3.8 delivers 1/12 on + `p1_bugfix` and 2/13 on `p1_testwrite`, while matching 3.6 at 100% delivery on all four + phase-2 families (7/7 each). At the vendor points, 3.6's per-family delivery ≥ 3.8's in + 12 of 12 families (5 strict, 7 ties). +- **Thinking removes most of it.** 3.8 delivers 48/51 (94.1%) in think mode at the matched + sampler against 65/95 (68.4%) no-think, and 72/72 at T1. 3.6 is at or near ceiling in + think mode on both samplers (107/108, 120/120). +- **Terminal rate is not the story** — 97.5–100% everywhere. 3.8's non-deliveries are runs + that looped until an operator or the harness gave up, not crashes. + +There is deliberately **no "abort rate" in this entry**. The operator's +identical-call-loop label disagrees with the mechanical loop signal on 24 of 802 cells in +both directions, so a rate built on the label would measure operator intervention as much +as model behavior — see the retraction `bench.qwen38.nothink-abort-rate` in +[`claims.yaml`](../../claims.yaml). + +## Quality, as graded + +Two scorings per pair, because they answer different questions and their disagreement is +the finding. All-cells counts loops as failures (end-to-end); graded-only conditions on +delivery (maximally charitable to a model that fails by not finishing). + +| Contrast | Scoring | 3.6 | 3.8 | Δ (3.8−3.6) | Fisher p | +|---|---|---|---|---:|---:| +| P1 matched no-think | All cells | 81/121 66.9% | 51/95 53.7% | −13.3 pp | 0.05052 | +| P1 matched no-think | Graded only | 81/114 71.1% | 51/65 78.5% | **+7.4 pp** | 0.2959 | +| P2 vendor no-think | All cells | 95/120 79.2% | 48/96 50.0% | −29.2 pp | 1.1e-05 | +| P2 vendor no-think | Graded only | 95/120 79.2% | 48/79 60.8% | −18.4 pp | 0.006102 | +| P3 matched think | All cells | 67/108 62.0% | 33/51 64.7% | +2.7 pp | 0.8607 | +| P3 matched think | Graded only | 67/107 62.6% | 33/48 68.8% | +6.1 pp | 0.5863 | +| P4 think at T1 | All cells | 105/120 87.5% | 54/72 75.0% | −12.5 pp | 0.03081 | + +In P1 the sign flips between the two scorings: 3.8 is 13.3 pp worse end-to-end and 7.4 pp +*better* among graded cells. Across the full seven-scoring sensitivity grid +([`results-tables.md`](results-tables.md) §7), P1 and P3 flip sign with the scoring rule +(no quality claim in either direction survives), while P2 and P4 are directionally robust +but carry structural caveats — P2 ties model identity to presence_penalty, and P4 runs 3.8 +off-spec at 3.6's vendor sampler. These are **as-graded** numbers containing the three +verified grader defects; per-family breakdowns with denominators are in +[`results-tables.md`](results-tables.md) §3, and no family row is individually conclusive +(n = 4–19 per side). + +No family-paired win count is published. At the matched sampler the count is 4 (3.6) / 5 +(3.8) / 3 tied on raw grades and 4 / 3 / 5 after correction — not stable, and two of the +twelve families were contributing wins on the strength of grader defects. See the +retraction `bench.family-paired.nine-three`. + +## Quality conditional on delivery + +After applying the correction overlay (D2 and D3 verdict corrections; D1 gate-invalidated +cells counted as non-failures — see [`grader-defects.md`](grader-defects.md) for why those +62 verdicts are counter-dependent rather than valid FAILs): + +| Contrast | 3.6 corrected | 3.8 corrected | Δ (3.8−3.6) | +|---|---|---|---:| +| P1 matched no-think | 98/114 86.0% | 56/65 86.2% | +0.2 pp | +| P1, excluding `p2_triage` | 89/105 84.8% | 49/58 84.5% | −0.3 pp | +| P3 matched think | 99/107 92.5% | 43/48 89.6% | −2.9 pp | +| P3, excluding `p2_triage` | 90/98 91.8% | 39/44 88.6% | −3.2 pp | +| P2 vendor no-think | 110/120 91.7% | 61/79 77.2% | **−14.5 pp** | +| P4 think at T1 | 116/120 96.7% | 72/72 100.0% | +3.3 pp | + +At the matched sampler all four contrasts sit within about three percentage points — 3.6 +nominally ahead in three, behind by 0.2 pp in one — and on 3.8-side denominators of 44–65 +cells a one-to-two-cell swing reverses any of them. Excluding the D1 cells from the +denominators instead of counting them as non-failures moves nothing outside noise +(no-think 85.7% vs 85.0%; think 91.4% vs 87.8%), so the reading does not hinge on the D1 +treatment. + +Two warnings that travel with this table: + +- **The conditioning is the whole problem.** Delivery is exactly where the models differ, + so conditioning on delivery conditions on a non-random subset — 3.8's delivered cells + are its surviving runs. This describes graded work, never expected outcome per attempt. +- **The vendor-point gap survives correction.** 91.7% vs 77.2% (excluding D1 cells: + 91.4% vs 73.9%) is not explained by any of the three defects and is unresolved. It is + also confounded by construction: the vendor design ties model identity to + presence_penalty (pp=1.5 on 3.8's card, pp=0 on 3.6's), and its no-quality-confound + replication (P1, pp=0 both sides) flips sign across scorings. + +## Cost + +Family-matched medians only (families differ by >10× in cost, and `completion_tokens` is +missing exactly for the cells that looped — 64 of 802, all ungraded — so unmatched pooled +medians are meaningless; comparability rules in [`results-tables.md`](results-tables.md) §4): + +- **No-think, matched (P1):** token ratio ×0.86 over the 9 comparable families — but the + excluded families (`p1_bugfix`, `p1_testwrite`, `p3_market`) are the expensive ones + (×5.7 the median cost of the retained ones) and are excluded *because* 3.8's looping + destroyed its cost accounting there. **Not readable as "3.8 is cheaper."** +- **Think (P3/P4):** token ratio ×2.18 / ×2.97, wall-clock ×2.20 / ×2.96. 3.8 pays a + large thinking tax relative to 3.6 with no accompanying pass-rate gain. +- **Effort ladder (within 3.8 think):** no pairwise effort step is statistically + distinguishable; `low` → `xhigh` raises median tokens 1.8–2.7× with no pass-rate change. + The apparent cost non-monotonicity (`low` costing more than `medium` at T0.3) traces to + the chat template — `medium` is the un-instructed baseline; `low` injects a brevity + instruction that does not buy brevity — but it does not reproduce at T1 and is claimed + only at that strength (see `bench.qwen38.reasoning-effort-cost-ladder`). + +## Q8_0 quantization control + +19 cells (2 replicates on the 7 phase-1/2 families, 1 on the 5 phase-3 families), 3.8 +Q8_0, no-think, matched sampler. At freeze #1 this arm had zero graded cells; freeze #2 +admits 8 (5 PASS, 3 FAIL), which upgrades it from existence-only evidence to **provisional +rates with wide intervals**: + +- **Loop rate 6/19 = 31.6%, Wilson 95% [15.4%, 54.0%] — the interval excludes zero.** The + identical-call loop occurs at Q8_0 at a real, non-negligible rate; it is not an artifact + of the UD-Q4_K_XL quantization. Maximum identical-call runs: 110, 109, 81, 80, 71. +- **Against 3.8-Q4 at the same sampler, the loop rate is statistically indistinguishable** + (29/95 = 30.5% vs 6/19 = 31.6%, Fisher p = 1.0) — consistent with quantization playing + no role in the loop, though the Q8_0 interval is wide. +- **Quality stays near-uninformative.** Graded-only 5/8 = 62.5% [30.6%, 86.3%]; the vs-Q4 + contrast (78.5% vs 62.5%, p = 0.38) fails this entry's own power screen and no quality + verdict is drawn. Of the three FAILs, two are length overruns by *both* word counters + (D1 does not rescue them) and one is the D2 literalism (corrected to PASS). +- **There is no matched 3.6 Q8_0 arm**, so this is not a quant A/B: it can neither + attribute the delivery regression to quantization nor exonerate it. + +Per-cell detail is in [`results-tables.md`](results-tables.md) §6. One additional Q8_0 +observation is unscored and excluded from every rate; it is reported in the next section. + +## Qualitative exhibits + +Three exhibits from the run evidence. These carry no statistical weight; they are here +because they show the *mechanisms* behind the numbers above. + +### 1. The `copyfileobj` misdiagnosis spiral (`p1_bugfix`, 3.8 no-think) + +`p1_bugfix_qwen38-nothink-offspec_v1` is one of the 10 loop-flagged 3.8 `p1_bugfix` cells +at the matched sampler. Its first nine iterations are exemplary: read the repo, run the +test suite, run the benchmark, reproduce the slowdown. At iteration 10 it decides to time +`load()` on a smaller slice of the 50 MB benchmark log, and builds the slice with: + +```python +shutil.copyfileobj(open('benchmarks/bench_50mb.log','rb'), + open('/tmp/bench_5mb.log','wb'), 5*1024*1024) +``` + +The third argument of `copyfileobj` is the **copy buffer size, not a byte limit** — this +call copies the entire 50 MB file regardless. So the "5 MB slice" was 50 MB, `load()` on +it hit the 300 s bash timeout, and the probe returned nothing. The model's response to +the timeout was to shrink the slice: 1 MB (240 s timeout), 100 KB (120 s), 10 KB (60 s), +1 KB (30 s), 100 bytes (20 s) — and then, with an 8-second timeout, byte by byte: 35, 30, +25, 20, 15, 12, 11, 10, 9, 8, 7, and finally a **6-byte "slice"**, each probe still +silently copying all 50 MB and timing out identically. 41 probes over iterations 10–51, +every one the same template differing only in numerals, ~18 of the run's 30 wall minutes +spent inside them. At no point did the model question the slicing primitive; every timeout +was read as "the input is still too large." The operator SIGTERM'd it at iteration 52 +under the ≥30-identical-digit-stripped-template rule. + +This is what the +28 pp loop delta is made of: not gibberish, but a plausible debugging +strategy built on one wrong API assumption, iterated with perfect confidence and zero +self-correction. The transcript is preserved in the campaign archive +(`logs/` is not published in this repo; see Reproduction below). + +### 2. The 90-day board signal (`p3_writing`): a real framing difference, and another keyword gate + +The `p3_writing` CEO-brief spec requires "second-occurrence-in-90-days framing (board +signal)". The grader checks it with the keyword list +`["90 days", "90-day", "second time", "second 4"]`. On the 59 graded cells: + +| Arm | Keyword hit | +|---|---| +| 3.8 (both modes, all samplers) | **21/21** | +| 3.6 think | 19/19 | +| 3.6 no-think | **8/19** (matched T0.3: 2/9; vendor T1: 6/10) | + +3.8 preserves the source memo's own framing essentially verbatim, e.g. +(`p3_writing_qwen38-nothink-offspec_v1`): + +> This is the second 4+ hour outage in 2026 (the first was the February database storage +> issue) — the second data point in 90 days. + +But the 11 "missing" 3.6 no-think briefs are not missing the signal. Every one carries an +explicit board-signal line — e.g. (`p3_writing_qwen36-nothink-offspec_v1`): + +> **Board Signal:** This is the second outage of 4+ hours in 2026. The first occurred in +> February. A third would compel a full reliability program review. + +The honest reading is layered. There **is** a real, systematic difference: 3.8 (and 3.6 +in think mode) reproduces the memo's sharp 90-day-window framing; 3.6 no-think reliably +paraphrases it into a calendar-year framing, dropping the 90-day window — the compression +that makes the signal urgent — and the paraphrase is near-identical across replicates +(fixed-seed replicate correlation; see KNOWN-LIMITATIONS). A CEO reading 3.6's version +gets a diluted signal. But the grader cannot see any of that: it can only see keywords, so +it scores "second outage of 4+ hours in 2026, first in February" — which contains the +second occurrence, the window's endpoints, and the escalation rule — identically to a +brief with no board signal at all. That is the same instrument shape as defect D2, in a +family this PR's overlay does **not** correct (only D1's length gate touches +`p3_writing`). The keyword rates above are reported as instrument readings, not as +"3.6 no-think misses the board signal." + +### 3. The Q8_0 rewrite loop (unscored) + +`p3_doc_qwen38q8-nothink-matched_v2` was quarantined in-flight at freeze #2 and is **not a +row in the frozen CSV**: it was observed 139 iterations deep in a rewrite loop — +re-emitting `brief.md` over and over with its context grown to ~228k tokens when +quarantined. This is a different failure subclass from the identical-call loop the +detectors count (`rewrite-loop` in +[`tooling/FAILURE-TAXONOMY.md`](../../tooling/FAILURE-TAXONOMY.md), first documented on +27B in `microbench-phase-b-2026-05-02`), it is UNSCORED, and it is excluded from every +rate and denominator in this entry. It is disclosed because a *second* loop subclass at +Q8_0 is qualitatively relevant to the quantization question even though it carries no +statistical weight. + +## Grader defects, in one screen + +Full report with sources, natural experiments, and per-cell appendix: +[`grader-defects.md`](grader-defects.md). Corrections are a non-destructive overlay +([`overlay/`](overlay/)); no `grade.json`, brief, or ground-truth file was modified. + +| | D1 word-gate tokenizer mismatch | D2 `p3_pm` keyword literalism | D3 `p2_triage` brief contradiction | +| --- | --- | --- | --- | +| Families | `p3_business`, `p3_doc`, `p3_writing` | `p3_pm` | `p2_triage` | +| Mechanism | grader counts `\b\w+\b`, models budget with `wc -w`; 62 of 71 length-only FAILs are under every ceiling by `wc -w` | grader requires "legal hasn't"; 3.6 writes "legal has not" | brief defines urgency `n/a` for spam; ground truth never uses it; all 64 cells penalized identically for complying | +| Verdict changes | 62 gate invalidations (23 of 3.6, 39 of 3.8) | 36 flips (31 / 5) | 24 flips (21 / 3) | +| Direction | favours 3.6 uncorrected | favours 3.8 uncorrected | favours 3.8 uncorrected | +| After correction | gate is not a valid discriminator | family saturates (61/64) | family saturates (64/64) | + +The three defects do not push the same way, and in each case the correct conclusion is +"this gate does not measure what the benchmark claims", not "the other model actually +won". Under the shipped graders the three length-gated families read as a 34-point win +for 3.6 (64.9% vs 30.4%); with the other equally-defensible counter they read as a 2-point +win for 3.8 (85.1% vs 87.0%). The gate was deciding the headline. + +D2 has a process finding worse than the grader bug: the repository already contained the +tested fix (`tooling/correct_gemma4_project_mgmt_grades.py`) — it never ran on these +campaigns because its cell enumeration was hardcoded to `p3_pm_gemma4-31b-q4_v{n}` names. +Prior-campaign exposure is measured in +[`grader-defects.md`](grader-defects.md) § What these defects mean for previously +published MMBT results: 2 of 10 Gemma4 `p2_triage` verdicts flip; every published +`p2_triage` urgency accuracy for every model is understated by 0.100; DeepSeek's verdicts +stand. + +## Retractions + +Five conclusions drawn earlier in this investigation are retracted in +[`claims.yaml`](../../claims.yaml) (entries persist there permanently; each carries its +full reason): + +1. `bench.qwen38.nothink-abort-rate` — an operator-label-based "abort rate"; the label + disagrees with the mechanical loop signal in both directions (24 of 802 cells). +2. `bench.p3_pm.qwen38-outscores-qwen36` — "3.8 wins p3_pm 17/21 vs 4/37"; the gap was + lexical (D2). Corrected: 26/26 vs 35/38, near-saturation. +3. `bench.p2_triage.family-result` — "3.8 wins p2_triage 19/22 vs 16/37"; the split was a + uniform 0.100 penalty interacting with a threshold (D3). Corrected: 64/64. +4. `bench.family-paired.nine-three` — a 9–3 family win split; not reproducible from the + frozen data at any sampler, not stable under correction, and built on the two defect + families above. +5. `bench.thinking.default-vs-default` — a thinking comparison described as + "default vs default"; the only shared think sampler is 3.6's vendor point, and 3.8's + think arms are effort mixtures. Nothing was ever run at 3.8's own vendor think point. + +## What we did not run + +- **No 3.8 think arm at 3.8's own vendor sampler (T0.7/p0.8/pp1.5).** This is the largest + hole in the comparison and cannot be patched by re-analysis; the cross-model thinking + question is formally held (`bench.qwen38-vs-qwen36.thinking-quality-head-to-head.held`). + Running it at a fixed effort level is the single highest-value follow-up. +- **No matched 3.6 Q8_0 arm**, so the quant question stays open (a Q8_0 arm at N≥5 per + family for both models, graded, would close it). +- **No hand-graded quality pass.** Every phase-3 grader is a keyword/word-count + instrument, and every hand-rating slot in all 292 phase-3 `grade.json` files is null. + Corrected pass rates inherit that limit in full. +- **No seed sweep** — 802 cells, one seed. + +## Reproduction + +Everything derives from [`data/mmbt-frozen-dataset-v2.csv`](data/mmbt-frozen-dataset-v2.csv) +(sha256 `d2ed0bec…`, stamp `2026-08-16T14:23:09Z`) plus immutable run artifacts. + +- **Tables:** [`tooling/build_results.sh`](tooling/build_results.sh) runs + `test_stats.py` (estimator self-validation against defining equations), + `mmbt_results.py` (writes `results.json`), `defect_diag.py` (appends grader-defect + diagnostics), `make_md.py` (renders `results-tables.md`). Pure python3 stdlib. The + rebuild is deterministic — two runs produce byte-identical output. +- **Overlay:** [`tooling/apply_grade_corrections.py`](tooling/apply_grade_corrections.py) + `--dataset ` regenerates [`overlay/`](overlay/); + [`tooling/verify_overlay.py`](tooling/verify_overlay.py) checks idempotence (byte-identical + re-run, stable digest), the leniency invariant (corrections only ever flip FAIL→PASS; + 686 cells checked, 0 violations), the write guard (`grade.json` / `receipt.json` / + tasks / ground_truth refuse writes; 853 protected files hash-verified unmutated), and + the empty `post_freeze_divergence` ledger. +- **Cross-file consistency:** [`tooling/validate_fixes.py`](tooling/validate_fixes.py) + recomputes every corrected rate, defect count, win count, and Q8_0 figure from the + frozen CSV plus overlay and checks them against this entry's documents — 174 checks. +- **Freeze:** [`tooling/freeze_dataset.py`](tooling/freeze_dataset.py) is the script that + produced the CSV from the run checkouts (identity from `receipt.json`, never from + directory names). +- **Fastest single check** (D2's natural experiment, three commands, no scripts): + [`grader-defects.md`](grader-defects.md) Appendix B. Expect `card_v2`/`card_v8` + ("Legal has not responded") to FAIL at `risk_recall 2/6` and `card_v4` + ("Legal hasn't responded") to PASS at `3/6` with every other score identical. + +Two practical notes. The scripts' default dataset paths point at the freeze location on +the bench host (`/home/michael/mmbt-frozen-dataset-v2.csv`); the identical CSV ships in +this entry's `data/`. Raw `logs/` (transcripts, workspaces, `grade.json`) are not in this +repository per `REPO-SPACE.md` — the overlay records carry the sha256 of every raw file +they touched, so any single correction is checkable in isolation against the archive. + +## Relation to prior MMBT entries + +- The loop subclasses seen here are the ones first catalogued in + [`microbench-phase-b-2026-05-02/findings.md`](../microbench-phase-b-2026-05-02/findings.md) + (`scroll-loop`, `word-trim-loop`, `rewrite-loop`); the copyfileobj spiral is a + scroll-loop-shaped probe loop, and the Q8_0 quarantined cell is a rewrite-loop. The + ≥30-identical-digit-stripped-template SIGTERM rule from that entry was the operator + policy for this campaign. +- The grader-defect findings retroactively affect prior campaigns that used the + byte-identical graders (Gemma4, DeepSeek V4 Flash) — quantified in + [`grader-defects.md`](grader-defects.md). The `p2_triage` understatement applies to + every campaign ever published from these graders. +- This is the first MMBT entry built from a frozen dataset with a stamped freeze, + non-destructive correction overlay, and machine-checked document consistency + (`validate_fixes.py`). The freeze discipline exists because live re-scans during this + investigation moved headline percentages by 4–9 points between passes. \ No newline at end of file diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/grader-defects.md b/benchmarks/qwen36-vs-qwen38-27b-2026-08/grader-defects.md new file mode 100644 index 00000000..5021d09e --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/grader-defects.md @@ -0,0 +1,736 @@ +# Grader defects in this comparison, and the corrections applied + +This document reports three defects **in MMBT's own graders**, found while grading the +Qwen3.6-27B vs Qwen3.8-27B comparison. Two of them (D1, D3) change the direction of a +family-level result. One of them (D2) was already known to this repository and already +fixed here — but the fix was hardcoded to a different model's cell names and so never ran +on these campaigns. That is disclosed in full below rather than buried, because it is the +first thing a reviewer will find. + +Nothing in the run evidence was edited. `grade.json`, task briefs and ground-truth files +are immutable; corrections ship as a separate overlay, following the convention the +repository already states in `tooling/correct_gemma4_project_mgmt_grades.py:165`: + +> `"policy": "immutable grade.json files remain raw evidence; this overlay changes no run artifact or raw verdict"` + +--- + +## Provenance and evidence rules + +| Item | Value | +| --- | --- | +| Frozen dataset | `/home/michael/mmbt-frozen-dataset-v2.csv` (freeze #2), 802 rows, frozen 2026-08-16T14:23:09Z | +| Frozen dataset sha256 | `d2ed0beca5b68e9ca63788e452235f23a299af06639893f558bef19f784cf018` | +| Supersedes | freeze #1, `mmbt-frozen-dataset.csv`, 746 rows, 2026-08-16T11:49:14Z | +| Measurements in this document read at | 2026-08-16T14:30Z – 14:50Z | +| Cells used for every count below | the 733 of 802 rows with `graded == 1` | + +Every count in this document is restricted to frozen-dataset rows with `graded == 1`. +That restriction is not cosmetic. **The corpus was still being written while the freeze-1 +analysis ran** — two reads of the `p2_triage` family eleven minutes apart returned 58 and +then 59 scored cells, because a `grade.json` for a cell freeze #1 marked `graded=0` +appeared on disk in between. Freeze #2 closes that gap: every campaign process is stopped, +in-flight cells are quarantined out of the extract, and the overlay manifest's +`post_freeze_divergence` ledger is empty — the on-disk verdict agrees with the frozen +verdict for every one of the 733 graded rows. The corpus behind this document is static. +Cell identity, model and sampler are taken verbatim from the frozen dataset and never +recomputed from directory names. + +The graders are **byte-identical across all seven checkouts on this host** — the five +Qwen checkouts (`mmbt-q38-card`, `mmbt-qwen38-eaaa8ca`, `mmbt-q36-card`, +`mmbt-qwen36-compare`, `mmbt-q38-q8`) and the two prior campaign trees +(`bench-gemma4-31b-q4`, `bench-deepseek-v4-flash-0731`). This is what makes the +"previously published results" section at the end unavoidable. + +| File | sha256 | +| --- | --- | +| `tooling/graders/phase3_project_mgmt_grade.py` | `e1c3a9190e6d19cffd76734c8c922450b22d674bd5fd2db052fef31f91777bca` | +| `tooling/graders/phase3_doc_synthesis_grade.py` | `8c117c0a3bfec94a91f94e49bb5b4281a458bcc4a650e9ae86e2450f0f5cac0a` | +| `tooling/graders/phase3_business_memo_grade.py` | `691c5c6cec0f3bbf2fc86e80c47dd3fc2bde4873dc630297a0aea13edd548d7b` | +| `tooling/graders/phase3_writing_editing_grade.py` | `a3461f31be9d24cc1c5132fff4bff96a27af77af2457dad839ef5498afba8f56` | +| `tooling/graders/phase2_triage_grade.py` | `ea3eebcf6a2fc085276e555f2cb1a54239e1dc9286bac3cb2300f01929b855a1` | +| `tooling/graders/ground_truth/phase2_triage.json` | `6ca3763f32abfe3a5b8dfa56662c9083f91f81c2c3d06d3117bfb974367669d1` | +| `tooling/tasks/task_triage.md` | `d92feeb839dbb8850d6447c6c821fc4c74ae68cc53c4917438f8a10f9531292b` | +| `tooling/correct_gemma4_project_mgmt_grades.py` | `86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581` | + +--- + +## Summary + +| | D1 word-gate tokenizer mismatch | D2 `p3_pm` keyword literalism | D3 `p2_triage` brief contradiction | +| --- | --- | --- | --- | +| Families hit | `p3_business`, `p3_doc`, `p3_writing` | `p3_pm` | `p2_triage` | +| Cells corrected | 62 (23 of 3.6, 39 of 3.8) | 39 (33 of 3.6, 6 of 3.8) | 64 (38 of 3.6, 26 of 3.8) | +| Verdict changes | 62 gate invalidations (23 / 39) | 36 (31 of 3.6, 5 of 3.8) | 24 (21 of 3.6, 3 of 3.8) | +| Direction | favours 3.6 uncorrected | favours 3.8 uncorrected | favours 3.8 uncorrected | +| Honest post-correction status | **gate is not a valid discriminator** | family saturates; not a discriminator | family saturates; not a discriminator | + +The 3.8 columns include the Q8_0 control's graded cells (freeze #2 admits 8): one Q8_0 +cell each in the D2 and D3 counts. The Q8_0 `p3_business` and `p3_doc` FAILs are length +FAILs that stand under **both** counters, so D1 does not correct them. + +Read the last row carefully. In all three cases the correct conclusion is *"this gate +does not measure what the benchmark claims it measures"*, not *"the other model actually +won"*. Two of the three corrected families end at a ceiling where every cell of both +models passes, which is exactly as uninformative as the uncorrected result — just +uninformative in a way that is visible. + +--- + +## D1 — Word-gate tokenizer mismatch + +### The grader source + +Four phase-3 graders let a word count decide the verdict, and all four count words with +the same Python regex. + +`tooling/graders/phase3_doc_synthesis_grade.py` + +``` +48: word_count = len(re.findall(r"\b\w+\b", brief)) +... +68: word_limit = 700 +70: "PASS" if (captured >= coverage_threshold and word_count <= word_limit) +``` + +`tooling/graders/phase3_business_memo_grade.py` + +``` +73: word_count_memo = len(re.findall(r"\b\w+\b", memo.read_text())) +... +113: word_limit = 700 +115: "PASS" if (bias_recall >= bias_threshold and stance_pushback and word_count_memo <= word_limit) +``` + +`tooling/graders/phase3_writing_editing_grade.py` + +``` +17: def word_count(text: str) -> int: +18: return len(re.findall(r"\b\w+\b", text)) +... +92: wc = word_count(text) +93: max_words = brief.get("max_words", 9999) +... +110: "verdict": "PASS" if (required_pass and prohibited_pass and wc <= max_words) else "FAIL", +``` + +`tooling/graders/phase3_project_mgmt_grade.py` + +``` +64: word_count = len(re.findall(r"\b\w+\b", text)) +... +100: and word_count <= 700 +``` + +A fifth grader, `phase3_market_research_grade.py:38`, computes `rec_words` with the same +regex but never gates on it — it only reports it as +`"recommendation_word_count"` (line 86). It is therefore not affected and is excluded +from every D1 count below. + +### What the models counted with + +The task briefs impose the ceiling without ever defining how a word is counted: + +``` +task_doc_synthesis.md:8 1. Be **at most 700 words** — this is for a partner who has 5 minutes. +task_business_memo.md:9 Produce **a 1-page memo** (≤700 words) for the executive committee with: +task_writing_editing.md:20 - **Stay within word limits.** The max_words per audience is in `audience_briefs.json`. Use it as a hard ceiling. +``` + +Given a Linux VM and no definition, both models reached for the obvious tool. Across the +293 graded phase-3 cells, the transcript contains a literal `wc -w` invocation in: + +| Model | Cells with a `wc -w` call | Graded phase-3 cells | Share | +| --- | --- | --- | --- | +| 3.6 | 159 | 185 | 85.9% | +| 3.8 | 95 | 108 | 88.0% | + +Restricted to the four length-gated families the shares are higher still: 152 of 152 +Qwen3.6 and 93 of 95 Qwen3.8 graded length-gated transcripts contain a `wc -w` call. + +Typical call, taken verbatim from `p3_business_qwen36-nothink-offspec_v1`: `wc -w /workspace/memo.md`. + +So the benchmark set a hard threshold on a quantity it did not define, the models +measured that quantity with the standard tool, and the grader measured it with a +different tool. + +### Proof that the two counters disagree by more than the margin + +Mechanism, measured on `p3_doc_qwen36-think-offspec_v1`'s `brief.md` (ceiling 700): + +``` +grader regex \b\w+\b : 707 -> FAIL +shell wc -w : 698 -> under the cap +``` + +The 9-word gap decomposes exactly: + +- 25 `wc -w` tokens contain no word character at all — pure markdown punctuation that + `wc` counts as words and the regex does not: `—` ×7, `---` ×6, `##` ×6, `-` ×5, `#` ×1. +- 34 `wc -w` tokens are split into more than one word by `\b\w+\b`: `Follow-On` → `Follow`, + `On`; `well-funded` → `well`, `funded`; `product-market` → `product`, `market`; + `($4.2M` → `4`, `2M`. That adds 34 words. +- 698 − 25 + 34 = 707. Exact. + +Across all 364 phase-3 deliverables extracted from `workspace_final.tar.gz` for the +graded cells, the two counters disagree in **both** directions: the regex returns the +larger count for 259 of 364, `wc -w` returns the larger count for 78 of 364, and they tie +for 27 of 364. The divergence ranges from −20.95% to +8.70% of the `wc -w` count, median ++1.95%. Documents heavy in markdown tables swing the other way, because every `|` is a +word to `wc`. + +**Neither counter is ground truth.** `wc -w` counting `|`, `##` and `---` as words is not +more correct than a regex splitting `product-market` into two words. Both are defensible; +they are simply different, and they differ by more than the margin at issue. + +### Blast radius + +A cell is a **length-attributable FAIL** if the grader returned FAIL, at least one +deliverable exceeded its ceiling by the grader's regex, and every non-length gate passed +(facts captured, bias recall and stance, required/prohibited content — so that removing +the length gate alone would flip the verdict). + +| | 3.6 | 3.8 | total | +| --- | --- | --- | --- | +| Length-attributable FAILs | 24 | 47 | 71 | +| …of which under **every** ceiling by `wc -w` | 23 | 39 | **62** | +| …of which still over by `wc -w` (genuinely long) | 1 | 8 | 9 | + +The 3.8 column includes the Q8_0 control's two length FAILs (`p3_business`, `p3_doc`), +both of which sit in the still-over row. By family, the 71: `p3_doc` 26, `p3_writing` 24, +`p3_business` 21. The grader's stored word count reproduced exactly from the archived +deliverable in 71 of 71 cases, so the recount is measuring the same bytes the grader +measured. The full cell-by-cell table with both counts is in Appendix A. + +Family pass rates under three treatments of the length gate (graded cells only): + +| Family | Model | n | Raw (regex gate) | If `wc -w` were the gate | Length gate removed | +| --- | --- | --- | --- | --- | --- | +| `p3_business` | 3.6 | 38 | 35 (92.1%) | 37 (97.4%) | 38 (100.0%) | +| `p3_business` | 3.8 | 23 | 4 (17.4%) | 17 (73.9%) | 22 (95.7%) | +| `p3_doc` | 3.6 | 38 | 28 (73.7%) | 38 (100.0%) | 38 (100.0%) | +| `p3_doc` | 3.8 | 25 | 9 (36.0%) | 23 (92.0%) | 25 (100.0%) | +| `p3_writing` | 3.6 | 38 | 11 (28.9%) | 22 (57.9%) | 22 (57.9%) | +| `p3_writing` | 3.8 | 21 | 8 (38.1%) | 20 (95.2%) | 21 (100.0%) | +| **all three** | **3.6** | **114** | **74 (64.9%)** | **97 (85.1%)** | **98 (86.0%)** | +| **all three** | **3.8** | **69** | **21 (30.4%)** | **60 (87.0%)** | **68 (98.6%)** | + +(The one `p3_business` 3.8 cell that does not pass even with the length gate removed is a +`MISSING_OUTPUT` verdict — no memo in the archived workspace — not a gate effect. The 3.8 +rows include the Q8_0 control's graded phase-3 cells.) + +This is the reason D1 matters. Under the shipped grader these three families read as a +34-point win for 3.6 (64.9% vs 30.4%). Swap in the other equally-defensible counter and +they read as a 2-point win for 3.8 (85.1% vs 87.0%). The gate is deciding the headline. + +`p3_pm` is unaffected: its ceiling is 700 and the largest regex word count across all 64 +graded `p3_pm` cells is 576, so the length gate never binds there. + +### Correction applied, and its limits + +The overlay marks the 62 cells that are over by the grader's regex and under by `wc -w`, +records both counts, both file hashes, and the `wc -w` locale used +(`LC_ALL=C.UTF-8`, matching the sandbox image), and records +`gate_invalidated: true` together with `verdict_without_length_gate: "PASS"` — +deliberately **not** a `corrected_verdict`, which the overlay reserves for the D2/D3 +verdict corrections — so that downstream aggregation must name its treatment of these +cells rather than silently counting them as passes. + +**The overlay's `verdict_without_length_gate: "PASS"` is not a claim that these +deliverables complied with the word limit.** It is a claim that the FAIL is not +defensible. The honest statement is: + +> For these 62 cells the verdict is *counter-dependent*. Two reasonable word counters +> disagree across the threshold. MMBT's length gate is not a valid discriminator at these +> margins and should not carry weight in any headline comparison until the benchmark +> defines its tokenizer and states it in the brief. + +Anyone aggregating this overlay should read the 62 as "not a valid FAIL", not as "a +verified PASS", and should prefer the "length gate removed" column above when comparing +models. The 9 cells that are over by both counters remain FAIL and are not touched. + +Limits, stated plainly: + +- This does not fix the benchmark. It documents that one gate was unmeasurable as + specified. The durable fix is upstream: define the counter in the brief, or replace the + hard threshold with a soft penalty, and re-run. +- `wc -w` behaviour is locale-dependent; the overlay pins `C.UTF-8` and verified the + shell result against a pure-Python `str.split()` recount for every deliverable. +- The 62 correction does not disturb non-length failures. `p3_writing` for 3.6 still fails + 16 of 38 cells on required/prohibited content, which is why its "length gate removed" + column stops at 57.9% rather than reaching 100%. + +--- + +## D2 — `p3_pm` keyword literalism (and the fix this repo already had) + +### The grader source + +`tooling/graders/phase3_project_mgmt_grade.py` + +``` +17: def normalize(s: str) -> str: +18: return re.sub(r"\s+", " ", s.lower()) +... +31: RISK_KEYWORDS = { +... +34: "R3": ["legal unresponsive", "legal silent", "legal hasn't", "blocking on legal"], +... +70: hit = next((kw for kw in keywords if normalize(kw) in norm), None) +... +97: and risk_recall >= 3 +``` + +`normalize` lowercases and collapses whitespace. It does not expand or contract +apostrophes. So R3 recognises `legal hasn't` and does not recognise `legal has not`. + +### The model output it misses + +`p3_pm_qwen36-nothink-card_v2`, `status_report.md` line 22 — verbatim: + +``` +| **High** | Private-beta contracts unsigned | Legal has not responded to draft [wk6] | +``` + +`p3_pm_qwen36-nothink-card_v8`, `status_report.md` line 22 — byte-identical to v2 +(both lines hash to `d34282e4e207623e234fb9731c144e41`): + +``` +| **High** | Private-beta contracts unsigned | Legal has not responded to draft [wk6] | +``` + +`p3_pm_qwen36-nothink-card_v4`, `status_report.md` line 22: + +``` +| **High** | Private-beta contracts unsigned | Legal hasn't responded to draft [wk6] | +``` + +Same table, same row, same risk, same evidence citation. The only difference is the +contraction. + +### The natural experiment + +These three cells are the same model at the same sampler point +(3.6, no-think, `T1/p0.95/pp0`, `mmbt-q36-card`), and their graded scores are identical on +every gate except R3: + +| Cell | line-22 phrasing | R3 | risk_recall | workstream | decision | milestone | sections | word_count (cap 700) | **verdict** | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| `p3_pm_qwen36-nothink-card_v2` | `Legal has not responded` | ✗ | **2/6** | 6/6 | 4/4 | 5/5 | 4/4 | 379 | **FAIL** | +| `p3_pm_qwen36-nothink-card_v8` | `Legal has not responded` | ✗ | **2/6** | 6/6 | 4/4 | 5/5 | 4/4 | 388 | **FAIL** | +| `p3_pm_qwen36-nothink-card_v4` | `Legal hasn't responded` | ✓ (`"legal hasn't"`) | **3/6** | 6/6 | 4/4 | 5/5 | 4/4 | 388 | **PASS** | + +The threshold is `risk_recall >= 3` (line 97). The apostrophe is the entire difference +between PASS and FAIL. This is not a judgement call about whether the model identified the +risk — all three identified it, in the same words, in the same row of the same table. + +The whole family turns on this gate: raw `risk_recall` is exactly `2/6` for all 39 FAIL +cells and `>= 3/6` for all 25 PASS cells, across 64 graded cells. No `p3_pm` cell fails on +any other gate. + +### The fix was already in this repository, unapplied + +`tooling/correct_gemma4_project_mgmt_grades.py` — present, unit-tested, and byte-identical +in all seven checkouts — already encodes exactly this correction: + +``` +22: "R3": { +23: "description": "legal/private-beta contract delay uses an equivalent non-contracted phrase", +24: "patterns": [ +25: r"\blegal\b.{0,200}\b(?:has\s+not|not\s+yet\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\b", +26: ], +27: }, +``` + +`tooling/test_correct_gemma4_project_mgmt_grades.py` covers it — +`test_semantic_equivalents_correct_only_the_known_lexical_misses` asserts R2, R3, +`D3_mobile` and `D4_option_b`; the suite passes (4 passed). +`tooling/deployments/gemma4-31b-q4-tower2/README.md:170-178` documents the defect in +prose, naming the exact miss: + +> "The raw project-management grader searches for a few contiguous phrases and misses +> semantically exact wording such as "Maevia … push back," "Legal has not yet responded," +> hyphenated "web-responsive," and "private beta (3-5 customers)."" + +It never ran on the Qwen campaigns for one reason — line 133 enumerates cells by a +hardcoded name pattern: + +``` +133: name = f"p3_pm_gemma4-31b-q4_v{replicate}" +``` + +So a correction this project had already written, tested, documented and shipped for the +Gemma4 campaign silently did not apply to any cell whose name did not start with +`p3_pm_gemma4-31b-q4_`. This is disclosed here rather than buried because it is the +single most damaging thing a reviewer could find unaided, and because it points at a +process defect worse than the grader bug: a correction whose scope is a filename prefix. + +### Blast radius + +Raw R3 matched in only 24 of 64 graded `p3_pm` cells. The overlay's R3 rule fires on 39 of +64 (the 40 raw non-matches minus one the regex also does not reach). + +| | 3.6 | 3.8 | total | +| --- | --- | --- | --- | +| Graded `p3_pm` cells | 38 | 26 | 64 | +| Raw PASS | 4 (10.5%) | 21 (80.8%) | 25 | +| R3 overlay fires | 33 | 6 | 39 | +| PASS after R3 correction | 35 (92.1%) | 26 (100.0%) | 61 | +| Verdict changes | 31 | 5 | **36** | + +(The 3.8 column includes the single graded Q8_0 `p3_pm` cell, whose raw FAIL is also an +R3 literalism and is corrected to PASS.) Uncorrected, `p3_pm` reads as a 70-point rout +for 3.8 (10.5% vs 80.8%). Corrected, it is 92.1% vs 100.0%. The uncorrected result was +measuring which model contracts its verbs. + +### Correction applied, and its limits + +The overlay reuses the repository's existing module rather than reimplementing it — it +imports `read_archived_report` and `apply_correction` from +`tooling/correct_gemma4_project_mgmt_grades.py` (sha256 +`86eeec68b6c7…`) and changes exactly one thing: cell selection is driven from the frozen +dataset instead of `build()`'s hardcoded `p3_pm_gemma4-31b-q4_v{n}` names. Report text is +read from `workspace_final.tar.gz:./status_report.md`, never from a live workspace. + +**Only the R3 rule is activated.** The upstream module also ships R2, `D3_mobile` and +`D4_option_b`. They are deliberately left inactive, and that decision must be disclosed +because it is visible in the overlay records (`"active_rules": ["R3"]`): + +- Only R3 has a decisive natural experiment. R2's keyword bundle + (`["maevia push", "maevia push-back", "maevia pushback", "expectations gap"]`) raw-matched + in just 2 of 64 cells, and the upstream replacement pattern + (`\bmaevia\b.{0,240}\b(?:push(?:ed)?[ -]?back|fallout|promis(?:e|ed)\s+ga|private[ -]?beta)\b`) + fires on 64 of 64. A rule that fires on 100% of a family is a rewrite of the rubric, not + a correction of a false negative, and it deserves its own scrutiny rather than a free + ride on R3's evidence. +- Activating all four upstream rules takes `p3_pm` to **64 of 64 PASS** for both models — + 3 more verdict changes than R3 alone. That is a ceiling, and a ceiling measures nothing. + +Limits: + +- Even with R3 alone, corrected `p3_pm` is 35/38 vs 26/26. That is close enough to + saturation that `p3_pm` should not carry weight as a discriminator in this comparison in + either direction. Report it, do not lean on it. +- The correction rescues a keyword match; it does not verify that the model's risk + description is *good*. The grader's hand-rating fields + (`structure_quality_1to5`, `fabrication_count`, `owner_accuracy_0to6`) are `null` in every + cell, so no qualitative check backs any `p3_pm` verdict, corrected or raw. +- R1 and R6 remain unmatched in most cells and are not corrected. No claim is made that + the corrected `risk_recall` is the true recall — only that the R3 component was a + verified false negative. + +--- + +## D3 — `p2_triage` brief / ground-truth contradiction + +### The contradiction + +`tooling/tasks/task_triage.md:27`, in the closed urgency vocabulary the brief requires the +model to use: + +``` +- `n/a` — for noise/spam where urgency doesn't apply +``` + +`tooling/graders/ground_truth/phase2_triage.json`, the three tickets the same ground truth +categorises as `spam-or-noise`: + +``` +23: "004": {"category": "spam-or-noise", "urgency": "low", "notes": "fictitious 'free Pixar collectables' premise; no such program"}, +28: "009": {"category": "spam-or-noise", "urgency": "low", "notes": "obvious phishing email impersonating account-security"}, +40: "021": {"category": "spam-or-noise", "urgency": "low", "notes": "affiliate-link blog-hosting spam"}, +``` + +The ground truth contains **zero** `n/a` urgency labels across all 30 tickets +(distribution: `normal` 17, `urgent` 7, `low` 6). The value the brief defines for +noise/spam is never the answer. + +The grader compares by exact string equality and applies a hard threshold: + +`tooling/graders/phase2_triage_grade.py` + +``` +53: if a_urg == gt_urg: +... +73: urg_threshold = 0.70 # 70% urgency accuracy (more subjective) +... +80: and (urg_accuracy or 0) >= urg_threshold +``` + +### Proof that it is a defect, not a judgement call + +Of the 64 graded `p2_triage` cells, **64 of 64 answered `n/a` for all three spam +tickets** — every replicate of both models, both thinking modes, every sampler point, and +the Q8_0 control cell. Zero cells answered `low`. Every cell therefore lost exactly +3/30 = 0.100 of urgency accuracy for following the brief, against a 0.700 threshold. + +A defect that penalises 100% of cells identically for complying with the written +instructions is not a discrimination signal; it is a constant subtracted from every score +before it is compared to a threshold. + +### A second contradiction in the same family, in the same direction + +A reviewer checking the error lists will find this, so it is stated here. Ticket 029 is an +extortion email (`"Pay $50,000 in BTC or I post them on twitter"`). The brief, +`tooling/tasks/task_triage.md:66`: + +``` +- **Be skeptical of obvious noise.** Phishing emails impersonating "Account Security + Team," extortion threats, fictitious-premise requests ("free collectables"), + affiliate-link spam — these are `spam-or-noise`, not real categories. +``` + +(reflowed for width; the source is one line). The ground truth: +`"029": {"category": "security-incident", "urgency": "urgent", ...}`. + +All 64 of 64 cells classified 029 as `spam-or-noise` — exactly as instructed — and all 64 +called its urgency `n/a`. It is the only universal category error in the family. So a +second brief-versus-ground-truth conflict costs every cell a further 1/30 = 0.033 of +category accuracy and 1/30 = 0.033 of urgency accuracy. **The overlay does not credit +029**, because unlike the spam trio there is a genuine substantive reading in which an +extortion threat is a security incident; the brief and the rubric simply disagree about +it. It is disclosed rather than corrected. + +### Blast radius + +| | 3.6 | 3.8 | total | +| --- | --- | --- | --- | +| Graded `p2_triage` cells | 38 | 26 | 64 | +| Answered `n/a` on all three spam tickets | 38 | 26 | **64** | +| Raw PASS | 17 (44.7%) | 23 (88.5%) | 40 | +| PASS after crediting `n/a` | 38 (100.0%) | 26 (100.0%) | 64 | +| Verdict changes | 21 | 3 | **24** | + +(The 3.8 column includes the single graded Q8_0 triage cell, a raw PASS at urgency 0.733.) +The apparent 44-point gap in 3.8's favour is an artefact of a constant 0.100 penalty +interacting with a hard threshold: 3.6's failing cells sit at 0.600–0.667 with its passing +tail at 0.733–0.767, while 3.8's cells sit at 0.700–0.800 apart from three at 0.667 — so +the same uniform penalty pushes most of one distribution below the line and leaves the +other above it. + +The family has no discriminating power on its other two axes either, which is easy to +verify and worth stating: across all 64 graded cells, `category_accuracy` is **0.867 in +every single cell** and `duplicate_recall` is **1.000 in every single cell**. Zero +variance on both, across both models. Urgency accuracy is the only axis that varies, and +it is the axis the defect corrupts. + +### Correction applied, and its limits + +The overlay credits the model's `n/a` as correct for tickets 004, 009 and 021 only — +the three the ground truth itself categorises as `spam-or-noise`, which is precisely the +set the brief's line 27 rule names — recomputes urgency accuracy (+0.100), and re-evaluates +the grader's own thresholds. Nothing else in the grade is touched; the grader's arithmetic +was reproduced from the stored error lists before any change was applied. + +Limits: + +- **The corrected family is 64 of 64 PASS.** It discriminates nothing. Combined with the + zero variance on category and duplicate recall, the honest reading is that + `p2_triage` produced no usable signal in this comparison, in either direction. It should + be reported as such and excluded from any aggregate ship-rate that is meant to compare + the two models. +- Crediting `n/a` is not a claim that `n/a` is the better label. It is a claim that a + model cannot be scored down for obeying an instruction the benchmark gave it. The + durable fix is upstream and is a one-line choice: either add `n/a` to the ground truth + for the three spam tickets, or delete line 27 from the brief. Until then this family + measures nothing. +- Ticket 029 is left uncorrected, as described above, which means the residual urgency + scores still contain a uniform 0.033 penalty of the same kind. + +--- + +## What these defects mean for previously published MMBT results + +The graders are byte-identical in `bench-gemma4-31b-q4` and +`bench-deepseek-v4-flash-0731`, and both campaigns ran the affected families +(`p2_triage`, `p3_business`, `p3_doc`, `p3_pm`, `p3_writing`). The defects are therefore +not new; they are newly *found*. Measured directly on those campaigns' run evidence: + +**Gemma4-31b-q4 (10 replicates per family)** + +| Family | Defect | Finding | +| --- | --- | --- | +| `p2_triage` | D3 | 10 of 10 cells answered `n/a` on all three spam tickets. Crediting them flips **2 of 10** verdicts. Published `p2_triage` numbers for this campaign are wrong by 2/10. | +| `p3_pm` | D2 | 0 of 10 raw PASS. R3 correction alone → 7 of 10; the full published overlay → 10 of 10. This campaign *did* ship the overlay, so its corrected total was already reported — but the raw 0/10 figure, if quoted anywhere without the overlay, is a contraction artefact. | +| `p3_business`, `p3_doc`, `p3_writing` | D1 | 0 length-attributable FAILs. **Unaffected.** | + +**DeepSeek-v4-flash-0731 (3 replicates per family)** + +| Family | Defect | Finding | +| --- | --- | --- | +| `p2_triage` | D3 | 3 of 3 cells answered `n/a` on all three spam tickets and lost 0.100 each; all three sat far enough above 0.700 that **0 verdicts change**. Scores are wrong; verdicts are not. | +| `p3_pm` | D2 | 3 of 3 raw PASS; no change under R3. The correction was never applied to this campaign, but nothing turned on it. | +| `p3_business` | D1 | 1 length-attributable FAIL, and it is over the ceiling by `wc -w` too. **Verdict stands.** | +| `p3_doc`, `p3_writing` | D1 | 0 length-attributable FAILs. **Unaffected.** | + +Conclusions for prior work, in order of how much they matter: + +1. **Every published MMBT `p2_triage` urgency accuracy, for every model ever run, is + understated by 0.100** (and by a further 0.033 if ticket 029 is counted). Every cell in + every campaign examined answered as the brief instructed. Any cross-model `p2_triage` + comparison published to date is a comparison of how far each model's remaining errors + happened to sit from a threshold that had a constant subtracted from it. Concretely, + 2 of 10 Gemma4 verdicts change. +2. **Published raw `p3_pm` results understate every model that writes "has not" instead of + "hasn't."** The Gemma4 campaign shipped an overlay so its corrected number is sound, but + its raw 0/10 should never be quoted standalone, and DeepSeek's `p3_pm` was never + overlaid at all (it happens not to matter, 3/3 raw PASS). Any other campaign graded with + this file and no overlay is suspect. +3. **D1 did not bite in the prior campaigns measured here** — 1 length-attributable FAIL + across both, and it fails under both counters. This is a length-of-output property of + those models, not evidence that the gate is sound. Any campaign whose deliverables land + within roughly 10% of a ceiling is exposed, and the Qwen comparison in this PR shows what + that looks like: 62 counter-dependent verdicts out of 183 graded cells in the three + affected families. +4. **The families outside these five are not implicated by this document.** `p1_bugfix`, + `p1_refactor`, `p1_testwrite`, `p2_ci`, `p2_extract`, `p2_hallucination`, + `p3_market`, and the `75pr` / `board_pres` / `invest_memo` suites use different graders + that contain none of the three defects. That is a statement about these three defects + only, not a clean bill of health for those graders. + +Recommended repository actions, none of which are in this PR's scope: + +- Define the word counter in every length-gated brief, or drop the hard threshold. +- Resolve the `task_triage.md` / `phase2_triage.json` contradiction on tickets 004, 009, + 021 and 029 in one direction, and re-grade. +- Replace the `p3_pm` literal keyword bundles with the semantic patterns the repository + already wrote and tested, in the grader itself rather than in an overlay. +- Never scope a correction script by a hardcoded cell-name prefix again. + +--- + +## Appendix A — the 71 length-attributable FAIL cells (D1) + +Format: `cap / grader-regex count / wc -w count`, listing only the deliverables that +exceed their cap under the grader's regex. A cell is marked **under every cap** when no +deliverable exceeds its cap under `wc -w`; those are the 62 the overlay corrects. + +| cell | model | deliverable cap / regex / `wc -w` | `wc -w` verdict | +| --- | --- | --- | --- | +| `p3_business_qwen36-nothink-offspec_v4` | 3.6 | memo 700/719/699 | **under every cap** | +| `p3_business_qwen36-nothink-offspec_v8` | 3.6 | memo 700/731/725 | over | +| `p3_business_qwen36-think-offspec_v7` | 3.6 | memo 700/703/700 | **under every cap** | +| `p3_doc_qwen36-nothink-offspec_v7` | 3.6 | brief 700/713/696 | **under every cap** | +| `p3_doc_qwen36-think-offspec_v1` | 3.6 | brief 700/707/698 | **under every cap** | +| `p3_doc_qwen36-think-offspec_v2` | 3.6 | brief 700/707/698 | **under every cap** | +| `p3_doc_qwen36-think-offspec_v3` | 3.6 | brief 700/707/698 | **under every cap** | +| `p3_doc_qwen36-think-offspec_v4` | 3.6 | brief 700/707/698 | **under every cap** | +| `p3_doc_qwen36-think-offspec_v5` | 3.6 | brief 700/707/698 | **under every cap** | +| `p3_doc_qwen36-think-offspec_v6` | 3.6 | brief 700/707/700 | **under every cap** | +| `p3_doc_qwen36-think-offspec_v7` | 3.6 | brief 700/707/698 | **under every cap** | +| `p3_doc_qwen36-think-offspec_v8` | 3.6 | brief 700/707/698 | **under every cap** | +| `p3_doc_qwen36-think-offspec_v9` | 3.6 | brief 700/709/698 | **under every cap** | +| `p3_writing_qwen36-nothink-card_v2` | 3.6 | ceo_brief 250/253/242; customer_email 350/358/347 | **under every cap** | +| `p3_writing_qwen36-nothink-card_v3` | 3.6 | ceo_brief 250/254/243 | **under every cap** | +| `p3_writing_qwen36-nothink-card_v7` | 3.6 | ceo_brief 250/257/245 | **under every cap** | +| `p3_writing_qwen36-nothink-card_v9` | 3.6 | ceo_brief 250/255/248 | **under every cap** | +| `p3_writing_qwen36-think-card_v2` | 3.6 | customer_email 350/358/346 | **under every cap** | +| `p3_writing_qwen36-think-card_v3` | 3.6 | customer_email 350/358/346 | **under every cap** | +| `p3_writing_qwen36-think-card_v6` | 3.6 | customer_email 350/358/346 | **under every cap** | +| `p3_writing_qwen36-think-offspec_v2` | 3.6 | ceo_brief 250/252/242 | **under every cap** | +| `p3_writing_qwen36-think-offspec_v5` | 3.6 | ceo_brief 250/252/242 | **under every cap** | +| `p3_writing_qwen36-think-offspec_v8` | 3.6 | ceo_brief 250/252/242 | **under every cap** | +| `p3_writing_qwen36-think-offspec_v9` | 3.6 | ceo_brief 250/252/242 | **under every cap** | +| `p3_business_qwen38-27b-udq4xl-nothink_v1` | 3.8 | memo 700/817/753 | over | +| `p3_business_qwen38-27b-udq4xl-think-xhigh_v1` | 3.8 | memo 700/723/694 | **under every cap** | +| `p3_business_qwen38-27b-udq4xl-think-xhigh_v2` | 3.8 | memo 700/742/694 | **under every cap** | +| `p3_business_qwen38-27b-udq4xl-think-xhigh_v3` | 3.8 | memo 700/749/697 | **under every cap** | +| `p3_business_qwen38-nothink-card_v1` | 3.8 | memo 700/860/832 | over | +| `p3_business_qwen38-nothink-card_v2` | 3.8 | memo 700/775/713 | over | +| `p3_business_qwen38-nothink-offspec_v1` | 3.8 | memo 700/738/699 | **under every cap** | +| `p3_business_qwen38-nothink-offspec_v2` | 3.8 | memo 700/742/707 | over | +| `p3_business_qwen38-nothink-offspec_v4` | 3.8 | memo 700/723/697 | **under every cap** | +| `p3_business_qwen38-nothink-offspec_v5` | 3.8 | memo 700/754/698 | **under every cap** | +| `p3_business_qwen38-nothink-offspec_v6` | 3.8 | memo 700/732/699 | **under every cap** | +| `p3_business_qwen38-nothink-offspec_v7` | 3.8 | memo 700/731/695 | **under every cap** | +| `p3_business_qwen38-think-low-card_v1` | 3.8 | memo 700/725/691 | **under every cap** | +| `p3_business_qwen38-think-medium-card_v1` | 3.8 | memo 700/744/698 | **under every cap** | +| `p3_business_qwen38-think-medium-offspec_v2` | 3.8 | memo 700/741/696 | **under every cap** | +| `p3_business_qwen38-think-xhigh-card_v1` | 3.8 | memo 700/725/699 | **under every cap** | +| `p3_business_qwen38-think-xhigh-offspec_v1` | 3.8 | memo 700/724/690 | **under every cap** | +| `p3_business_qwen38q8-nothink-matched_v1` | 3.8 | memo 700/748/714 | over | +| `p3_doc_qwen38-27b-udq4xl-nothink_v1` | 3.8 | brief 700/702/679 | **under every cap** | +| `p3_doc_qwen38-27b-udq4xl-nothink_v2` | 3.8 | brief 700/727/700 | **under every cap** | +| `p3_doc_qwen38-27b-udq4xl-think-xhigh_v1` | 3.8 | brief 700/722/682 | **under every cap** | +| `p3_doc_qwen38-27b-udq4xl-think-xhigh_v2` | 3.8 | brief 700/719/689 | **under every cap** | +| `p3_doc_qwen38-27b-udq4xl-think-xhigh_v3` | 3.8 | brief 700/725/694 | **under every cap** | +| `p3_doc_qwen38-nothink-card_v1` | 3.8 | brief 700/722/694 | **under every cap** | +| `p3_doc_qwen38-nothink-card_v2` | 3.8 | brief 700/722/694 | **under every cap** | +| `p3_doc_qwen38-nothink-card_v3` | 3.8 | brief 700/718/688 | **under every cap** | +| `p3_doc_qwen38-nothink-card_v4` | 3.8 | brief 700/730/699 | **under every cap** | +| `p3_doc_qwen38-nothink-card_v5` | 3.8 | brief 700/733/700 | **under every cap** | +| `p3_doc_qwen38-think-medium-card_v1` | 3.8 | brief 700/701/692 | **under every cap** | +| `p3_doc_qwen38-think-medium-offspec_v1` | 3.8 | brief 700/708/696 | **under every cap** | +| `p3_doc_qwen38-think-medium-offspec_v2` | 3.8 | brief 700/720/713 | over | +| `p3_doc_qwen38-think-xhigh-card_v1` | 3.8 | brief 700/726/697 | **under every cap** | +| `p3_doc_qwen38-think-xhigh-offspec_v1` | 3.8 | brief 700/723/696 | **under every cap** | +| `p3_doc_qwen38q8-nothink-matched_v1` | 3.8 | brief 700/772/757 | over | +| `p3_writing_qwen38-27b-udq4xl-nothink_v1` | 3.8 | ceo_brief 250/255/246; customer_email 350/358/348 | **under every cap** | +| `p3_writing_qwen38-27b-udq4xl-nothink_v3` | 3.8 | ceo_brief 250/308/301; customer_email 350/355/346 | over | +| `p3_writing_qwen38-27b-udq4xl-think-xhigh_v1` | 3.8 | ceo_brief 250/258/246; legal_summary 400/410/394 | **under every cap** | +| `p3_writing_qwen38-27b-udq4xl-think-xhigh_v2` | 3.8 | ceo_brief 250/258/249; customer_email 350/351/345 | **under every cap** | +| `p3_writing_qwen38-27b-udq4xl-think-xhigh_v3` | 3.8 | ceo_brief 250/260/246; legal_summary 400/401/391 | **under every cap** | +| `p3_writing_qwen38-nothink-card_v2` | 3.8 | ceo_brief 250/258/248; customer_email 350/358/348 | **under every cap** | +| `p3_writing_qwen38-nothink-card_v5` | 3.8 | ceo_brief 250/256/249; customer_email 350/360/350 | **under every cap** | +| `p3_writing_qwen38-think-low-card_v1` | 3.8 | ceo_brief 250/252/249; customer_email 350/359/348 | **under every cap** | +| `p3_writing_qwen38-think-medium-card_v1` | 3.8 | ceo_brief 250/264/246 | **under every cap** | +| `p3_writing_qwen38-think-medium-offspec_v1` | 3.8 | ceo_brief 250/260/250 | **under every cap** | +| `p3_writing_qwen38-think-medium-offspec_v2` | 3.8 | ceo_brief 250/259/249 | **under every cap** | +| `p3_writing_qwen38-think-xhigh-card_v1` | 3.8 | ceo_brief 250/251/243; customer_email 350/351/347 | **under every cap** | +| `p3_writing_qwen38-think-xhigh-offspec_v1` | 3.8 | ceo_brief 250/257/249 | **under every cap** | + +71 rows: 62 under every cap (corrected), 9 over (verdict stands). The two Q8_0 control +cells (`p3_business_qwen38q8-nothink-matched_v1`, `p3_doc_qwen38q8-nothink-matched_v1`) +are in the 3.8 rows and both stand as over. + +--- + +## Appendix B — reproducing these claims + +Every claim above reads only immutable artefacts: `grade.json`, `workspace_final.tar.gz`, +`transcript.jsonl`, the graders, the briefs and the ground truth. Nothing was rewritten. + +**D2's natural experiment**, the single fastest check — three commands, no scripts: + +```sh +cd mmbt-q36-card +for v in v2 v4 v8; do + tar -xzOf logs/p3_pm_qwen36-nothink-card_$v/workspace_final.tar.gz ./status_report.md \ + | sed -n 22p +done +for v in v2 v4 v8; do + python3 -c "import json;g=json.load(open('logs/p3_pm_qwen36-nothink-card_$v/grade.json'));\ +print('$v', g['verdict'], g['scores']['risk_recall'], g['details']['risks']['R3'])" +done +``` + +Expect: v2 and v8 print `Legal has not responded`, `FAIL`, `2/6`, R3 unmatched; v4 prints +`Legal hasn't responded`, `PASS`, `3/6`, R3 matched on `legal hasn't`. Every other score +in the three grades is identical. + +**D1's counter divergence**, on any length-attributable FAIL from Appendix A: + +```sh +tar -xzOf mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v1/workspace_final.tar.gz \ + ./brief.md > /tmp/brief.md +python3 -c "import re;t=open('/tmp/brief.md').read();print('regex',len(re.findall(r'\b\w+\b',t)))" +LC_ALL=C.UTF-8 wc -w /tmp/brief.md +``` + +Expect 707 and 698 against a ceiling of 700. + +**D3's contradiction**, one command each: + +```sh +sed -n 27p mmbt-q38-card/tooling/tasks/task_triage.md +python3 -c "import json;d=json.load(open('mmbt-q38-card/tooling/graders/ground_truth/phase2_triage.json'));\ +print({k:v['urgency'] for k,v in d['tickets'].items() if v['category']=='spam-or-noise'})" +``` + +Expect the brief to define `n/a` for noise/spam and the ground truth to answer +`{'004': 'low', '009': 'low', '021': 'low'}`. + +The per-cell overlay records under `overlay//logs//grade.corrected.json` carry +the raw `grade.json` sha256, the deliverable sha256, the frozen-dataset verdict, the +defect evidence and both word counts, so any single correction can be checked in isolation +without re-running the pipeline. diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/manifest.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/manifest.json new file mode 100644 index 00000000..f4e582e2 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/manifest.json @@ -0,0 +1,289 @@ +{ + "d2_mechanism": { + "matched_spans": [ + { + "cell": "p3_pm_qwen36-nothink-card_v1", + "item": "R3", + "matched_span": "legal contracts unsigned | private-beta contract", + "repo": "mmbt-q36-card" + }, + { + "cell": "p3_pm_qwen36-nothink-card_v10", + "item": "R3", + "matched_span": "legal has not", + "repo": "mmbt-q36-card" + }, + { + "cell": "p3_pm_qwen36-nothink-card_v2", + "item": "R3", + "matched_span": "legal has not responded to draft [wk6] | | **high** | maevia expectation gap | promised ga at conference; pushback expected [wk6] | | **medium** | iam effort slips | would delay", + "repo": "mmbt-q36-card" + }, + { + "cell": "p3_pm_qwen36-nothink-card_v3", + "item": "R3", + "matched_span": "legal contracts unsigned | private-beta contract", + "repo": "mmbt-q36-card" + }, + { + "cell": "p3_pm_qwen36-nothink-card_v5", + "item": "R3", + "matched_span": "legal contracts | legal (tbd) | draft sent; no response (w6) | yes | --- ## risks | severity | risk | description | |---|---|---| | **high** | legal contracts not finalized | contract", + "repo": "mmbt-q36-card" + }, + { + "cell": "p3_pm_qwen36-nothink-card_v6", + "item": "R3", + "matched_span": "legal; no response yet [wk6] | | **high** | maevia expectation gap | promised ga at conference; will push back on private beta [wk6] | | **medium** | iam delay", + "repo": "mmbt-q36-card" + }, + { + "cell": "p3_pm_qwen36-nothink-card_v7", + "item": "R3", + "matched_span": "legal tbd) | draft sent, no response (w6) | yes | ## risks (by severity) | severity | risk | description | |---|---|---| | high | legal delay", + "repo": "mmbt-q36-card" + }, + { + "cell": "p3_pm_qwen36-nothink-card_v8", + "item": "R3", + "matched_span": "legal has not responded to draft [wk6] | | **high** | maevia expectation gap | promised ga sdk at conference; pushback expected [wk6] | | **medium** | iam 4-week effort | delay", + "repo": "mmbt-q36-card" + }, + { + "cell": "p3_pm_qwen36-think-card_v10", + "item": "R3", + "matched_span": "legal contracts pending [w6] | yes | --- ## risks (ordered by severity) | severity | risk | description | |---|---|---| | **high** | legal contract delay | private-beta contracts not finalized; legal has not", + "repo": "mmbt-q36-card" + }, + { + "cell": "p3_pm_qwen36-think-card_v2", + "item": "R3", + "matched_span": "legal delay on private-beta contracts | legal has not", + "repo": "mmbt-q36-card" + }, + { + "cell": "p3_pm_qwen36-think-card_v3", + "item": "R3", + "matched_span": "legal contract delay | private-beta contracts not yet finalized; legal has not", + "repo": "mmbt-q36-card" + }, + { + "cell": "p3_pm_qwen36-think-card_v4", + "item": "R3", + "matched_span": "legal delay on private-beta contracts | legal has not", + "repo": "mmbt-q36-card" + }, + { + "cell": "p3_pm_qwen36-think-card_v6", + "item": "R3", + "matched_span": "legal delay on private-beta contracts | legal has not responded to draft. no contracts = no sdk in v1 (w5, w6). | | 🔴 high | no timeline slack | access-control discovery consumed all buffer. any further delay", + "repo": "mmbt-q36-card" + }, + { + "cell": "p3_pm_qwen36-think-card_v7", + "item": "R3", + "matched_span": "legal has not yet responded to the private-beta contract", + "repo": "mmbt-q36-card" + }, + { + "cell": "p3_pm_qwen36-think-card_v8", + "item": "R3", + "matched_span": "legal contracts pending [w6] | yes — contracts needed before launch | --- ## risks (ordered by severity) | severity | risk | description | |---|---|---| | **high** | legal contract delay", + "repo": "mmbt-q36-card" + }, + { + "cell": "p3_pm_qwen36-think-card_v9", + "item": "R3", + "matched_span": "legal delay on private-beta contracts** — legal has not", + "repo": "mmbt-q36-card" + }, + { + "cell": "p3_pm_qwen38-think-low-card_v1", + "item": "R3", + "matched_span": "legal has not responded to the private-beta contract", + "repo": "mmbt-q38-card" + }, + { + "cell": "p3_pm_qwen38-think-medium-card_v1", + "item": "R3", + "matched_span": "legal has not engaged on the private-beta contract", + "repo": "mmbt-q38-card" + }, + { + "cell": "p3_pm_qwen38q8-nothink-matched_v1", + "item": "R3", + "matched_span": "legal has not responded to the access-control contract", + "repo": "mmbt-q38-q8" + }, + { + "cell": "p3_pm_qwen36-nothink-offspec_v1", + "item": "R3", + "matched_span": "legal) | sara | draft sent; no response (wk6) | **yes** (sdk portion) | ## risks | severity | risk | description | |---|---|---| | **high** | legal delay", + "repo": "mmbt-qwen36-compare" + }, + { + "cell": "p3_pm_qwen36-nothink-offspec_v2", + "item": "R3", + "matched_span": "legal contract delay | private-beta contracts drafted; legal has not", + "repo": "mmbt-qwen36-compare" + }, + { + "cell": "p3_pm_qwen36-nothink-offspec_v3", + "item": "R3", + "matched_span": "legal contract delay | private-beta contracts drafted; legal has not", + "repo": "mmbt-qwen36-compare" + }, + { + "cell": "p3_pm_qwen36-nothink-offspec_v5", + "item": "R3", + "matched_span": "legal) | sara | draft sent; no response (wk6) | **yes** (sdk portion) | ## risks | severity | risk | description | |---|---|---| | **high** | legal delay", + "repo": "mmbt-qwen36-compare" + }, + { + "cell": "p3_pm_qwen36-nothink-offspec_v6", + "item": "R3", + "matched_span": "legal contract delay | private-beta contracts drafted; legal has not", + "repo": "mmbt-qwen36-compare" + }, + { + "cell": "p3_pm_qwen36-nothink-offspec_v7", + "item": "R3", + "matched_span": "legal contract delay | private-beta contracts drafted but legal has not", + "repo": "mmbt-qwen36-compare" + }, + { + "cell": "p3_pm_qwen36-nothink-offspec_v8", + "item": "R3", + "matched_span": "legal contract delay | private-beta contracts drafted; legal has not", + "repo": "mmbt-qwen36-compare" + }, + { + "cell": "p3_pm_qwen36-nothink-offspec_v9", + "item": "R3", + "matched_span": "legal contract delay | private-beta contracts drafted; legal has not", + "repo": "mmbt-qwen36-compare" + }, + { + "cell": "p3_pm_qwen36-think-offspec_v1", + "item": "R3", + "matched_span": "legal delay on private-beta contracts | legal has not responded to contract", + "repo": "mmbt-qwen36-compare" + }, + { + "cell": "p3_pm_qwen36-think-offspec_v2", + "item": "R3", + "matched_span": "legal delay on private-beta contracts | legal has not responded to contract", + "repo": "mmbt-qwen36-compare" + }, + { + "cell": "p3_pm_qwen36-think-offspec_v3", + "item": "R3", + "matched_span": "legal delay on private-beta contracts | legal has not responded to contract", + "repo": "mmbt-qwen36-compare" + }, + { + "cell": "p3_pm_qwen36-think-offspec_v4", + "item": "R3", + "matched_span": "legal delay on private-beta contracts | legal has not responded to contract", + "repo": "mmbt-qwen36-compare" + }, + { + "cell": "p3_pm_qwen36-think-offspec_v5", + "item": "R3", + "matched_span": "legal delay on private-beta contracts | legal has not responded to contract", + "repo": "mmbt-qwen36-compare" + }, + { + "cell": "p3_pm_qwen36-think-offspec_v6", + "item": "R3", + "matched_span": "legal delay on private-beta contracts | legal has not", + "repo": "mmbt-qwen36-compare" + }, + { + "cell": "p3_pm_qwen36-think-offspec_v7", + "item": "R3", + "matched_span": "legal delay on private-beta contracts | legal has not responded to contract", + "repo": "mmbt-qwen36-compare" + }, + { + "cell": "p3_pm_qwen36-think-offspec_v8", + "item": "R3", + "matched_span": "legal has not responded to the private-beta contract", + "repo": "mmbt-qwen36-compare" + }, + { + "cell": "p3_pm_qwen36-think-offspec_v9", + "item": "R3", + "matched_span": "legal delay on private-beta contracts | legal has not responded to contract", + "repo": "mmbt-qwen36-compare" + }, + { + "cell": "p3_pm_qwen38-think-low-offspec_v1", + "item": "R3", + "matched_span": "legal dependency** — *med-high.* private-beta contract", + "repo": "mmbt-qwen38-eaaa8ca" + }, + { + "cell": "p3_pm_qwen38-think-medium-offspec_v1", + "item": "R3", + "matched_span": "legal not engaged.** private-beta contract", + "repo": "mmbt-qwen38-eaaa8ca" + }, + { + "cell": "p3_pm_qwen38-think-medium-offspec_v2", + "item": "R3", + "matched_span": "legal sign-off", + "repo": "mmbt-qwen38-eaaa8ca" + } + ], + "note": "the mechanism actually applied is upstream rule R3, an adjacency regex (the word legal within 200 characters of a completion term), which matches strictly more than contractions; matched_span is the exact normalised report text that satisfied the regex in each corrected cell", + "patterns": [ + "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + ] + }, + "frozen_dataset": "/home/michael/mmbt-frozen-dataset-v2.csv", + "frozen_dataset_sha256": "d2ed0beca5b68e9ca63788e452235f23a299af06639893f558bef19f784cf018", + "frozen_dataset_stamp": "2026-08-16T14:23:09Z", + "generated_at": "2026-08-16T14:35:19.120502+00:00", + "invariant_checked": 686, + "invariant_violations": [], + "note": "overlay_digest excludes generated_at and is stable across runs", + "overlay_digest": "e332cd2c78ad94fe264aed7d31e6c64f5273cd9598f889dc1a8fac539e971351", + "post_freeze_divergence": [], + "schema_version": "mmbt-grade-correction-overlay/2", + "skipped": { + "no_grade_json": 69, + "quarantined": 0, + "unreadable_grade_json": 0 + }, + "summary": { + "by_defect": { + "D1": { + "3.6": 23, + "3.8": 39, + "gate_invalidated_3.6": 23, + "gate_invalidated_3.8": 39, + "gate_invalidated_total": 62, + "total": 62 + }, + "D2": { + "3.6": 33, + "3.8": 6, + "total": 39, + "verdict_changed_3.6": 31, + "verdict_changed_3.8": 5, + "verdict_changed_total": 36 + }, + "D3": { + "3.6": 38, + "3.8": 26, + "total": 64, + "verdict_changed_3.6": 21, + "verdict_changed_3.8": 3, + "verdict_changed_total": 24 + } + }, + "cells_touched_frozen": 165, + "cells_touched_including_post_freeze": 165 + } +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v1/grade.corrected.json new file mode 100644 index 00000000..c42ec58c --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v1/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-nothink-card_v1", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.833, + "urgency_accuracy_original": 0.733, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "c8cf968396524766b15880bcd8be8e3577ae32e59313e35a83d7c13880d5fdef", + "identity": { + "arm": "qwen36-nothink-card", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v10/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v10/grade.corrected.json new file mode 100644 index 00000000..8cbed9a2 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v10/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-nothink-card_v10", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.833, + "urgency_accuracy_original": 0.733, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "c8cf968396524766b15880bcd8be8e3577ae32e59313e35a83d7c13880d5fdef", + "identity": { + "arm": "qwen36-nothink-card", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "10", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v2/grade.corrected.json new file mode 100644 index 00000000..8ccb4af1 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v2/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-nothink-card_v2", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.833, + "urgency_accuracy_original": 0.733, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "c8cf968396524766b15880bcd8be8e3577ae32e59313e35a83d7c13880d5fdef", + "identity": { + "arm": "qwen36-nothink-card", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v3/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v3/grade.corrected.json new file mode 100644 index 00000000..e0d2001a --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v3/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-nothink-card_v3", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.833, + "urgency_accuracy_original": 0.733, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "c8cf968396524766b15880bcd8be8e3577ae32e59313e35a83d7c13880d5fdef", + "identity": { + "arm": "qwen36-nothink-card", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "3", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v4/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v4/grade.corrected.json new file mode 100644 index 00000000..7b21098d --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v4/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-nothink-card_v4", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.833, + "urgency_accuracy_original": 0.733, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "c8cf968396524766b15880bcd8be8e3577ae32e59313e35a83d7c13880d5fdef", + "identity": { + "arm": "qwen36-nothink-card", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "4", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v5/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v5/grade.corrected.json new file mode 100644 index 00000000..56f51fc9 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v5/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-nothink-card_v5", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.767, + "urgency_accuracy_original": 0.667, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "f2bb191f8b19342e864076b2d8af57222633c519790a9fa80b385759174c04d0", + "identity": { + "arm": "qwen36-nothink-card", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "5", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v6/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v6/grade.corrected.json new file mode 100644 index 00000000..7f6f9b57 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v6/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-nothink-card_v6", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.833, + "urgency_accuracy_original": 0.733, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "c8cf968396524766b15880bcd8be8e3577ae32e59313e35a83d7c13880d5fdef", + "identity": { + "arm": "qwen36-nothink-card", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "6", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v7/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v7/grade.corrected.json new file mode 100644 index 00000000..a295344c --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v7/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-nothink-card_v7", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.7, + "urgency_accuracy_original": 0.6, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "14763f4c81938a1927418f257ccfa8569a3079250db2d3712516e09c2aeed0b1", + "identity": { + "arm": "qwen36-nothink-card", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "7", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v8/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v8/grade.corrected.json new file mode 100644 index 00000000..83ea1693 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v8/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-nothink-card_v8", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.833, + "urgency_accuracy_original": 0.733, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "c8cf968396524766b15880bcd8be8e3577ae32e59313e35a83d7c13880d5fdef", + "identity": { + "arm": "qwen36-nothink-card", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "8", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v9/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v9/grade.corrected.json new file mode 100644 index 00000000..b7648259 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-nothink-card_v9/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-nothink-card_v9", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.767, + "urgency_accuracy_original": 0.667, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "f2bb191f8b19342e864076b2d8af57222633c519790a9fa80b385759174c04d0", + "identity": { + "arm": "qwen36-nothink-card", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "9", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v1/grade.corrected.json new file mode 100644 index 00000000..7da7455f --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v1/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-think-card_v1", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.867, + "urgency_accuracy_original": 0.767, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "8f7506f3e196f54df6e53d719e8b98a5f6c8ccc20d00ad5a14f316721df53778", + "identity": { + "arm": "qwen36-think-card", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v10/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v10/grade.corrected.json new file mode 100644 index 00000000..059a6a37 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v10/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-think-card_v10", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.867, + "urgency_accuracy_original": 0.767, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "8f7506f3e196f54df6e53d719e8b98a5f6c8ccc20d00ad5a14f316721df53778", + "identity": { + "arm": "qwen36-think-card", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "10", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v2/grade.corrected.json new file mode 100644 index 00000000..5fa4da48 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v2/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-think-card_v2", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.867, + "urgency_accuracy_original": 0.767, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "8f7506f3e196f54df6e53d719e8b98a5f6c8ccc20d00ad5a14f316721df53778", + "identity": { + "arm": "qwen36-think-card", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v3/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v3/grade.corrected.json new file mode 100644 index 00000000..71bd1fd1 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v3/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-think-card_v3", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.867, + "urgency_accuracy_original": 0.767, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "8f7506f3e196f54df6e53d719e8b98a5f6c8ccc20d00ad5a14f316721df53778", + "identity": { + "arm": "qwen36-think-card", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "3", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v4/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v4/grade.corrected.json new file mode 100644 index 00000000..25605e61 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v4/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-think-card_v4", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.867, + "urgency_accuracy_original": 0.767, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "8f7506f3e196f54df6e53d719e8b98a5f6c8ccc20d00ad5a14f316721df53778", + "identity": { + "arm": "qwen36-think-card", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "4", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v5/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v5/grade.corrected.json new file mode 100644 index 00000000..69722bd7 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v5/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-think-card_v5", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.867, + "urgency_accuracy_original": 0.767, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "8f7506f3e196f54df6e53d719e8b98a5f6c8ccc20d00ad5a14f316721df53778", + "identity": { + "arm": "qwen36-think-card", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "5", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v6/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v6/grade.corrected.json new file mode 100644 index 00000000..23df0a31 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v6/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-think-card_v6", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.867, + "urgency_accuracy_original": 0.767, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "8f7506f3e196f54df6e53d719e8b98a5f6c8ccc20d00ad5a14f316721df53778", + "identity": { + "arm": "qwen36-think-card", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "6", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v7/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v7/grade.corrected.json new file mode 100644 index 00000000..5fa59df9 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v7/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-think-card_v7", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.867, + "urgency_accuracy_original": 0.767, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "8f7506f3e196f54df6e53d719e8b98a5f6c8ccc20d00ad5a14f316721df53778", + "identity": { + "arm": "qwen36-think-card", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "7", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v8/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v8/grade.corrected.json new file mode 100644 index 00000000..15156969 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v8/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-think-card_v8", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.867, + "urgency_accuracy_original": 0.767, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "8f7506f3e196f54df6e53d719e8b98a5f6c8ccc20d00ad5a14f316721df53778", + "identity": { + "arm": "qwen36-think-card", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "8", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v9/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v9/grade.corrected.json new file mode 100644 index 00000000..e8b31657 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p2_triage_qwen36-think-card_v9/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-think-card_v9", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.867, + "urgency_accuracy_original": 0.767, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "8f7506f3e196f54df6e53d719e8b98a5f6c8ccc20d00ad5a14f316721df53778", + "identity": { + "arm": "qwen36-think-card", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "9", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v1/grade.corrected.json new file mode 100644 index 00000000..675202fa --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v1/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-nothink-card_v1", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2762, + "ceiling": 700, + "counter_delta": -72, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 404, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "2571b5b40ed2ea20ecd87c4f5674f0cccd85911f66425301c03623bba4471d16", + "stored_grader_count": 404, + "wc_w_count": 476, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 476, + "wc_w_shell": 476, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal contracts unsigned | private-beta contract", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "branding deferred", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native in v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "mid-july", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 404, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "*high** | legal contracts unsigned | private-beta contract draft not returned from legal [wk6] | | **high** | maevia expectation gap | maevia was promised ga sdk; pushback expect", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1122, + "matched_span": "legal contracts unsigned | private-beta contract", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1074 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "2571b5b40ed2ea20ecd87c4f5674f0cccd85911f66425301c03623bba4471d16", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "09f6f954ec96df34c28f5b8155f578e6ab65a1a41c50ed8bfa0d4636bf6051ce", + "identity": { + "arm": "qwen36-nothink-card", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v10/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v10/grade.corrected.json new file mode 100644 index 00000000..be65cd20 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v10/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-nothink-card_v10", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2619, + "ceiling": 700, + "counter_delta": -85, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 381, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "fd95dfe915618ee409091826c9fd76c776ed8c26f444070b4a260b7c42df1c2d", + "stored_grader_count": 381, + "wc_w_count": 466, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 466, + "wc_w_shell": 466, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal has not", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "cut custom branding", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native in v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "mid-july", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "40-panel", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 381, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "nsigned | legal has not responded to draft [wk6] | | **high** | maevia expectation gap | was promised ga at conference; pushback expected [wk6]", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1087, + "matched_span": "legal has not", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1074 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "fd95dfe915618ee409091826c9fd76c776ed8c26f444070b4a260b7c42df1c2d", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "ae258d1a45812b07a76d202f79218439dd7ed79946351c5806d37958d03e643f", + "identity": { + "arm": "qwen36-nothink-card", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "10", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v2/grade.corrected.json new file mode 100644 index 00000000..00b4b84e --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v2/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-nothink-card_v2", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2600, + "ceiling": 700, + "counter_delta": -76, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 379, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "219f90bd8c4f432affb3093ec875862e8847050723a5b8d536ffcbf390c6aa37", + "stored_grader_count": 379, + "wc_w_count": 455, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 455, + "wc_w_shell": 455, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal has not responded to draft [wk6] | | **high** | maevia expectation gap | promised ga at conference; pushback expected [wk6] | | **medium** | iam effort slips | would delay", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "cut custom branding", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "mid-july", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 379, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "nsigned | legal has not responded to draft [wk6] | | **high** | maevia expectation gap | promised ga at conference; pushback expected [wk6] | | **medium** | iam effort slips | would delay sdk ga past august [wk5] | | **medium** | query-layer misses mid-may | o", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1219, + "matched_span": "legal has not responded to draft [wk6] | | **high** | maevia expectation gap | promised ga at conference; pushback expected [wk6] | | **medium** | iam effort slips | would delay", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1042 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "219f90bd8c4f432affb3093ec875862e8847050723a5b8d536ffcbf390c6aa37", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "daa573b2fdf97953f2cb8fedbeae19195e9e8580dadad85821142785454c7719", + "identity": { + "arm": "qwen36-nothink-card", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v3/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v3/grade.corrected.json new file mode 100644 index 00000000..33bdcc76 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v3/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-nothink-card_v3", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2604, + "ceiling": 700, + "counter_delta": -72, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 381, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "a2649b72e4459e79363605f9d2052aad1fcedd99db8500f9a670b9b50464589f", + "stored_grader_count": 381, + "wc_w_count": 453, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 453, + "wc_w_shell": 453, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal contracts unsigned | private-beta contract", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "branding deferred", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "mid-july", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 381, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "*high** | legal contracts unsigned | private-beta contract draft unreturned by legal [wk6] | | **high** | maevia expectation gap | promised ga sdk at conference; pushback expecte", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1105, + "matched_span": "legal contracts unsigned | private-beta contract", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1057 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "a2649b72e4459e79363605f9d2052aad1fcedd99db8500f9a670b9b50464589f", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "f7bf8ad50cc1addd43add9fc664d3484131ed5dd8c8e37b61e5bc44e2715ff9a", + "identity": { + "arm": "qwen36-nothink-card", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "3", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v5/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v5/grade.corrected.json new file mode 100644 index 00000000..edb449ba --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v5/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-nothink-card_v5", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2850, + "ceiling": 700, + "counter_delta": -106, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 418, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "651884dd7201a12b204fcc18de42411b132ea79b6fa9e64b4f40ad4e5e5f05b7", + "stored_grader_count": 418, + "wc_w_count": 524, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 524, + "wc_w_shell": 524, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal contracts | legal (tbd) | draft sent; no response (w6) | yes | --- ## risks | severity | risk | description | |---|---|---| | **high** | legal contracts not finalized | contract", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "branding deferred", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native in v2", + "matched": true + }, + "D4_option_b": { + "keyword": null, + "matched": false + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "mid-july", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom-branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "3/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 418, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "vate-beta legal contracts | legal (tbd) | draft sent; no response (w6) | yes | --- ## risks | severity | risk | description | |---|---|---| | **high** | legal contracts not finalized | contract draft sent to legal, no response; blocks pilot sign-on (w6). | | *", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1147, + "matched_span": "legal contracts | legal (tbd) | draft sent; no response (w6) | yes | --- ## risks | severity | risk | description | |---|---|---| | **high** | legal contracts not finalized | contract", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 964 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "651884dd7201a12b204fcc18de42411b132ea79b6fa9e64b4f40ad4e5e5f05b7", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "19e5fe2c6a0d6b0a31e78ffd470da117d753770278f828d7fa3d3840320c08d4", + "identity": { + "arm": "qwen36-nothink-card", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "5", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v6/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v6/grade.corrected.json new file mode 100644 index 00000000..336d0668 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v6/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-nothink-card_v6", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2644, + "ceiling": 700, + "counter_delta": -71, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 395, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "76b69d5bfc0ae43d057dd5afcb3bc8e6a38b481affcc6dfe4d176aa68fa98bdb", + "stored_grader_count": 395, + "wc_w_count": 466, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 466, + "wc_w_shell": 466, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal; no response yet [wk6] | | **high** | maevia expectation gap | promised ga at conference; will push back on private beta [wk6] | | **medium** | iam delay", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "branding deferred", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "mid-july", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "deferred", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 395, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "s sent to legal; no response yet [wk6] | | **high** | maevia expectation gap | promised ga at conference; will push back on private beta [wk6] | | **medium** | iam delay pushes sdk ga | 4-wk effort; if late, sdk ga slips past august [wk5] | | **medium** | quer", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1219, + "matched_span": "legal; no response yet [wk6] | | **high** | maevia expectation gap | promised ga at conference; will push back on private beta [wk6] | | **medium** | iam delay", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1060 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "76b69d5bfc0ae43d057dd5afcb3bc8e6a38b481affcc6dfe4d176aa68fa98bdb", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "99b2891ac1c583fd965ec3627171258c3a7f43ed32123125382e84896480e857", + "identity": { + "arm": "qwen36-nothink-card", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "6", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v7/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v7/grade.corrected.json new file mode 100644 index 00000000..580049bf --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v7/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-nothink-card_v7", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2580, + "ceiling": 700, + "counter_delta": -93, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 385, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "fd0a1a3d353b2b67cef0813a5fb9bd2b605099d149f2c5340cbbc4ba640168ad", + "stored_grader_count": 385, + "wc_w_count": 478, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 478, + "wc_w_shell": 478, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal tbd) | draft sent, no response (w6) | yes | ## risks (by severity) | severity | risk | description | |---|---|---| | high | legal delay", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "cut custom branding", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native in v2", + "matched": true + }, + "D4_option_b": { + "keyword": "private beta with", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": null, + "matched": false + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "4/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 385, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "s | sara (legal tbd) | draft sent, no response (w6) | yes | ## risks (by severity) | severity | risk | description | |---|---|---| | high | legal delay | contracts stalled; blocks private-beta launch (w6). | | high | maevia expectation | promised ga at confere", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 945, + "matched_span": "legal tbd) | draft sent, no response (w6) | yes | ## risks (by severity) | severity | risk | description | |---|---|---| | high | legal delay", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 804 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "fd0a1a3d353b2b67cef0813a5fb9bd2b605099d149f2c5340cbbc4ba640168ad", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "e0196224902b254b4059a31ca68b06965dd41e101e49546f58226e18c46b2206", + "identity": { + "arm": "qwen36-nothink-card", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "7", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v8/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v8/grade.corrected.json new file mode 100644 index 00000000..a4b11011 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v8/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-nothink-card_v8", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2656, + "ceiling": 700, + "counter_delta": -74, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 388, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "a060dc0ebd8056b4254e462757cc7720c2189e7f4d26d49b5762d7d4b05e216e", + "stored_grader_count": 388, + "wc_w_count": 462, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 462, + "wc_w_shell": 462, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal has not responded to draft [wk6] | | **high** | maevia expectation gap | promised ga sdk at conference; pushback expected [wk6] | | **medium** | iam 4-week effort | delay", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "branding deferred", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "mid-july", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 388, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "nsigned | legal has not responded to draft [wk6] | | **high** | maevia expectation gap | promised ga sdk at conference; pushback expected [wk6] | | **medium** | iam 4-week effort | delay slips sdk ga past august [wk5] | | **medium** | query-layer slips | 60% d", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1312, + "matched_span": "legal has not responded to draft [wk6] | | **high** | maevia expectation gap | promised ga sdk at conference; pushback expected [wk6] | | **medium** | iam 4-week effort | delay", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1136 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "a060dc0ebd8056b4254e462757cc7720c2189e7f4d26d49b5762d7d4b05e216e", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "be52d66cc4265b4d14bded6cfb727c9fba4a1cb290441f179f6613fa9d04c5f5", + "identity": { + "arm": "qwen36-nothink-card", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "8", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v10/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v10/grade.corrected.json new file mode 100644 index 00000000..b5624568 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v10/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-think-card_v10", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2792, + "ceiling": 700, + "counter_delta": -91, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 409, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "7c0532a0d023abced86d0752bcbbf8fd0a5f57573c4f27c012206da9b2128120", + "stored_grader_count": 409, + "wc_w_count": 500, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 500, + "wc_w_shell": 500, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal contracts pending [w6] | yes | --- ## risks (ordered by severity) | severity | risk | description | |---|---|---| | **high** | legal contract delay | private-beta contracts not finalized; legal has not", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "branding cut", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "responsive in v1", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": null, + "matched": false + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "4/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 409, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "ustomers; legal contracts pending [w6] | yes | --- ## risks (ordered by severity) | severity | risk | description | |---|---|---| | **high** | legal contract delay | private-beta contracts not finalized; legal has not responded [w6] | | **high** | maevia expec", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1221, + "matched_span": "legal contracts pending [w6] | yes | --- ## risks (ordered by severity) | severity | risk | description | |---|---|---| | **high** | legal contract delay | private-beta contracts not finalized; legal has not", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1014 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "7c0532a0d023abced86d0752bcbbf8fd0a5f57573c4f27c012206da9b2128120", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "a56f6e56400ce8158a801bb8cc783dc5efae05738a44b9657f3a06bc43a0f161", + "identity": { + "arm": "qwen36-think-card", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "10", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v2/grade.corrected.json new file mode 100644 index 00000000..b749f0d2 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v2/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-think-card_v2", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2748, + "ceiling": 700, + "counter_delta": -90, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 397, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "28a94b897b8ec04dcd8bbc46191dd2de411e7dd8c78380882629cd08770a69a2", + "stored_grader_count": 397, + "wc_w_count": 487, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 487, + "wc_w_shell": 487, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 3, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal delay on private-beta contracts | legal has not", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "cut custom branding", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native in v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "mid-july", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": null, + "matched": false + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "4/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 397, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "🔴 high | legal delay on private-beta contracts | legal has not responded to draft contracts. no contracts = no sdk in v1 (w5, w6). | | 🔴 high | no timeline slack | access-control dis", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1078, + "matched_span": "legal delay on private-beta contracts | legal has not", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1025 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "28a94b897b8ec04dcd8bbc46191dd2de411e7dd8c78380882629cd08770a69a2", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "3ac1f2702aa9a37c6a2e327a4971c5c6578da67f8dd8d33d288b44d305415425", + "identity": { + "arm": "qwen36-think-card", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v3/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v3/grade.corrected.json new file mode 100644 index 00000000..728bf2c1 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v3/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-think-card_v3", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2831, + "ceiling": 700, + "counter_delta": -100, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 415, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "9c3a4e403156dfb356af149d4c3badac132fcdf8670365d8b2e5fab0e52f9f82", + "stored_grader_count": 415, + "wc_w_count": 515, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 515, + "wc_w_shell": 515, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 3, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal contract delay | private-beta contracts not yet finalized; legal has not", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "branding cut", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "responsive in v1", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "july 2026", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 415, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "*high** | legal contract delay | private-beta contracts not yet finalized; legal has not responded [w6] | | **high** | maevia expectation gap | maevia was promised ga sdk at the conference; will push back [w6", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1245, + "matched_span": "legal contract delay | private-beta contracts not yet finalized; legal has not", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1167 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "9c3a4e403156dfb356af149d4c3badac132fcdf8670365d8b2e5fab0e52f9f82", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "eccc4b46e35238ec42beecd1ebc7a6c9ecaf7bdac8c660c54fd50c9b91eacd4c", + "identity": { + "arm": "qwen36-think-card", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "3", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v4/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v4/grade.corrected.json new file mode 100644 index 00000000..fc3c6e86 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v4/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-think-card_v4", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2748, + "ceiling": 700, + "counter_delta": -90, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 397, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "28a94b897b8ec04dcd8bbc46191dd2de411e7dd8c78380882629cd08770a69a2", + "stored_grader_count": 397, + "wc_w_count": 487, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 487, + "wc_w_shell": 487, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 3, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal delay on private-beta contracts | legal has not", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "cut custom branding", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native in v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "mid-july", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": null, + "matched": false + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "4/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 397, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "🔴 high | legal delay on private-beta contracts | legal has not responded to draft contracts. no contracts = no sdk in v1 (w5, w6). | | 🔴 high | no timeline slack | access-control dis", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1078, + "matched_span": "legal delay on private-beta contracts | legal has not", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1025 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "28a94b897b8ec04dcd8bbc46191dd2de411e7dd8c78380882629cd08770a69a2", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "3ac1f2702aa9a37c6a2e327a4971c5c6578da67f8dd8d33d288b44d305415425", + "identity": { + "arm": "qwen36-think-card", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "4", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v6/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v6/grade.corrected.json new file mode 100644 index 00000000..735a0328 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v6/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-think-card_v6", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2776, + "ceiling": 700, + "counter_delta": -93, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 400, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "f073baf7e0f127d9f0a670f40f17fffbddcaf580bfa0d7625e61dfb8fadee32a", + "stored_grader_count": 400, + "wc_w_count": 493, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 493, + "wc_w_shell": 493, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 3, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal delay on private-beta contracts | legal has not responded to draft. no contracts = no sdk in v1 (w5, w6). | | 🔴 high | no timeline slack | access-control discovery consumed all buffer. any further delay", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "cut custom branding", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "responsive in v1", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "mid-july", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 400, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "🔴 high | legal delay on private-beta contracts | legal has not responded to draft. no contracts = no sdk in v1 (w5, w6). | | 🔴 high | no timeline slack | access-control discovery consumed all buffer. any further delay pushes past mid-july board ceiling (w3, w5", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1245, + "matched_span": "legal delay on private-beta contracts | legal has not responded to draft. no contracts = no sdk in v1 (w5, w6). | | 🔴 high | no timeline slack | access-control discovery consumed all buffer. any further delay", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1037 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "f073baf7e0f127d9f0a670f40f17fffbddcaf580bfa0d7625e61dfb8fadee32a", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "558e18bd49729014b9107d61369091f553a6892172eb9f232b166e3f2d4766e7", + "identity": { + "arm": "qwen36-think-card", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "6", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v7/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v7/grade.corrected.json new file mode 100644 index 00000000..b5dcdfc0 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v7/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-think-card_v7", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2625, + "ceiling": 700, + "counter_delta": -86, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 388, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "60f577f326c5f1d95b4deabf6199a70cf9261f01978217d45ebe3f67991f4085", + "stored_grader_count": 388, + "wc_w_count": 474, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 474, + "wc_w_shell": 474, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal has not yet responded to the private-beta contract", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "branding deferred", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "responsive in v1", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "mid-july", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "40-panel", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 388, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "blocker: legal has not yet responded to the private-beta contract draft. --- ## workstreams | workstream | owner | current status | blocks v1? | |---|---|---|---| | query layer / backen", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 338, + "matched_span": "legal has not yet responded to the private-beta contract", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 282 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "60f577f326c5f1d95b4deabf6199a70cf9261f01978217d45ebe3f67991f4085", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "cc53a905e290113ecea1f1efbedaaa4c0f8a5a2c61367a40d4c9cc4eb557e93a", + "identity": { + "arm": "qwen36-think-card", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "7", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v8/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v8/grade.corrected.json new file mode 100644 index 00000000..ae1e59ad --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v8/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-think-card_v8", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 3075, + "ceiling": 700, + "counter_delta": -99, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 454, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "e225548a3c256aaf036733cd880e01c4ca802b0a4df7ff22da5801170de65c70", + "stored_grader_count": 454, + "wc_w_count": 553, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 553, + "wc_w_shell": 553, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 1, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal contracts pending [w6] | yes — contracts needed before launch | --- ## risks (ordered by severity) | severity | risk | description | |---|---|---| | **high** | legal contract delay", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "branding cut", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "responsive in v1", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "july 2026", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 454, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "ustomers; legal contracts pending [w6] | yes — contracts needed before launch | --- ## risks (ordered by severity) | severity | risk | description | |---|---|---| | **high** | legal contract delay | private-beta contracts not yet finalized; legal has not respo", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1257, + "matched_span": "legal contracts pending [w6] | yes — contracts needed before launch | --- ## risks (ordered by severity) | severity | risk | description | |---|---|---| | **high** | legal contract delay", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1071 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "e225548a3c256aaf036733cd880e01c4ca802b0a4df7ff22da5801170de65c70", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "7c96304be637169ed8e55b801493651d64a34a519712ea01e59bc07f1d6d5d7c", + "identity": { + "arm": "qwen36-think-card", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "8", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v9/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v9/grade.corrected.json new file mode 100644 index 00000000..23487e10 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_pm_qwen36-think-card_v9/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-think-card_v9", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2696, + "ceiling": 700, + "counter_delta": -90, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 385, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "0ea0c9742d5ee3b273a4640929242243d44744f300459efd239938144bb7afe7", + "stored_grader_count": 385, + "wc_w_count": 475, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 475, + "wc_w_shell": 475, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 3, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal delay on private-beta contracts** — legal has not", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "cut custom branding", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "responsive in v1", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "mid-july", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 385, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "high | **legal delay on private-beta contracts** — legal has not responded to the draft. no contracts = no sdk in v1 (w5, w6). | | 🔴 high | **no timeline slack** — access-control disco", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 996, + "matched_span": "legal delay on private-beta contracts** — legal has not", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 941 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "0ea0c9742d5ee3b273a4640929242243d44744f300459efd239938144bb7afe7", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "5fb3ebac19c739ab5c9f0815a60fb9ed29d189012105ba7f7a376b772f9a79fc", + "identity": { + "arm": "qwen36-think-card", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "9", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_writing_qwen36-nothink-card_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_writing_qwen36-nothink-card_v2/grade.corrected.json new file mode 100644 index 00000000..a945cd32 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_writing_qwen36-nothink-card_v2/grade.corrected.json @@ -0,0 +1,107 @@ +{ + "cell": "p3_writing_qwen36-nothink-card_v2", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + "length_attributable_fail": true, + "measurements": [ + { + "audience": "ceo_brief", + "bytes": 1744, + "ceiling": 250, + "counter_delta": 11, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 253, + "over_ceiling_by_grader_regex": true, + "relname": "ceo_brief.md", + "sha256": "633ef27d03aa8762366af8fe116713d08ca3a774abf96ae80a7d051c056f62d0", + "stored_grader_count": 253, + "wc_w_count": 242, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 242, + "wc_w_shell": 242, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "customer_email", + "bytes": 2151, + "ceiling": 350, + "counter_delta": 11, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 358, + "over_ceiling_by_grader_regex": true, + "relname": "customer_email.md", + "sha256": "20cece4d05419d13c0b33eb04988160a259d77013afa674fd7f7db006e9327c1", + "stored_grader_count": 358, + "wc_w_count": 347, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 347, + "wc_w_shell": 347, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "legal_summary", + "bytes": 2153, + "ceiling": 400, + "counter_delta": 15, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 312, + "over_ceiling_by_grader_regex": false, + "relname": "legal_summary.md", + "sha256": "f4eb27439d2f09694506845d0c05e464adf529783afda9e1924475857dcfaadf", + "stored_grader_count": 312, + "wc_w_count": 297, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 297, + "wc_w_shell": 297, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 10, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_writing", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "11c1f11385ca02494c8be6b18785228d21968a3195580b8dd54407720f668c46", + "identity": { + "arm": "qwen36-nothink-card", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "writing_editing", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_writing_qwen36-nothink-card_v3/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_writing_qwen36-nothink-card_v3/grade.corrected.json new file mode 100644 index 00000000..084af11a --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_writing_qwen36-nothink-card_v3/grade.corrected.json @@ -0,0 +1,107 @@ +{ + "cell": "p3_writing_qwen36-nothink-card_v3", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + "length_attributable_fail": true, + "measurements": [ + { + "audience": "ceo_brief", + "bytes": 1745, + "ceiling": 250, + "counter_delta": 11, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 254, + "over_ceiling_by_grader_regex": true, + "relname": "ceo_brief.md", + "sha256": "b4a2c080e9e7856e4797139b0b8406ae36295cb325120985a6e80db7d982a350", + "stored_grader_count": 254, + "wc_w_count": 243, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 243, + "wc_w_shell": 243, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "customer_email", + "bytes": 2050, + "ceiling": 350, + "counter_delta": 7, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 321, + "over_ceiling_by_grader_regex": false, + "relname": "customer_email.md", + "sha256": "bf3c688d3f135566f33c8a3b3b36dd3acab6c41fb3eace764f630f12ba6a035f", + "stored_grader_count": 321, + "wc_w_count": 314, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 314, + "wc_w_shell": 314, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "legal_summary", + "bytes": 2540, + "ceiling": 400, + "counter_delta": 19, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 364, + "over_ceiling_by_grader_regex": false, + "relname": "legal_summary.md", + "sha256": "a9234ccbf3964d44eefdb4c97871cd50abd703dea55893f32e6a86ff3d48bbaf", + "stored_grader_count": 364, + "wc_w_count": 345, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 345, + "wc_w_shell": 345, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 12, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_writing", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "62f72601bae331af843528eb16f72409e77541b583e5e850a3f579aff7a658f3", + "identity": { + "arm": "qwen36-nothink-card", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "3", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "writing_editing", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_writing_qwen36-nothink-card_v7/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_writing_qwen36-nothink-card_v7/grade.corrected.json new file mode 100644 index 00000000..ef7cb167 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_writing_qwen36-nothink-card_v7/grade.corrected.json @@ -0,0 +1,107 @@ +{ + "cell": "p3_writing_qwen36-nothink-card_v7", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + "length_attributable_fail": true, + "measurements": [ + { + "audience": "ceo_brief", + "bytes": 1752, + "ceiling": 250, + "counter_delta": 12, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 257, + "over_ceiling_by_grader_regex": true, + "relname": "ceo_brief.md", + "sha256": "7418519a7dc60ebb246ed82c5a50bd77f22b8ca71b09bd5231278b95a07b26b4", + "stored_grader_count": 257, + "wc_w_count": 245, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 245, + "wc_w_shell": 245, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "customer_email", + "bytes": 2029, + "ceiling": 350, + "counter_delta": 5, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 320, + "over_ceiling_by_grader_regex": false, + "relname": "customer_email.md", + "sha256": "0ebfc428ad520c18bce5dff74e46fe6f708615b7e6d16787037df88ed617bf6e", + "stored_grader_count": 320, + "wc_w_count": 315, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 315, + "wc_w_shell": 315, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "legal_summary", + "bytes": 2132, + "ceiling": 400, + "counter_delta": 6, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 303, + "over_ceiling_by_grader_regex": false, + "relname": "legal_summary.md", + "sha256": "7410ca5071313794a298c27042f921688edad1e5ad21003d6977e40097f8a30c", + "stored_grader_count": 303, + "wc_w_count": 297, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 297, + "wc_w_shell": 297, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 13, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_writing", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "9106581368e6f7bebb2cd36520a0b9ddc823d5a80e7b642b06c1be9e6c245781", + "identity": { + "arm": "qwen36-nothink-card", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "7", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "writing_editing", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_writing_qwen36-nothink-card_v9/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_writing_qwen36-nothink-card_v9/grade.corrected.json new file mode 100644 index 00000000..e2824ec8 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_writing_qwen36-nothink-card_v9/grade.corrected.json @@ -0,0 +1,107 @@ +{ + "cell": "p3_writing_qwen36-nothink-card_v9", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + "length_attributable_fail": true, + "measurements": [ + { + "audience": "ceo_brief", + "bytes": 1705, + "ceiling": 250, + "counter_delta": 7, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 255, + "over_ceiling_by_grader_regex": true, + "relname": "ceo_brief.md", + "sha256": "64635cf8829074ce5db32fe026fbf6ef0433e0043c0236fa7a9813932d7227f2", + "stored_grader_count": 255, + "wc_w_count": 248, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 248, + "wc_w_shell": 248, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "customer_email", + "bytes": 1970, + "ceiling": 350, + "counter_delta": 3, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 316, + "over_ceiling_by_grader_regex": false, + "relname": "customer_email.md", + "sha256": "8d40518b1263e1d861c9b076302c70be78e0f1741c39d397b7bfd853f895da37", + "stored_grader_count": 316, + "wc_w_count": 313, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 313, + "wc_w_shell": 313, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "legal_summary", + "bytes": 2424, + "ceiling": 400, + "counter_delta": 11, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 344, + "over_ceiling_by_grader_regex": false, + "relname": "legal_summary.md", + "sha256": "3e7683eedac40fa948dd2c100d104c3f1fc450992c3dbd85716228b844e0fd1e", + "stored_grader_count": 344, + "wc_w_count": 333, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 333, + "wc_w_shell": 333, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 9, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_writing", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "597efaa38804bd923ed137af23f1d6dc5f14afd676f07f3779f14af5d922541f", + "identity": { + "arm": "qwen36-nothink-card", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "9", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "writing_editing", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_writing_qwen36-think-card_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_writing_qwen36-think-card_v2/grade.corrected.json new file mode 100644 index 00000000..59094696 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_writing_qwen36-think-card_v2/grade.corrected.json @@ -0,0 +1,107 @@ +{ + "cell": "p3_writing_qwen36-think-card_v2", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + "length_attributable_fail": true, + "measurements": [ + { + "audience": "ceo_brief", + "bytes": 1797, + "ceiling": 250, + "counter_delta": 5, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 245, + "over_ceiling_by_grader_regex": false, + "relname": "ceo_brief.md", + "sha256": "710da5c8dc43dc287b978fb32b290f3cce1abe0c13af608bb652e9fd1f6830aa", + "stored_grader_count": 245, + "wc_w_count": 240, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 240, + "wc_w_shell": 240, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "customer_email", + "bytes": 2168, + "ceiling": 350, + "counter_delta": 12, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 358, + "over_ceiling_by_grader_regex": true, + "relname": "customer_email.md", + "sha256": "63d39ef0bee461f54bac5d03547580ff44421da923755bcbf88184f35b487a06", + "stored_grader_count": 358, + "wc_w_count": 346, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 346, + "wc_w_shell": 346, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "legal_summary", + "bytes": 2101, + "ceiling": 400, + "counter_delta": 3, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 292, + "over_ceiling_by_grader_regex": false, + "relname": "legal_summary.md", + "sha256": "b22212de0823e7fca2a643a1d61e0fa8117829c7084af18903b3881cd4919542", + "stored_grader_count": 292, + "wc_w_count": 289, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 289, + "wc_w_shell": 289, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 6, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_writing", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "0b46c5b98bb6a79cf1d128e4d7e80d307639c46e38d5121c77a159ed58b589a9", + "identity": { + "arm": "qwen36-think-card", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "writing_editing", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_writing_qwen36-think-card_v3/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_writing_qwen36-think-card_v3/grade.corrected.json new file mode 100644 index 00000000..a7706878 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_writing_qwen36-think-card_v3/grade.corrected.json @@ -0,0 +1,107 @@ +{ + "cell": "p3_writing_qwen36-think-card_v3", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + "length_attributable_fail": true, + "measurements": [ + { + "audience": "ceo_brief", + "bytes": 1790, + "ceiling": 250, + "counter_delta": 4, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 248, + "over_ceiling_by_grader_regex": false, + "relname": "ceo_brief.md", + "sha256": "082014f92353bc9a47b953ed4940d2994f5710b8fc4b1407891d30c07e3d401f", + "stored_grader_count": 248, + "wc_w_count": 244, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 244, + "wc_w_shell": 244, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "customer_email", + "bytes": 2169, + "ceiling": 350, + "counter_delta": 12, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 358, + "over_ceiling_by_grader_regex": true, + "relname": "customer_email.md", + "sha256": "34d888f16384391fc1475ca205da868e10d1677bde1c9c46de0e91823b5a14c1", + "stored_grader_count": 358, + "wc_w_count": 346, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 346, + "wc_w_shell": 346, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "legal_summary", + "bytes": 2237, + "ceiling": 400, + "counter_delta": 12, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 325, + "over_ceiling_by_grader_regex": false, + "relname": "legal_summary.md", + "sha256": "29578f85056103a6d880ef289526e58b4db2473d64b374c4204f7c173b533360", + "stored_grader_count": 325, + "wc_w_count": 313, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 313, + "wc_w_shell": 313, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 6, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_writing", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "f967393d93350566b02ddc49996111952de6ee73ebdeccb75b86c4e5a1c64e2a", + "identity": { + "arm": "qwen36-think-card", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "3", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "writing_editing", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_writing_qwen36-think-card_v6/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_writing_qwen36-think-card_v6/grade.corrected.json new file mode 100644 index 00000000..73495f65 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q36-card/logs/p3_writing_qwen36-think-card_v6/grade.corrected.json @@ -0,0 +1,107 @@ +{ + "cell": "p3_writing_qwen36-think-card_v6", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + "length_attributable_fail": true, + "measurements": [ + { + "audience": "ceo_brief", + "bytes": 1784, + "ceiling": 250, + "counter_delta": 4, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 244, + "over_ceiling_by_grader_regex": false, + "relname": "ceo_brief.md", + "sha256": "dd7105b1a5b529d524419cdffafe212f94531ee591f4f371df0a9ee8a8934e6a", + "stored_grader_count": 244, + "wc_w_count": 240, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 240, + "wc_w_shell": 240, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "customer_email", + "bytes": 2169, + "ceiling": 350, + "counter_delta": 12, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 358, + "over_ceiling_by_grader_regex": true, + "relname": "customer_email.md", + "sha256": "34d888f16384391fc1475ca205da868e10d1677bde1c9c46de0e91823b5a14c1", + "stored_grader_count": 358, + "wc_w_count": 346, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 346, + "wc_w_shell": 346, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "legal_summary", + "bytes": 2101, + "ceiling": 400, + "counter_delta": 3, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 292, + "over_ceiling_by_grader_regex": false, + "relname": "legal_summary.md", + "sha256": "b22212de0823e7fca2a643a1d61e0fa8117829c7084af18903b3881cd4919542", + "stored_grader_count": 292, + "wc_w_count": 289, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 289, + "wc_w_shell": 289, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 8, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_writing", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "45ed29c4606d76785398deb62adba70c070d46d0c63a6385004ec569efd32760", + "identity": { + "arm": "qwen36-think-card", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "6", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q36-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "writing_editing", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-nothink-card_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-nothink-card_v1/grade.corrected.json new file mode 100644 index 00000000..c49873b9 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-nothink-card_v1/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-nothink-card_v1", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.833, + "urgency_accuracy_original": 0.733, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "5461c9cf0c5c1ac314577f21c9bab9e0eaf5e044db49f3948ad7d50564dfdee6", + "identity": { + "arm": "qwen38-nothink-card", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.7/p0.8/pp1.5", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-nothink-card_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-nothink-card_v2/grade.corrected.json new file mode 100644 index 00000000..11a213ed --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-nothink-card_v2/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-nothink-card_v2", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.767, + "urgency_accuracy_original": 0.667, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "fd3dfa2ddfa92a944514ca7ec99aead62a51ba08f4ae5b8b4e802b2b89733bd7", + "identity": { + "arm": "qwen38-nothink-card", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.7/p0.8/pp1.5", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-nothink-card_v3/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-nothink-card_v3/grade.corrected.json new file mode 100644 index 00000000..f30e35b0 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-nothink-card_v3/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-nothink-card_v3", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.833, + "urgency_accuracy_original": 0.733, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "ba37718324017ae8202186f2b3197672e02043beded44c9ff867431166d3f6f1", + "identity": { + "arm": "qwen38-nothink-card", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.7/p0.8/pp1.5", + "replicate": "3", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-nothink-card_v4/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-nothink-card_v4/grade.corrected.json new file mode 100644 index 00000000..78db78c6 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-nothink-card_v4/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-nothink-card_v4", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.833, + "urgency_accuracy_original": 0.733, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "5461c9cf0c5c1ac314577f21c9bab9e0eaf5e044db49f3948ad7d50564dfdee6", + "identity": { + "arm": "qwen38-nothink-card", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.7/p0.8/pp1.5", + "replicate": "4", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-nothink-card_v5/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-nothink-card_v5/grade.corrected.json new file mode 100644 index 00000000..fff065d3 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-nothink-card_v5/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-nothink-card_v5", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.833, + "urgency_accuracy_original": 0.733, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "5461c9cf0c5c1ac314577f21c9bab9e0eaf5e044db49f3948ad7d50564dfdee6", + "identity": { + "arm": "qwen38-nothink-card", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.7/p0.8/pp1.5", + "replicate": "5", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-think-low-card_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-think-low-card_v1/grade.corrected.json new file mode 100644 index 00000000..ade0f95e --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-think-low-card_v1/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-think-low-card_v1", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.833, + "urgency_accuracy_original": 0.733, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "436c89372168a5733c003ebdc5205e75a6588d55082de50c0f6e6f5429d51610", + "identity": { + "arm": "qwen38-think-low-card", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-think-medium-card_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-think-medium-card_v1/grade.corrected.json new file mode 100644 index 00000000..f9a1f4cc --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-think-medium-card_v1/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-think-medium-card_v1", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.8, + "urgency_accuracy_original": 0.7, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "6c2c6fc25f99dc8a7636b3539059dd89daa9aa4c994290c15bcdced5726d1087", + "identity": { + "arm": "qwen38-think-medium-card", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-think-xhigh-card_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-think-xhigh-card_v1/grade.corrected.json new file mode 100644 index 00000000..0b945d74 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p2_triage_qwen38-think-xhigh-card_v1/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-think-xhigh-card_v1", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.867, + "urgency_accuracy_original": 0.767, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "8f7506f3e196f54df6e53d719e8b98a5f6c8ccc20d00ad5a14f316721df53778", + "identity": { + "arm": "qwen38-think-xhigh-card", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_business_qwen38-think-low-card_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_business_qwen38-think-low-card_v1/grade.corrected.json new file mode 100644 index 00000000..1e60ff6c --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_business_qwen38-think-low-card_v1/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_business_qwen38-think-low-card_v1", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_business_memo_grade.py:115", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4577, + "ceiling": 700, + "counter_delta": 34, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 725, + "over_ceiling_by_grader_regex": true, + "relname": "memo.md", + "sha256": "19fb5c692b729f44fec55c1524d5049367b5acffa2cc68484fc3815439694af7", + "stored_grader_count": 725, + "wc_w_count": 691, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 691, + "wc_w_shell": 691, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 3, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_business", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "1f218279298b32e8b7e577aca48f4f4c859482ec59e7d8a7132b6608b11ba328", + "identity": { + "arm": "qwen38-think-low-card", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "business_memo", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_business_qwen38-think-medium-card_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_business_qwen38-think-medium-card_v1/grade.corrected.json new file mode 100644 index 00000000..027f77b3 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_business_qwen38-think-medium-card_v1/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_business_qwen38-think-medium-card_v1", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_business_memo_grade.py:115", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4466, + "ceiling": 700, + "counter_delta": 46, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 744, + "over_ceiling_by_grader_regex": true, + "relname": "memo.md", + "sha256": "025ef52ae4b13c603cd974a26e0a1605c2f564591afb7ffdacf0fa219d879bdb", + "stored_grader_count": 744, + "wc_w_count": 698, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 698, + "wc_w_shell": 698, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_business", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "5d3e2555a6b5384d4b4be786eb616e39499cdc2cb6052e709b5f2275414d0475", + "identity": { + "arm": "qwen38-think-medium-card", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "business_memo", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_business_qwen38-think-xhigh-card_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_business_qwen38-think-xhigh-card_v1/grade.corrected.json new file mode 100644 index 00000000..6941c9f5 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_business_qwen38-think-xhigh-card_v1/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_business_qwen38-think-xhigh-card_v1", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_business_memo_grade.py:115", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4423, + "ceiling": 700, + "counter_delta": 26, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 725, + "over_ceiling_by_grader_regex": true, + "relname": "memo.md", + "sha256": "a606e93432bbcabe6ac6e774c469aed1953f79df2c1695c5235ebddc9c750a4d", + "stored_grader_count": 725, + "wc_w_count": 699, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 699, + "wc_w_shell": 699, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 14, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_business", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "2a73516d8555274a3221a733c690281e712fd27c9ca0fe93f3f28d55626900f4", + "identity": { + "arm": "qwen38-think-xhigh-card", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "business_memo", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_doc_qwen38-nothink-card_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_doc_qwen38-nothink-card_v1/grade.corrected.json new file mode 100644 index 00000000..8b6177c5 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_doc_qwen38-nothink-card_v1/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen38-nothink-card_v1", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4314, + "ceiling": 700, + "counter_delta": 28, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 722, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "c503fb50d5dde45e4104e677f1bf70df179f96f78701e4a9ae407c9d0dbfda34", + "stored_grader_count": 722, + "wc_w_count": 694, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 694, + "wc_w_shell": 694, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 4, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "f2fe21d61916f4395ff124cc8b32cbb19ff10be1a764e1400caba5551b205336", + "identity": { + "arm": "qwen38-nothink-card", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.7/p0.8/pp1.5", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_doc_qwen38-nothink-card_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_doc_qwen38-nothink-card_v2/grade.corrected.json new file mode 100644 index 00000000..7cc0dd1a --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_doc_qwen38-nothink-card_v2/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen38-nothink-card_v2", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4309, + "ceiling": 700, + "counter_delta": 28, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 722, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "33a0103fb974028a8165142d0d0d51e8add1121890039cbd89c1f65f78f77d38", + "stored_grader_count": 722, + "wc_w_count": 694, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 694, + "wc_w_shell": 694, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 4, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "f2fe21d61916f4395ff124cc8b32cbb19ff10be1a764e1400caba5551b205336", + "identity": { + "arm": "qwen38-nothink-card", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.7/p0.8/pp1.5", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_doc_qwen38-nothink-card_v3/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_doc_qwen38-nothink-card_v3/grade.corrected.json new file mode 100644 index 00000000..06e8a6dd --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_doc_qwen38-nothink-card_v3/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen38-nothink-card_v3", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4264, + "ceiling": 700, + "counter_delta": 30, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 718, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "0ad5c6b7d72814bb3a90f560a98d70f3be9f20e0a1ee7ddaef4bf5f8798bea42", + "stored_grader_count": 718, + "wc_w_count": 688, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 688, + "wc_w_shell": 688, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 4, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "b42347d2809babc0df298c749b6f1de111648183b7135e6e1b15c218f68f9c9e", + "identity": { + "arm": "qwen38-nothink-card", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.7/p0.8/pp1.5", + "replicate": "3", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_doc_qwen38-nothink-card_v4/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_doc_qwen38-nothink-card_v4/grade.corrected.json new file mode 100644 index 00000000..bd2c20f0 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_doc_qwen38-nothink-card_v4/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen38-nothink-card_v4", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4361, + "ceiling": 700, + "counter_delta": 31, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 730, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "b8783d1d25e3314b55569a956e16ef73e2307fc1bb22219755c98a115c22466a", + "stored_grader_count": 730, + "wc_w_count": 699, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 699, + "wc_w_shell": 699, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 4, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "c7d7deaa2efad7677b8ef4c6b1b5b173ac5212d649bdcfd4a679e2e84688692c", + "identity": { + "arm": "qwen38-nothink-card", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.7/p0.8/pp1.5", + "replicate": "4", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_doc_qwen38-nothink-card_v5/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_doc_qwen38-nothink-card_v5/grade.corrected.json new file mode 100644 index 00000000..8bbb9a17 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_doc_qwen38-nothink-card_v5/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen38-nothink-card_v5", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4326, + "ceiling": 700, + "counter_delta": 33, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 733, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "fa021336a40139b75c97e8fbbd17fd401c5259dfc8856e6aa82931cc2c20386b", + "stored_grader_count": 733, + "wc_w_count": 700, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 700, + "wc_w_shell": 700, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 4, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "f2eb0d3223e2e3f6086a697716479b98e6eab30adf17e0d09c6c6aecac346351", + "identity": { + "arm": "qwen38-nothink-card", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.7/p0.8/pp1.5", + "replicate": "5", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_doc_qwen38-think-medium-card_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_doc_qwen38-think-medium-card_v1/grade.corrected.json new file mode 100644 index 00000000..2dc8661d --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_doc_qwen38-think-medium-card_v1/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen38-think-medium-card_v1", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4277, + "ceiling": 700, + "counter_delta": 9, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 701, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "25e2395d5f720827cba6745f767ccdc20d70ef3862c94312666dc87db6b2f123", + "stored_grader_count": 701, + "wc_w_count": 692, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 692, + "wc_w_shell": 692, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 6, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "e5bd66cdc48a42d2e5d986066565ae715564f61eeb5991560ef26f537ae9d2a8", + "identity": { + "arm": "qwen38-think-medium-card", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_doc_qwen38-think-xhigh-card_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_doc_qwen38-think-xhigh-card_v1/grade.corrected.json new file mode 100644 index 00000000..12dc677b --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_doc_qwen38-think-xhigh-card_v1/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen38-think-xhigh-card_v1", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4259, + "ceiling": 700, + "counter_delta": 29, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 726, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "005ae6a4ba41a9410df443fc1a238ce075bd0827b8a8702d1457391317648eef", + "stored_grader_count": 726, + "wc_w_count": 697, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 697, + "wc_w_shell": 697, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 9, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "a1656bdf94ca7414c22f483d41511412e6dbb6d83ec787d992ea2b6fc8cba463", + "identity": { + "arm": "qwen38-think-xhigh-card", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_pm_qwen38-think-low-card_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_pm_qwen38-think-low-card_v1/grade.corrected.json new file mode 100644 index 00000000..7a80a612 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_pm_qwen38-think-low-card_v1/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen38-think-low-card_v1", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 3072, + "ceiling": 700, + "counter_delta": -65, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 488, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "66f94152243306f86bf908f519f695f112dcf2c6d7dde9c31e56400c81b3157c", + "stored_grader_count": 488, + "wc_w_count": 553, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 553, + "wc_w_shell": 553, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal has not responded to the private-beta contract", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "cut custom-branding", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "responsive in v1", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": null, + "matched": false + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": "maevia push", + "matched": true + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom-branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "4/5", + "risk_recall": "4/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 488, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "| high | legal has not responded to the private-beta contract draft. if legal slips, the option b path (brendan's chosen route) cannot be executed. | week 6 | | medium | 40-panel-pe", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1330, + "matched_span": "legal has not responded to the private-beta contract", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1278 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "66f94152243306f86bf908f519f695f112dcf2c6d7dde9c31e56400c81b3157c", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "753f72fe8c14d5a50d2ac932b2ce626268d5fa2b4dca0cde98d633087edc2715", + "identity": { + "arm": "qwen38-think-low-card", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_pm_qwen38-think-medium-card_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_pm_qwen38-think-medium-card_v1/grade.corrected.json new file mode 100644 index 00000000..a99cd9a2 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_pm_qwen38-think-medium-card_v1/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen38-think-medium-card_v1", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 3329, + "ceiling": 700, + "counter_delta": -51, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 534, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "ae92f988722bc81a259b74d9cb5e9618b7843463301503d1e0a919b217f8cffe", + "stored_grader_count": 534, + "wc_w_count": 585, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 585, + "wc_w_shell": 585, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal has not engaged on the private-beta contract", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "branding cut", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "responsive in v1", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "mid-july", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 534, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "*high** | legal has not engaged on the private-beta contract draft; beta launch is gated on it (wk 6). | | **high** | customer commitments from the conference: mid-july is defensib", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1586, + "matched_span": "legal has not engaged on the private-beta contract", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1536 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "ae92f988722bc81a259b74d9cb5e9618b7843463301503d1e0a919b217f8cffe", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "734396796aaab1c15c544a0d8aaa2a016cfade3fe988330fa6d0d90ed0a81651", + "identity": { + "arm": "qwen38-think-medium-card", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_writing_qwen38-nothink-card_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_writing_qwen38-nothink-card_v2/grade.corrected.json new file mode 100644 index 00000000..72e2a455 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_writing_qwen38-nothink-card_v2/grade.corrected.json @@ -0,0 +1,107 @@ +{ + "cell": "p3_writing_qwen38-nothink-card_v2", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + "length_attributable_fail": true, + "measurements": [ + { + "audience": "ceo_brief", + "bytes": 1643, + "ceiling": 250, + "counter_delta": 10, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 258, + "over_ceiling_by_grader_regex": true, + "relname": "ceo_brief.md", + "sha256": "913edb39f774f1a309063ead52e9f0b4a0393ff94807f12f0556139696cfcf83", + "stored_grader_count": 258, + "wc_w_count": 248, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 248, + "wc_w_shell": 248, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "customer_email", + "bytes": 2082, + "ceiling": 350, + "counter_delta": 10, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 358, + "over_ceiling_by_grader_regex": true, + "relname": "customer_email.md", + "sha256": "8d614adfb057cdc92ab086bd36677a7936304d8c1e2c27c256145551af083435", + "stored_grader_count": 358, + "wc_w_count": 348, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 348, + "wc_w_shell": 348, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "legal_summary", + "bytes": 2257, + "ceiling": 400, + "counter_delta": 13, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 328, + "over_ceiling_by_grader_regex": false, + "relname": "legal_summary.md", + "sha256": "25eb2754d1221108a848525c41710ad8b356384e757b2d666e9b004a54f3119a", + "stored_grader_count": 328, + "wc_w_count": 315, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 315, + "wc_w_shell": 315, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 17, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_writing", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "9307f879a77fc1d25b213d417e0535bdc1cd0eb39b63fd597f466858a94fd53b", + "identity": { + "arm": "qwen38-nothink-card", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.7/p0.8/pp1.5", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "writing_editing", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_writing_qwen38-nothink-card_v5/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_writing_qwen38-nothink-card_v5/grade.corrected.json new file mode 100644 index 00000000..c6882c09 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_writing_qwen38-nothink-card_v5/grade.corrected.json @@ -0,0 +1,107 @@ +{ + "cell": "p3_writing_qwen38-nothink-card_v5", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + "length_attributable_fail": true, + "measurements": [ + { + "audience": "ceo_brief", + "bytes": 1654, + "ceiling": 250, + "counter_delta": 7, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 256, + "over_ceiling_by_grader_regex": true, + "relname": "ceo_brief.md", + "sha256": "7657d1341011e828c1392d2ff8f2b94b6b986964318078d672d079004bb34678", + "stored_grader_count": 256, + "wc_w_count": 249, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 249, + "wc_w_shell": 249, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "customer_email", + "bytes": 2091, + "ceiling": 350, + "counter_delta": 10, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 360, + "over_ceiling_by_grader_regex": true, + "relname": "customer_email.md", + "sha256": "210f612811f3ad1407b8b1a7725294480bbf3b2e52890f18d198bc166b0b255b", + "stored_grader_count": 360, + "wc_w_count": 350, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 350, + "wc_w_shell": 350, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "legal_summary", + "bytes": 2257, + "ceiling": 400, + "counter_delta": 13, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 328, + "over_ceiling_by_grader_regex": false, + "relname": "legal_summary.md", + "sha256": "25eb2754d1221108a848525c41710ad8b356384e757b2d666e9b004a54f3119a", + "stored_grader_count": 328, + "wc_w_count": 315, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 315, + "wc_w_shell": 315, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 7, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_writing", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "968f61489f05f6e2319b004d51da209e33ca944d7aee8a50a4aaf077d4ebc0ec", + "identity": { + "arm": "qwen38-nothink-card", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.7/p0.8/pp1.5", + "replicate": "5", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "writing_editing", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_writing_qwen38-think-low-card_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_writing_qwen38-think-low-card_v1/grade.corrected.json new file mode 100644 index 00000000..31b16c49 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_writing_qwen38-think-low-card_v1/grade.corrected.json @@ -0,0 +1,107 @@ +{ + "cell": "p3_writing_qwen38-think-low-card_v1", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + "length_attributable_fail": true, + "measurements": [ + { + "audience": "ceo_brief", + "bytes": 1672, + "ceiling": 250, + "counter_delta": 3, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 252, + "over_ceiling_by_grader_regex": true, + "relname": "ceo_brief.md", + "sha256": "83e7708f943439261dcfddd63cfc06c73e220260aed733b5fa2a3116a1199a57", + "stored_grader_count": 252, + "wc_w_count": 249, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 249, + "wc_w_shell": 249, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "customer_email", + "bytes": 2078, + "ceiling": 350, + "counter_delta": 11, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 359, + "over_ceiling_by_grader_regex": true, + "relname": "customer_email.md", + "sha256": "a48a85605a046bc0a2cdf18c44b2cf3f69d9e2b6477f883af8c61476d99b02d3", + "stored_grader_count": 359, + "wc_w_count": 348, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 348, + "wc_w_shell": 348, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "legal_summary", + "bytes": 2644, + "ceiling": 400, + "counter_delta": 17, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 389, + "over_ceiling_by_grader_regex": false, + "relname": "legal_summary.md", + "sha256": "65a2c92755bf026116f8c0c481739ba8a449f9f25685e663fc1b160446b3c2e0", + "stored_grader_count": 389, + "wc_w_count": 372, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 372, + "wc_w_shell": 372, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 4, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_writing", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "db02c11b56226bf4d89a07b118067edca8ac87927849c24870b324fe9beab78c", + "identity": { + "arm": "qwen38-think-low-card", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "writing_editing", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_writing_qwen38-think-medium-card_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_writing_qwen38-think-medium-card_v1/grade.corrected.json new file mode 100644 index 00000000..5baf76ff --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_writing_qwen38-think-medium-card_v1/grade.corrected.json @@ -0,0 +1,107 @@ +{ + "cell": "p3_writing_qwen38-think-medium-card_v1", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + "length_attributable_fail": true, + "measurements": [ + { + "audience": "ceo_brief", + "bytes": 1664, + "ceiling": 250, + "counter_delta": 18, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 264, + "over_ceiling_by_grader_regex": true, + "relname": "ceo_brief.md", + "sha256": "d9caba8d7467371828ceab1dce417c6efcb3364cbf58f83012df32ee1fcf8a46", + "stored_grader_count": 264, + "wc_w_count": 246, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 246, + "wc_w_shell": 246, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "customer_email", + "bytes": 1981, + "ceiling": 350, + "counter_delta": 6, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 341, + "over_ceiling_by_grader_regex": false, + "relname": "customer_email.md", + "sha256": "c70988d6e5d38e35ad06d05eb8d58c98c8dd7072dfac095f8796eec3a783ecf4", + "stored_grader_count": 341, + "wc_w_count": 335, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 335, + "wc_w_shell": 335, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "legal_summary", + "bytes": 2062, + "ceiling": 400, + "counter_delta": 16, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 309, + "over_ceiling_by_grader_regex": false, + "relname": "legal_summary.md", + "sha256": "71c235b0443fd47d308ee289f3c8f45fa308d7a35e5d01b111406e3ab3da4f98", + "stored_grader_count": 309, + "wc_w_count": 293, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 293, + "wc_w_shell": 293, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 6, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_writing", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "d229087c5e2b685845f4d5886e6f1bf6ecc156c1d8cfdaf3f7d3169225e4e3d0", + "identity": { + "arm": "qwen38-think-medium-card", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "writing_editing", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_writing_qwen38-think-xhigh-card_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_writing_qwen38-think-xhigh-card_v1/grade.corrected.json new file mode 100644 index 00000000..4cfc1306 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-card/logs/p3_writing_qwen38-think-xhigh-card_v1/grade.corrected.json @@ -0,0 +1,107 @@ +{ + "cell": "p3_writing_qwen38-think-xhigh-card_v1", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + "length_attributable_fail": true, + "measurements": [ + { + "audience": "ceo_brief", + "bytes": 1628, + "ceiling": 250, + "counter_delta": 8, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 251, + "over_ceiling_by_grader_regex": true, + "relname": "ceo_brief.md", + "sha256": "35bcd8c844c00a602cc102e6e9a2e120a8c7af1f1437394d19f01b37d064fc3c", + "stored_grader_count": 251, + "wc_w_count": 243, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 243, + "wc_w_shell": 243, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "customer_email", + "bytes": 2040, + "ceiling": 350, + "counter_delta": 4, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 351, + "over_ceiling_by_grader_regex": true, + "relname": "customer_email.md", + "sha256": "47d3d9255e0280ad344282e086db8ed78189671c3296af964d89b2227096f7ab", + "stored_grader_count": 351, + "wc_w_count": 347, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 347, + "wc_w_shell": 347, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "legal_summary", + "bytes": 2759, + "ceiling": 400, + "counter_delta": 15, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 399, + "over_ceiling_by_grader_regex": false, + "relname": "legal_summary.md", + "sha256": "b79550d4d25eb09a9793719a03d99a9db10466e672c40f8a65030928595e6454", + "stored_grader_count": 399, + "wc_w_count": 384, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 384, + "wc_w_shell": 384, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 7, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_writing", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "a0e72c8e2e474a524076190c8cc1e854e5db296a57725574fb84fd742b5edc8c", + "identity": { + "arm": "qwen38-think-xhigh-card", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-card", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "writing_editing", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-q8/logs/p2_triage_qwen38q8-nothink-matched_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-q8/logs/p2_triage_qwen38q8-nothink-matched_v1/grade.corrected.json new file mode 100644 index 00000000..b94144e2 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-q8/logs/p2_triage_qwen38q8-nothink-matched_v1/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38q8-nothink-matched_v1", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.833, + "urgency_accuracy_original": 0.733, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "6c279e2fbe78b10ed86df29acc99b9948a47379a7e1deb800a1f504552bbc8a6", + "identity": { + "arm": "qwen38q8-nothink-matched", + "mode": "no-think", + "model": "3.8", + "quant": "Q8_0", + "regime": "T0.3/p0.8/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-q8", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-q8/logs/p3_pm_qwen38q8-nothink-matched_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-q8/logs/p3_pm_qwen38q8-nothink-matched_v1/grade.corrected.json new file mode 100644 index 00000000..ddf32770 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-q38-q8/logs/p3_pm_qwen38q8-nothink-matched_v1/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen38q8-nothink-matched_v1", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2919, + "ceiling": 700, + "counter_delta": -14, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 463, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "a87b53af8b0cd6d07a34d8f7b5324a0e4f4757ccad1a6de504311ef54867b75b", + "stored_grader_count": 463, + "wc_w_count": 477, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 477, + "wc_w_shell": 477, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 1, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal has not responded to the access-control contract", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "cut custom branding", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "responsive in v1", + "matched": true + }, + "D4_option_b": { + "keyword": null, + "matched": false + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "mid-july", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "v1.5", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "3/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 463, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "tracts:** legal has not responded to the access-control contract draft; no slack in timeline (weeks 5–6). 3. **medium — panel-density limit:** 40 panels/page cap ships in v1 as a docum", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1294, + "matched_span": "legal has not responded to the access-control contract", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1240 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "a87b53af8b0cd6d07a34d8f7b5324a0e4f4757ccad1a6de504311ef54867b75b", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "156f12e41eb0b75fc15f918ea92211e65b1cc23a4004985935cac83beab6ba5e", + "identity": { + "arm": "qwen38q8-nothink-matched", + "mode": "no-think", + "model": "3.8", + "quant": "Q8_0", + "regime": "T0.3/p0.8/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-q38-q8", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v1/grade.corrected.json new file mode 100644 index 00000000..506223cb --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v1/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-nothink-offspec_v1", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.767, + "urgency_accuracy_original": 0.667, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "0f3580cbf074331e9a5ffb34f3bf99bf3506d63f98fcc245811369d9e4055ac4", + "identity": { + "arm": "qwen36-nothink-offspec", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v2/grade.corrected.json new file mode 100644 index 00000000..b910f36f --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v2/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-nothink-offspec_v2", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.767, + "urgency_accuracy_original": 0.667, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "0f3580cbf074331e9a5ffb34f3bf99bf3506d63f98fcc245811369d9e4055ac4", + "identity": { + "arm": "qwen36-nothink-offspec", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v3/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v3/grade.corrected.json new file mode 100644 index 00000000..3189af98 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v3/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-nothink-offspec_v3", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.767, + "urgency_accuracy_original": 0.667, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "0f3580cbf074331e9a5ffb34f3bf99bf3506d63f98fcc245811369d9e4055ac4", + "identity": { + "arm": "qwen36-nothink-offspec", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "3", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v4/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v4/grade.corrected.json new file mode 100644 index 00000000..e0159554 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v4/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-nothink-offspec_v4", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.767, + "urgency_accuracy_original": 0.667, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "0f3580cbf074331e9a5ffb34f3bf99bf3506d63f98fcc245811369d9e4055ac4", + "identity": { + "arm": "qwen36-nothink-offspec", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "4", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v5/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v5/grade.corrected.json new file mode 100644 index 00000000..3dd71008 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v5/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-nothink-offspec_v5", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.767, + "urgency_accuracy_original": 0.667, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "86ede5a9c74695233fa0c0f2dff6c8191e6141f261c33f78d58808c8b8434283", + "identity": { + "arm": "qwen36-nothink-offspec", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "5", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v6/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v6/grade.corrected.json new file mode 100644 index 00000000..8c483104 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v6/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-nothink-offspec_v6", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.767, + "urgency_accuracy_original": 0.667, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "0f3580cbf074331e9a5ffb34f3bf99bf3506d63f98fcc245811369d9e4055ac4", + "identity": { + "arm": "qwen36-nothink-offspec", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "6", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v7/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v7/grade.corrected.json new file mode 100644 index 00000000..bc8729fb --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v7/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-nothink-offspec_v7", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.767, + "urgency_accuracy_original": 0.667, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "0f3580cbf074331e9a5ffb34f3bf99bf3506d63f98fcc245811369d9e4055ac4", + "identity": { + "arm": "qwen36-nothink-offspec", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "7", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v8/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v8/grade.corrected.json new file mode 100644 index 00000000..61b8e677 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v8/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-nothink-offspec_v8", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.767, + "urgency_accuracy_original": 0.667, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "0f3580cbf074331e9a5ffb34f3bf99bf3506d63f98fcc245811369d9e4055ac4", + "identity": { + "arm": "qwen36-nothink-offspec", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "8", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v9/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v9/grade.corrected.json new file mode 100644 index 00000000..01a58536 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-nothink-offspec_v9/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-nothink-offspec_v9", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.767, + "urgency_accuracy_original": 0.667, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "0f3580cbf074331e9a5ffb34f3bf99bf3506d63f98fcc245811369d9e4055ac4", + "identity": { + "arm": "qwen36-nothink-offspec", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "9", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v1/grade.corrected.json new file mode 100644 index 00000000..3f980d7c --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v1/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-think-offspec_v1", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.733, + "urgency_accuracy_original": 0.633, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "d0eae3b83c4bd050507079ea91a3ef00335decf484cdf166708fc3ed1a0c5e6d", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v2/grade.corrected.json new file mode 100644 index 00000000..6cc29db5 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v2/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-think-offspec_v2", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.733, + "urgency_accuracy_original": 0.633, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "d0eae3b83c4bd050507079ea91a3ef00335decf484cdf166708fc3ed1a0c5e6d", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v3/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v3/grade.corrected.json new file mode 100644 index 00000000..f164ef83 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v3/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-think-offspec_v3", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.733, + "urgency_accuracy_original": 0.633, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "d0eae3b83c4bd050507079ea91a3ef00335decf484cdf166708fc3ed1a0c5e6d", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "3", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v4/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v4/grade.corrected.json new file mode 100644 index 00000000..5b5982c5 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v4/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-think-offspec_v4", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.733, + "urgency_accuracy_original": 0.633, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "d0eae3b83c4bd050507079ea91a3ef00335decf484cdf166708fc3ed1a0c5e6d", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "4", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v5/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v5/grade.corrected.json new file mode 100644 index 00000000..83f79c1f --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v5/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-think-offspec_v5", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.733, + "urgency_accuracy_original": 0.633, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "d0eae3b83c4bd050507079ea91a3ef00335decf484cdf166708fc3ed1a0c5e6d", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "5", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v6/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v6/grade.corrected.json new file mode 100644 index 00000000..8545e0f0 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v6/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-think-offspec_v6", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.733, + "urgency_accuracy_original": 0.633, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "d0eae3b83c4bd050507079ea91a3ef00335decf484cdf166708fc3ed1a0c5e6d", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "6", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v7/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v7/grade.corrected.json new file mode 100644 index 00000000..d01772fa --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v7/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-think-offspec_v7", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.733, + "urgency_accuracy_original": 0.633, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "d0eae3b83c4bd050507079ea91a3ef00335decf484cdf166708fc3ed1a0c5e6d", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "7", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v8/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v8/grade.corrected.json new file mode 100644 index 00000000..c8a5cc04 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v8/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-think-offspec_v8", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.733, + "urgency_accuracy_original": 0.633, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "d0eae3b83c4bd050507079ea91a3ef00335decf484cdf166708fc3ed1a0c5e6d", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "8", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v9/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v9/grade.corrected.json new file mode 100644 index 00000000..b0015cea --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p2_triage_qwen36-think-offspec_v9/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen36-think-offspec_v9", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.733, + "urgency_accuracy_original": 0.633, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "d0eae3b83c4bd050507079ea91a3ef00335decf484cdf166708fc3ed1a0c5e6d", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "9", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_business_qwen36-nothink-offspec_v4/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_business_qwen36-nothink-offspec_v4/grade.corrected.json new file mode 100644 index 00000000..0e4d869d --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_business_qwen36-nothink-offspec_v4/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_business_qwen36-nothink-offspec_v4", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_business_memo_grade.py:115", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4513, + "ceiling": 700, + "counter_delta": 20, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 719, + "over_ceiling_by_grader_regex": true, + "relname": "memo.md", + "sha256": "345622b4486b1912b2da908928d7dc917a17f847767c34e5bce9e21698991b99", + "stored_grader_count": 719, + "wc_w_count": 699, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 699, + "wc_w_shell": 699, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 9, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_business", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "08e227529014cdd78f9aaae9468489251e4350a0497c4d64f5893cef9eb7099a", + "identity": { + "arm": "qwen36-nothink-offspec", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "4", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "business_memo", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_business_qwen36-think-offspec_v7/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_business_qwen36-think-offspec_v7/grade.corrected.json new file mode 100644 index 00000000..a4589138 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_business_qwen36-think-offspec_v7/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_business_qwen36-think-offspec_v7", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_business_memo_grade.py:115", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4498, + "ceiling": 700, + "counter_delta": 3, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 703, + "over_ceiling_by_grader_regex": true, + "relname": "memo.md", + "sha256": "2f8cfa9ade7a5735167998aa6542161e349558c42d9b7faf791f0c7c0d48e578", + "stored_grader_count": 703, + "wc_w_count": 700, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 700, + "wc_w_shell": 700, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 32, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_business", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "1c75e34fe287c130b65ed732db470a8364a5918461086977484e226fb8983e60", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "7", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "business_memo", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-nothink-offspec_v7/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-nothink-offspec_v7/grade.corrected.json new file mode 100644 index 00000000..b1eca484 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-nothink-offspec_v7/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen36-nothink-offspec_v7", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4661, + "ceiling": 700, + "counter_delta": 17, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 713, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "7df955c238dab50838cc0dd7f061633993de042bd13295de8ea0583ae94a0cda", + "stored_grader_count": 713, + "wc_w_count": 696, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 696, + "wc_w_shell": 696, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 3, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "72b5a8e8a9227741a9427a644192b93db135fa6fc81ee08cb72529496c82fcc1", + "identity": { + "arm": "qwen36-nothink-offspec", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "7", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v1/grade.corrected.json new file mode 100644 index 00000000..8888cc62 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v1/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen36-think-offspec_v1", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4630, + "ceiling": 700, + "counter_delta": 9, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 707, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "a93d118ff87bb4129f22d17fc0fdeaad2ba9bccb6f86d4a42675e6fbe449db1b", + "stored_grader_count": 707, + "wc_w_count": 698, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 698, + "wc_w_shell": 698, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "48552c3e52db827b7945ef8424d472ca21bf8f94352fded810288991d4d83e2a", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v2/grade.corrected.json new file mode 100644 index 00000000..30c71ef9 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v2/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen36-think-offspec_v2", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4630, + "ceiling": 700, + "counter_delta": 9, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 707, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "a93d118ff87bb4129f22d17fc0fdeaad2ba9bccb6f86d4a42675e6fbe449db1b", + "stored_grader_count": 707, + "wc_w_count": 698, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 698, + "wc_w_shell": 698, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "48552c3e52db827b7945ef8424d472ca21bf8f94352fded810288991d4d83e2a", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v3/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v3/grade.corrected.json new file mode 100644 index 00000000..35e39481 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v3/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen36-think-offspec_v3", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4630, + "ceiling": 700, + "counter_delta": 9, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 707, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "a93d118ff87bb4129f22d17fc0fdeaad2ba9bccb6f86d4a42675e6fbe449db1b", + "stored_grader_count": 707, + "wc_w_count": 698, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 698, + "wc_w_shell": 698, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 1, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "48552c3e52db827b7945ef8424d472ca21bf8f94352fded810288991d4d83e2a", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "3", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v4/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v4/grade.corrected.json new file mode 100644 index 00000000..eaf45446 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v4/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen36-think-offspec_v4", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4630, + "ceiling": 700, + "counter_delta": 9, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 707, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "a93d118ff87bb4129f22d17fc0fdeaad2ba9bccb6f86d4a42675e6fbe449db1b", + "stored_grader_count": 707, + "wc_w_count": 698, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 698, + "wc_w_shell": 698, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "48552c3e52db827b7945ef8424d472ca21bf8f94352fded810288991d4d83e2a", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "4", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v5/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v5/grade.corrected.json new file mode 100644 index 00000000..a4106fda --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v5/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen36-think-offspec_v5", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4630, + "ceiling": 700, + "counter_delta": 9, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 707, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "a93d118ff87bb4129f22d17fc0fdeaad2ba9bccb6f86d4a42675e6fbe449db1b", + "stored_grader_count": 707, + "wc_w_count": 698, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 698, + "wc_w_shell": 698, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "48552c3e52db827b7945ef8424d472ca21bf8f94352fded810288991d4d83e2a", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "5", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v6/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v6/grade.corrected.json new file mode 100644 index 00000000..50122a93 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v6/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen36-think-offspec_v6", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4655, + "ceiling": 700, + "counter_delta": 7, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 707, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "4417ba04da4057334858cacdd8f22abf936bf49d5357dbb28f94cf9d0a67acf5", + "stored_grader_count": 707, + "wc_w_count": 700, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 700, + "wc_w_shell": 700, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 7, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "7b1e50f9e2e5564a7fe762c0fb4aeb99bcd725fedf36ba824aafa6c807975962", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "6", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v7/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v7/grade.corrected.json new file mode 100644 index 00000000..41489602 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v7/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen36-think-offspec_v7", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4630, + "ceiling": 700, + "counter_delta": 9, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 707, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "a93d118ff87bb4129f22d17fc0fdeaad2ba9bccb6f86d4a42675e6fbe449db1b", + "stored_grader_count": 707, + "wc_w_count": 698, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 698, + "wc_w_shell": 698, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "48552c3e52db827b7945ef8424d472ca21bf8f94352fded810288991d4d83e2a", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "7", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v8/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v8/grade.corrected.json new file mode 100644 index 00000000..bde47160 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v8/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen36-think-offspec_v8", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4630, + "ceiling": 700, + "counter_delta": 9, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 707, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "a93d118ff87bb4129f22d17fc0fdeaad2ba9bccb6f86d4a42675e6fbe449db1b", + "stored_grader_count": 707, + "wc_w_count": 698, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 698, + "wc_w_shell": 698, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 1, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "48552c3e52db827b7945ef8424d472ca21bf8f94352fded810288991d4d83e2a", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "8", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v9/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v9/grade.corrected.json new file mode 100644 index 00000000..f4e3a43b --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_doc_qwen36-think-offspec_v9/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen36-think-offspec_v9", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4703, + "ceiling": 700, + "counter_delta": 11, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 709, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "abe1191ea1a2539bf6a4ad17ce5dedf57cdf6e2eb2bce464f023426d1fe2b65d", + "stored_grader_count": 709, + "wc_w_count": 698, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 698, + "wc_w_shell": 698, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 5, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "8e9b94fac808d17f8f99d0bf266edd5284c4ab9546fce5fd310699fdddf7bcba", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "9", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v1/grade.corrected.json new file mode 100644 index 00000000..f7035dda --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v1/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-nothink-offspec_v1", + "corrected_verdict": "FAIL", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2860, + "ceiling": 700, + "counter_delta": -87, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 413, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "c9ccc89b380e4398415378f41595e486042e1027e674253402d58e66c78e2cd5", + "stored_grader_count": 413, + "wc_w_count": 500, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 500, + "wc_w_shell": 500, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal) | sara | draft sent; no response (wk6) | **yes** (sdk portion) | ## risks | severity | risk | description | |---|---|---| | **high** | legal delay", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "branding deferred", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": null, + "matched": false + }, + "D4_option_b": { + "keyword": null, + "matched": false + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "july 2026", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "2/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 413, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "ontracts (legal) | sara | draft sent; no response (wk6) | **yes** (sdk portion) | ## risks | severity | risk | description | |---|---|---| | **high** | legal delay on beta contracts | no response to draft; blocks sdk beta onboarding (wk6). | | **high** | maevi", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1131, + "matched_span": "legal) | sara | draft sent; no response (wk6) | **yes** (sdk portion) | ## risks | severity | risk | description | |---|---|---| | **high** | legal delay", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 978 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "c9ccc89b380e4398415378f41595e486042e1027e674253402d58e66c78e2cd5", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "FAIL", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "a169814791174ca8f5c6d9acc6c3c2622d1e20396d47f956019276896ef0f20c", + "identity": { + "arm": "qwen36-nothink-offspec", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v2/grade.corrected.json new file mode 100644 index 00000000..4ad1d882 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v2/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-nothink-offspec_v2", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2780, + "ceiling": 700, + "counter_delta": -62, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 411, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "ab03f0be6e27ced8ddcca6436180c22b8dfa7e8b726fd8d3f3c25d2c757440bd", + "stored_grader_count": 411, + "wc_w_count": 473, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 473, + "wc_w_shell": 473, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal contract delay | private-beta contracts drafted; legal has not", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "branding deferred", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native in v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "mid-july", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom-branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query-layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 411, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "*high** | legal contract delay | private-beta contracts drafted; legal has not responded (week 6). blocks onboarding. | | **medium** | tight mid-july timeline | query-layer and mobile both due mid-m", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1257, + "matched_span": "legal contract delay | private-beta contracts drafted; legal has not", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1189 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "ab03f0be6e27ced8ddcca6436180c22b8dfa7e8b726fd8d3f3c25d2c757440bd", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "e97186194bd700d6f3f9796f73d45c4e0937d659ac8bd3f2b5100cefd6f38d69", + "identity": { + "arm": "qwen36-nothink-offspec", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v3/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v3/grade.corrected.json new file mode 100644 index 00000000..e3980c72 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v3/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-nothink-offspec_v3", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2780, + "ceiling": 700, + "counter_delta": -62, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 411, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "ab03f0be6e27ced8ddcca6436180c22b8dfa7e8b726fd8d3f3c25d2c757440bd", + "stored_grader_count": 411, + "wc_w_count": 473, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 473, + "wc_w_shell": 473, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal contract delay | private-beta contracts drafted; legal has not", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "branding deferred", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native in v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "mid-july", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom-branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query-layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 411, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "*high** | legal contract delay | private-beta contracts drafted; legal has not responded (week 6). blocks onboarding. | | **medium** | tight mid-july timeline | query-layer and mobile both due mid-m", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1257, + "matched_span": "legal contract delay | private-beta contracts drafted; legal has not", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1189 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "ab03f0be6e27ced8ddcca6436180c22b8dfa7e8b726fd8d3f3c25d2c757440bd", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "e97186194bd700d6f3f9796f73d45c4e0937d659ac8bd3f2b5100cefd6f38d69", + "identity": { + "arm": "qwen36-nothink-offspec", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "3", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v5/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v5/grade.corrected.json new file mode 100644 index 00000000..84303c03 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v5/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-nothink-offspec_v5", + "corrected_verdict": "FAIL", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2863, + "ceiling": 700, + "counter_delta": -86, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 414, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "622f125432a5b53986f7192a9986b379811f0ee7125180c5809c0980c0f60696", + "stored_grader_count": 414, + "wc_w_count": 500, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 500, + "wc_w_shell": 500, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal) | sara | draft sent; no response (wk6) | **yes** (sdk portion) | ## risks | severity | risk | description | |---|---|---| | **high** | legal delay", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "branding deferred", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": null, + "matched": false + }, + "D4_option_b": { + "keyword": null, + "matched": false + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "july 2026", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "2/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 414, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "ontracts (legal) | sara | draft sent; no response (wk6) | **yes** (sdk portion) | ## risks | severity | risk | description | |---|---|---| | **high** | legal delay on beta contracts | no response to draft; blocks sdk beta onboarding (wk6). | | **high** | maevi", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1134, + "matched_span": "legal) | sara | draft sent; no response (wk6) | **yes** (sdk portion) | ## risks | severity | risk | description | |---|---|---| | **high** | legal delay", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 981 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "622f125432a5b53986f7192a9986b379811f0ee7125180c5809c0980c0f60696", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "FAIL", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "5487e14e1e293c33f7271bb07e55964a6606a7d999d02f652938b70ac860d957", + "identity": { + "arm": "qwen36-nothink-offspec", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "5", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v6/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v6/grade.corrected.json new file mode 100644 index 00000000..67eccff9 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v6/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-nothink-offspec_v6", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2780, + "ceiling": 700, + "counter_delta": -62, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 411, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "ab03f0be6e27ced8ddcca6436180c22b8dfa7e8b726fd8d3f3c25d2c757440bd", + "stored_grader_count": 411, + "wc_w_count": 473, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 473, + "wc_w_shell": 473, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal contract delay | private-beta contracts drafted; legal has not", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "branding deferred", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native in v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "mid-july", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom-branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query-layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 411, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "*high** | legal contract delay | private-beta contracts drafted; legal has not responded (week 6). blocks onboarding. | | **medium** | tight mid-july timeline | query-layer and mobile both due mid-m", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1257, + "matched_span": "legal contract delay | private-beta contracts drafted; legal has not", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1189 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "ab03f0be6e27ced8ddcca6436180c22b8dfa7e8b726fd8d3f3c25d2c757440bd", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "e97186194bd700d6f3f9796f73d45c4e0937d659ac8bd3f2b5100cefd6f38d69", + "identity": { + "arm": "qwen36-nothink-offspec", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "6", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v7/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v7/grade.corrected.json new file mode 100644 index 00000000..b18c2e87 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v7/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-nothink-offspec_v7", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2870, + "ceiling": 700, + "counter_delta": -62, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 426, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "aecf8aa9e308a17407c2141d99b58a051001cd845358a76525788d13caf188df", + "stored_grader_count": 426, + "wc_w_count": 488, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 488, + "wc_w_shell": 488, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal contract delay | private-beta contracts drafted but legal has not", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "branding deferred", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native in v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "mid-july", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom-branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query-layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 426, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "*high** | legal contract delay | private-beta contracts drafted but legal has not responded (week 6). blocks onboarding. | | **medium** | tight mid-july timeline | query-layer and mobile both due mid-m", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1260, + "matched_span": "legal contract delay | private-beta contracts drafted but legal has not", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1189 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "aecf8aa9e308a17407c2141d99b58a051001cd845358a76525788d13caf188df", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "2b32d677a08a134a8cbec9aadcba8bb88690a4d7c13ae0345f638e0c0d5ae8e8", + "identity": { + "arm": "qwen36-nothink-offspec", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "7", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v8/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v8/grade.corrected.json new file mode 100644 index 00000000..70b15db1 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v8/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-nothink-offspec_v8", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2780, + "ceiling": 700, + "counter_delta": -62, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 411, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "ab03f0be6e27ced8ddcca6436180c22b8dfa7e8b726fd8d3f3c25d2c757440bd", + "stored_grader_count": 411, + "wc_w_count": 473, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 473, + "wc_w_shell": 473, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal contract delay | private-beta contracts drafted; legal has not", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "branding deferred", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native in v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "mid-july", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom-branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query-layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 411, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "*high** | legal contract delay | private-beta contracts drafted; legal has not responded (week 6). blocks onboarding. | | **medium** | tight mid-july timeline | query-layer and mobile both due mid-m", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1257, + "matched_span": "legal contract delay | private-beta contracts drafted; legal has not", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1189 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "ab03f0be6e27ced8ddcca6436180c22b8dfa7e8b726fd8d3f3c25d2c757440bd", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "e97186194bd700d6f3f9796f73d45c4e0937d659ac8bd3f2b5100cefd6f38d69", + "identity": { + "arm": "qwen36-nothink-offspec", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "8", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v9/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v9/grade.corrected.json new file mode 100644 index 00000000..ebbd1b05 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-nothink-offspec_v9/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-nothink-offspec_v9", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2780, + "ceiling": 700, + "counter_delta": -62, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 411, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "ab03f0be6e27ced8ddcca6436180c22b8dfa7e8b726fd8d3f3c25d2c757440bd", + "stored_grader_count": 411, + "wc_w_count": 473, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 473, + "wc_w_shell": 473, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal contract delay | private-beta contracts drafted; legal has not", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "branding deferred", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native in v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "mid-july", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom-branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query-layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 411, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "*high** | legal contract delay | private-beta contracts drafted; legal has not responded (week 6). blocks onboarding. | | **medium** | tight mid-july timeline | query-layer and mobile both due mid-m", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1257, + "matched_span": "legal contract delay | private-beta contracts drafted; legal has not", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1189 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "ab03f0be6e27ced8ddcca6436180c22b8dfa7e8b726fd8d3f3c25d2c757440bd", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "e97186194bd700d6f3f9796f73d45c4e0937d659ac8bd3f2b5100cefd6f38d69", + "identity": { + "arm": "qwen36-nothink-offspec", + "mode": "no-think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "9", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v1/grade.corrected.json new file mode 100644 index 00000000..b9fd4229 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v1/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-think-offspec_v1", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2756, + "ceiling": 700, + "counter_delta": -85, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 411, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "aa7dff14b6a949490fedde5729071f23b954ec9292633232c80dca15513610e9", + "stored_grader_count": 411, + "wc_w_count": 496, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 496, + "wc_w_shell": 496, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 1, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal delay on private-beta contracts | legal has not responded to contract", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "cut custom branding", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native in v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "july 2026", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "architectural fix", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 411, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "| high | legal delay on private-beta contracts | legal has not responded to contract draft (week 6). blocks private-beta launch. | | high | maevia expectation mismatch | maevia was promised ga sdk at the", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1100, + "matched_span": "legal delay on private-beta contracts | legal has not responded to contract", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1025 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "aa7dff14b6a949490fedde5729071f23b954ec9292633232c80dca15513610e9", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "8d0adc16202c63985aa4b27f3f97f37378c5aea96695381d33f9a4f9ce27c921", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v2/grade.corrected.json new file mode 100644 index 00000000..8de27553 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v2/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-think-offspec_v2", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2756, + "ceiling": 700, + "counter_delta": -85, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 411, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "aa7dff14b6a949490fedde5729071f23b954ec9292633232c80dca15513610e9", + "stored_grader_count": 411, + "wc_w_count": 496, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 496, + "wc_w_shell": 496, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 1, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal delay on private-beta contracts | legal has not responded to contract", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "cut custom branding", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native in v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "july 2026", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "architectural fix", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 411, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "| high | legal delay on private-beta contracts | legal has not responded to contract draft (week 6). blocks private-beta launch. | | high | maevia expectation mismatch | maevia was promised ga sdk at the", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1100, + "matched_span": "legal delay on private-beta contracts | legal has not responded to contract", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1025 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "aa7dff14b6a949490fedde5729071f23b954ec9292633232c80dca15513610e9", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "8d0adc16202c63985aa4b27f3f97f37378c5aea96695381d33f9a4f9ce27c921", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v3/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v3/grade.corrected.json new file mode 100644 index 00000000..4ac26760 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v3/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-think-offspec_v3", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2756, + "ceiling": 700, + "counter_delta": -85, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 411, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "aa7dff14b6a949490fedde5729071f23b954ec9292633232c80dca15513610e9", + "stored_grader_count": 411, + "wc_w_count": 496, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 496, + "wc_w_shell": 496, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 1, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal delay on private-beta contracts | legal has not responded to contract", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "cut custom branding", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native in v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "july 2026", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "architectural fix", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 411, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "| high | legal delay on private-beta contracts | legal has not responded to contract draft (week 6). blocks private-beta launch. | | high | maevia expectation mismatch | maevia was promised ga sdk at the", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1100, + "matched_span": "legal delay on private-beta contracts | legal has not responded to contract", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1025 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "aa7dff14b6a949490fedde5729071f23b954ec9292633232c80dca15513610e9", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "8d0adc16202c63985aa4b27f3f97f37378c5aea96695381d33f9a4f9ce27c921", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "3", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v4/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v4/grade.corrected.json new file mode 100644 index 00000000..ce8fae99 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v4/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-think-offspec_v4", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2756, + "ceiling": 700, + "counter_delta": -85, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 411, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "aa7dff14b6a949490fedde5729071f23b954ec9292633232c80dca15513610e9", + "stored_grader_count": 411, + "wc_w_count": 496, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 496, + "wc_w_shell": 496, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 1, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal delay on private-beta contracts | legal has not responded to contract", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "cut custom branding", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native in v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "july 2026", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "architectural fix", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 411, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "| high | legal delay on private-beta contracts | legal has not responded to contract draft (week 6). blocks private-beta launch. | | high | maevia expectation mismatch | maevia was promised ga sdk at the", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1100, + "matched_span": "legal delay on private-beta contracts | legal has not responded to contract", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1025 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "aa7dff14b6a949490fedde5729071f23b954ec9292633232c80dca15513610e9", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "8d0adc16202c63985aa4b27f3f97f37378c5aea96695381d33f9a4f9ce27c921", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "4", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v5/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v5/grade.corrected.json new file mode 100644 index 00000000..0a673ab3 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v5/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-think-offspec_v5", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2756, + "ceiling": 700, + "counter_delta": -85, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 411, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "aa7dff14b6a949490fedde5729071f23b954ec9292633232c80dca15513610e9", + "stored_grader_count": 411, + "wc_w_count": 496, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 496, + "wc_w_shell": 496, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 1, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal delay on private-beta contracts | legal has not responded to contract", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "cut custom branding", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native in v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "july 2026", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "architectural fix", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 411, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "| high | legal delay on private-beta contracts | legal has not responded to contract draft (week 6). blocks private-beta launch. | | high | maevia expectation mismatch | maevia was promised ga sdk at the", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1100, + "matched_span": "legal delay on private-beta contracts | legal has not responded to contract", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1025 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "aa7dff14b6a949490fedde5729071f23b954ec9292633232c80dca15513610e9", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "8d0adc16202c63985aa4b27f3f97f37378c5aea96695381d33f9a4f9ce27c921", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "5", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v6/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v6/grade.corrected.json new file mode 100644 index 00000000..946a65ab --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v6/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-think-offspec_v6", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2673, + "ceiling": 700, + "counter_delta": -93, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 398, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "67b9aa697c47abeb857d239cc51aa9f7f7a3db46354abd9b4e9b63398e12a8d6", + "stored_grader_count": 398, + "wc_w_count": 491, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 491, + "wc_w_shell": 491, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal delay on private-beta contracts | legal has not", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "cut custom branding", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native in v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "july 2026", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "architectural fix", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 398, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "| high | legal delay on private-beta contracts | legal has not responded to draft (wk 6). blocks sdk launch. | | high | maevia expectation mismatch | promised ga sdk at conference; w", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1109, + "matched_span": "legal delay on private-beta contracts | legal has not", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1056 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "67b9aa697c47abeb857d239cc51aa9f7f7a3db46354abd9b4e9b63398e12a8d6", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "2c69cc5015d2a51eb94643dc5ec6a6cdaae517f675653dbd3ee6a4ce6f888cdc", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "6", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v7/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v7/grade.corrected.json new file mode 100644 index 00000000..e2019d4c --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v7/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-think-offspec_v7", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2756, + "ceiling": 700, + "counter_delta": -85, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 411, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "aa7dff14b6a949490fedde5729071f23b954ec9292633232c80dca15513610e9", + "stored_grader_count": 411, + "wc_w_count": 496, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 496, + "wc_w_shell": 496, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 1, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal delay on private-beta contracts | legal has not responded to contract", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "cut custom branding", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native in v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "july 2026", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "architectural fix", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 411, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "| high | legal delay on private-beta contracts | legal has not responded to contract draft (week 6). blocks private-beta launch. | | high | maevia expectation mismatch | maevia was promised ga sdk at the", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1100, + "matched_span": "legal delay on private-beta contracts | legal has not responded to contract", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1025 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "aa7dff14b6a949490fedde5729071f23b954ec9292633232c80dca15513610e9", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "8d0adc16202c63985aa4b27f3f97f37378c5aea96695381d33f9a4f9ce27c921", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "7", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v8/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v8/grade.corrected.json new file mode 100644 index 00000000..39bee9a4 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v8/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-think-offspec_v8", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2796, + "ceiling": 700, + "counter_delta": -82, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 417, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "46575ce1e8343385807aab7ceed9ef7afc938cd105caf1b9824066d224328647", + "stored_grader_count": 417, + "wc_w_count": 499, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 499, + "wc_w_shell": 499, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 3, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal has not responded to the private-beta contract", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "cut custom-branding", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native in v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": null, + "matched": false + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom-branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "architectural fix", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "4/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 417, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "ttention: legal has not responded to the private-beta contract draft, and customer maevia was promised ga sdk at the conference. ## workstreams | workstream | owner | status | v1-blo", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 303, + "matched_span": "legal has not responded to the private-beta contract", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 251 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "46575ce1e8343385807aab7ceed9ef7afc938cd105caf1b9824066d224328647", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "7ce5d8f00250b40628a5dcfbb5f64caa6705e432d491c4dfe613c70fb10cd155", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "8", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v9/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v9/grade.corrected.json new file mode 100644 index 00000000..49fcda7d --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_pm_qwen36-think-offspec_v9/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen36-think-offspec_v9", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2756, + "ceiling": 700, + "counter_delta": -85, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 411, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "aa7dff14b6a949490fedde5729071f23b954ec9292633232c80dca15513610e9", + "stored_grader_count": 411, + "wc_w_count": 496, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 496, + "wc_w_shell": 496, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 1, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal delay on private-beta contracts | legal has not responded to contract", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "cut custom branding", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native in v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "july 2026", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "architectural fix", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 411, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "| high | legal delay on private-beta contracts | legal has not responded to contract draft (week 6). blocks private-beta launch. | | high | maevia expectation mismatch | maevia was promised ga sdk at the", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1100, + "matched_span": "legal delay on private-beta contracts | legal has not responded to contract", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1025 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "aa7dff14b6a949490fedde5729071f23b954ec9292633232c80dca15513610e9", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "8d0adc16202c63985aa4b27f3f97f37378c5aea96695381d33f9a4f9ce27c921", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "9", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_writing_qwen36-think-offspec_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_writing_qwen36-think-offspec_v2/grade.corrected.json new file mode 100644 index 00000000..9183e9c2 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_writing_qwen36-think-offspec_v2/grade.corrected.json @@ -0,0 +1,107 @@ +{ + "cell": "p3_writing_qwen36-think-offspec_v2", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + "length_attributable_fail": true, + "measurements": [ + { + "audience": "ceo_brief", + "bytes": 1748, + "ceiling": 250, + "counter_delta": 10, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 252, + "over_ceiling_by_grader_regex": true, + "relname": "ceo_brief.md", + "sha256": "33d020f9bb284860a0ee16aee2bb262681d430e312a60a3c2a95a41fcaf42fc8", + "stored_grader_count": 252, + "wc_w_count": 242, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 242, + "wc_w_shell": 242, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "customer_email", + "bytes": 1668, + "ceiling": 350, + "counter_delta": 1, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 259, + "over_ceiling_by_grader_regex": false, + "relname": "customer_email.md", + "sha256": "ec4c0d04ebdf76a2416aab1dea867cf48f2b0e707cdba35a16569f9133399a4c", + "stored_grader_count": 259, + "wc_w_count": 258, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 258, + "wc_w_shell": 258, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "legal_summary", + "bytes": 2220, + "ceiling": 400, + "counter_delta": 15, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 321, + "over_ceiling_by_grader_regex": false, + "relname": "legal_summary.md", + "sha256": "12eeb6ccb43bf8c9d373d12bfe50bd8916b225c7b81a56e448158d906b0df9e3", + "stored_grader_count": 321, + "wc_w_count": 306, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 306, + "wc_w_shell": 306, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 4, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_writing", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "d5b57054859692b68a2fc8f9b35241442cc56cec5b1fa3557e17dcc770ae1699", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "writing_editing", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_writing_qwen36-think-offspec_v5/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_writing_qwen36-think-offspec_v5/grade.corrected.json new file mode 100644 index 00000000..188910d0 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_writing_qwen36-think-offspec_v5/grade.corrected.json @@ -0,0 +1,107 @@ +{ + "cell": "p3_writing_qwen36-think-offspec_v5", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + "length_attributable_fail": true, + "measurements": [ + { + "audience": "ceo_brief", + "bytes": 1748, + "ceiling": 250, + "counter_delta": 10, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 252, + "over_ceiling_by_grader_regex": true, + "relname": "ceo_brief.md", + "sha256": "33d020f9bb284860a0ee16aee2bb262681d430e312a60a3c2a95a41fcaf42fc8", + "stored_grader_count": 252, + "wc_w_count": 242, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 242, + "wc_w_shell": 242, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "customer_email", + "bytes": 1668, + "ceiling": 350, + "counter_delta": 1, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 259, + "over_ceiling_by_grader_regex": false, + "relname": "customer_email.md", + "sha256": "ec4c0d04ebdf76a2416aab1dea867cf48f2b0e707cdba35a16569f9133399a4c", + "stored_grader_count": 259, + "wc_w_count": 258, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 258, + "wc_w_shell": 258, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "legal_summary", + "bytes": 2220, + "ceiling": 400, + "counter_delta": 15, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 321, + "over_ceiling_by_grader_regex": false, + "relname": "legal_summary.md", + "sha256": "12eeb6ccb43bf8c9d373d12bfe50bd8916b225c7b81a56e448158d906b0df9e3", + "stored_grader_count": 321, + "wc_w_count": 306, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 306, + "wc_w_shell": 306, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 6, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_writing", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "d5b57054859692b68a2fc8f9b35241442cc56cec5b1fa3557e17dcc770ae1699", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "5", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "writing_editing", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_writing_qwen36-think-offspec_v8/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_writing_qwen36-think-offspec_v8/grade.corrected.json new file mode 100644 index 00000000..8b0d5968 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_writing_qwen36-think-offspec_v8/grade.corrected.json @@ -0,0 +1,107 @@ +{ + "cell": "p3_writing_qwen36-think-offspec_v8", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + "length_attributable_fail": true, + "measurements": [ + { + "audience": "ceo_brief", + "bytes": 1748, + "ceiling": 250, + "counter_delta": 10, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 252, + "over_ceiling_by_grader_regex": true, + "relname": "ceo_brief.md", + "sha256": "33d020f9bb284860a0ee16aee2bb262681d430e312a60a3c2a95a41fcaf42fc8", + "stored_grader_count": 252, + "wc_w_count": 242, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 242, + "wc_w_shell": 242, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "customer_email", + "bytes": 1668, + "ceiling": 350, + "counter_delta": 1, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 259, + "over_ceiling_by_grader_regex": false, + "relname": "customer_email.md", + "sha256": "ec4c0d04ebdf76a2416aab1dea867cf48f2b0e707cdba35a16569f9133399a4c", + "stored_grader_count": 259, + "wc_w_count": 258, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 258, + "wc_w_shell": 258, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "legal_summary", + "bytes": 2220, + "ceiling": 400, + "counter_delta": 15, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 321, + "over_ceiling_by_grader_regex": false, + "relname": "legal_summary.md", + "sha256": "12eeb6ccb43bf8c9d373d12bfe50bd8916b225c7b81a56e448158d906b0df9e3", + "stored_grader_count": 321, + "wc_w_count": 306, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 306, + "wc_w_shell": 306, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 6, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_writing", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "d5b57054859692b68a2fc8f9b35241442cc56cec5b1fa3557e17dcc770ae1699", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "8", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "writing_editing", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_writing_qwen36-think-offspec_v9/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_writing_qwen36-think-offspec_v9/grade.corrected.json new file mode 100644 index 00000000..c0faf374 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen36-compare/logs/p3_writing_qwen36-think-offspec_v9/grade.corrected.json @@ -0,0 +1,107 @@ +{ + "cell": "p3_writing_qwen36-think-offspec_v9", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + "length_attributable_fail": true, + "measurements": [ + { + "audience": "ceo_brief", + "bytes": 1748, + "ceiling": 250, + "counter_delta": 10, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 252, + "over_ceiling_by_grader_regex": true, + "relname": "ceo_brief.md", + "sha256": "33d020f9bb284860a0ee16aee2bb262681d430e312a60a3c2a95a41fcaf42fc8", + "stored_grader_count": 252, + "wc_w_count": 242, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 242, + "wc_w_shell": 242, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "customer_email", + "bytes": 1668, + "ceiling": 350, + "counter_delta": 1, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 259, + "over_ceiling_by_grader_regex": false, + "relname": "customer_email.md", + "sha256": "ec4c0d04ebdf76a2416aab1dea867cf48f2b0e707cdba35a16569f9133399a4c", + "stored_grader_count": 259, + "wc_w_count": 258, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 258, + "wc_w_shell": 258, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "legal_summary", + "bytes": 2220, + "ceiling": 400, + "counter_delta": 15, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 321, + "over_ceiling_by_grader_regex": false, + "relname": "legal_summary.md", + "sha256": "12eeb6ccb43bf8c9d373d12bfe50bd8916b225c7b81a56e448158d906b0df9e3", + "stored_grader_count": 321, + "wc_w_count": 306, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 306, + "wc_w_shell": 306, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 6, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_writing", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "d5b57054859692b68a2fc8f9b35241442cc56cec5b1fa3557e17dcc770ae1699", + "identity": { + "arm": "qwen36-think-offspec", + "mode": "think", + "model": "3.6", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "9", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen36-compare", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "writing_editing", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-27b-udq4xl-nothink_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-27b-udq4xl-nothink_v1/grade.corrected.json new file mode 100644 index 00000000..f8d007d8 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-27b-udq4xl-nothink_v1/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-27b-udq4xl-nothink_v1", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.767, + "urgency_accuracy_original": 0.667, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "a2d8c22b9998892b73b6d3be3d60f726d1ebee4569fbeba7b909ade45c2bd693", + "identity": { + "arm": "qwen38-27b-udq4xl-nothink", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.7/p0.8/pp1.5", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-27b-udq4xl-nothink_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-27b-udq4xl-nothink_v2/grade.corrected.json new file mode 100644 index 00000000..4e88d230 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-27b-udq4xl-nothink_v2/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-27b-udq4xl-nothink_v2", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.767, + "urgency_accuracy_original": 0.667, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "fd3dfa2ddfa92a944514ca7ec99aead62a51ba08f4ae5b8b4e802b2b89733bd7", + "identity": { + "arm": "qwen38-27b-udq4xl-nothink", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.7/p0.8/pp1.5", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-27b-udq4xl-nothink_v3/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-27b-udq4xl-nothink_v3/grade.corrected.json new file mode 100644 index 00000000..d925a338 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-27b-udq4xl-nothink_v3/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-27b-udq4xl-nothink_v3", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.833, + "urgency_accuracy_original": 0.733, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "5461c9cf0c5c1ac314577f21c9bab9e0eaf5e044db49f3948ad7d50564dfdee6", + "identity": { + "arm": "qwen38-27b-udq4xl-nothink", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.7/p0.8/pp1.5", + "replicate": "3", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-27b-udq4xl-think-xhigh_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-27b-udq4xl-think-xhigh_v1/grade.corrected.json new file mode 100644 index 00000000..79cab0a7 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-27b-udq4xl-think-xhigh_v1/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-27b-udq4xl-think-xhigh_v1", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.867, + "urgency_accuracy_original": 0.767, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "8f7506f3e196f54df6e53d719e8b98a5f6c8ccc20d00ad5a14f316721df53778", + "identity": { + "arm": "qwen38-27b-udq4xl-think-xhigh", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-27b-udq4xl-think-xhigh_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-27b-udq4xl-think-xhigh_v2/grade.corrected.json new file mode 100644 index 00000000..038cee5c --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-27b-udq4xl-think-xhigh_v2/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-27b-udq4xl-think-xhigh_v2", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.867, + "urgency_accuracy_original": 0.767, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "8f7506f3e196f54df6e53d719e8b98a5f6c8ccc20d00ad5a14f316721df53778", + "identity": { + "arm": "qwen38-27b-udq4xl-think-xhigh", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-27b-udq4xl-think-xhigh_v3/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-27b-udq4xl-think-xhigh_v3/grade.corrected.json new file mode 100644 index 00000000..1dd229a6 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-27b-udq4xl-think-xhigh_v3/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-27b-udq4xl-think-xhigh_v3", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.867, + "urgency_accuracy_original": 0.767, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "8f7506f3e196f54df6e53d719e8b98a5f6c8ccc20d00ad5a14f316721df53778", + "identity": { + "arm": "qwen38-27b-udq4xl-think-xhigh", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "3", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-nothink-offspec_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-nothink-offspec_v1/grade.corrected.json new file mode 100644 index 00000000..5948df3c --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-nothink-offspec_v1/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-nothink-offspec_v1", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.867, + "urgency_accuracy_original": 0.767, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "de0eafe375b45ecf68d3e8ef0378d8ec5fabe0c1907a55c27980abafbbb7ff31", + "identity": { + "arm": "qwen38-nothink-offspec", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-nothink-offspec_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-nothink-offspec_v2/grade.corrected.json new file mode 100644 index 00000000..553f1686 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-nothink-offspec_v2/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-nothink-offspec_v2", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.867, + "urgency_accuracy_original": 0.767, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "de0eafe375b45ecf68d3e8ef0378d8ec5fabe0c1907a55c27980abafbbb7ff31", + "identity": { + "arm": "qwen38-nothink-offspec", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-nothink-offspec_v3/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-nothink-offspec_v3/grade.corrected.json new file mode 100644 index 00000000..aee92074 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-nothink-offspec_v3/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-nothink-offspec_v3", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.867, + "urgency_accuracy_original": 0.767, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "de0eafe375b45ecf68d3e8ef0378d8ec5fabe0c1907a55c27980abafbbb7ff31", + "identity": { + "arm": "qwen38-nothink-offspec", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "3", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-nothink-offspec_v4/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-nothink-offspec_v4/grade.corrected.json new file mode 100644 index 00000000..90fa3e57 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-nothink-offspec_v4/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-nothink-offspec_v4", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.867, + "urgency_accuracy_original": 0.767, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "de0eafe375b45ecf68d3e8ef0378d8ec5fabe0c1907a55c27980abafbbb7ff31", + "identity": { + "arm": "qwen38-nothink-offspec", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "4", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-nothink-offspec_v5/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-nothink-offspec_v5/grade.corrected.json new file mode 100644 index 00000000..01c73aa0 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-nothink-offspec_v5/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-nothink-offspec_v5", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.867, + "urgency_accuracy_original": 0.767, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "de0eafe375b45ecf68d3e8ef0378d8ec5fabe0c1907a55c27980abafbbb7ff31", + "identity": { + "arm": "qwen38-nothink-offspec", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "5", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-nothink-offspec_v6/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-nothink-offspec_v6/grade.corrected.json new file mode 100644 index 00000000..41226c8a --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-nothink-offspec_v6/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-nothink-offspec_v6", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.867, + "urgency_accuracy_original": 0.767, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "de0eafe375b45ecf68d3e8ef0378d8ec5fabe0c1907a55c27980abafbbb7ff31", + "identity": { + "arm": "qwen38-nothink-offspec", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "6", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-nothink-offspec_v7/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-nothink-offspec_v7/grade.corrected.json new file mode 100644 index 00000000..e91e5e90 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-nothink-offspec_v7/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-nothink-offspec_v7", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.867, + "urgency_accuracy_original": 0.767, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "de0eafe375b45ecf68d3e8ef0378d8ec5fabe0c1907a55c27980abafbbb7ff31", + "identity": { + "arm": "qwen38-nothink-offspec", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "7", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-think-low-offspec_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-think-low-offspec_v1/grade.corrected.json new file mode 100644 index 00000000..e18454dc --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-think-low-offspec_v1/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-think-low-offspec_v1", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.833, + "urgency_accuracy_original": 0.733, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "436c89372168a5733c003ebdc5205e75a6588d55082de50c0f6e6f5429d51610", + "identity": { + "arm": "qwen38-think-low-offspec", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-think-medium-offspec_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-think-medium-offspec_v1/grade.corrected.json new file mode 100644 index 00000000..1d9e5065 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-think-medium-offspec_v1/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-think-medium-offspec_v1", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.867, + "urgency_accuracy_original": 0.767, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "0e0940ea885d2d528b31a7bf6d7f1f0b466847cd30fc73f17b011bffec1c7638", + "identity": { + "arm": "qwen38-think-medium-offspec", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-think-medium-offspec_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-think-medium-offspec_v2/grade.corrected.json new file mode 100644 index 00000000..ed07af92 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-think-medium-offspec_v2/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-think-medium-offspec_v2", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.867, + "urgency_accuracy_original": 0.767, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "0e0940ea885d2d528b31a7bf6d7f1f0b466847cd30fc73f17b011bffec1c7638", + "identity": { + "arm": "qwen38-think-medium-offspec", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-think-xhigh-offspec_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-think-xhigh-offspec_v1/grade.corrected.json new file mode 100644 index 00000000..ddbb0043 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p2_triage_qwen38-think-xhigh-offspec_v1/grade.corrected.json @@ -0,0 +1,99 @@ +{ + "cell": "p2_triage_qwen38-think-xhigh-offspec_v1", + "corrected_verdict": "PASS", + "defects_applied": [ + "D3" + ], + "evidence": { + "D3": { + "applies": true, + "brief": { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "category_vocabulary": [ + "bug-report", + "billing-refund", + "billing-other", + "feature-request", + "incident-active", + "account-management", + "auth-permissions", + "general-question", + "spam-or-noise", + "security-incident", + "external-business", + "legal-compliance" + ], + "noise_categories": [ + "spam-or-noise" + ], + "noise_urgency_brief_line": "- `n/a` — for noise/spam where urgency doesn't apply", + "noise_urgency_brief_line_number": 27, + "noise_urgency_value": "n/a", + "urgency_vocabulary": [ + "urgent", + "normal", + "low", + "n/a" + ] + }, + "category_accuracy": 0.867, + "contradiction": "brief line 27 defines urgency 'n/a' for noise/spam; the ground truth labels those same tickets differently, and the grader does exact string match", + "corrected_verdict": "PASS", + "credited_count": 3, + "credited_tickets": [ + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "004" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "009" + }, + { + "authorised_by_brief_line": 27, + "ground_truth_category": "spam-or-noise", + "ground_truth_label": "low", + "model_answer": "n/a", + "ticket_id": "021" + } + ], + "defect": "D3", + "duplicate_recall": 1.0, + "grader_arithmetic_reproduced": true, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "tickets_total": 30, + "title": "p2_triage brief / ground-truth contradiction", + "urgency_accuracy_corrected": 0.9, + "urgency_accuracy_original": 0.8, + "urgency_threshold": 0.7 + } + }, + "family": "p2_triage", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "PASS" + }, + "grade_json_sha256": "175b4a70a9d7431ddaae6e10f6459676c29471ea347cbb0d88686b76d840a52b", + "identity": { + "arm": "qwen38-think-xhigh-offspec", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "PASS", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "triage", + "verdict_changed": false +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-27b-udq4xl-think-xhigh_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-27b-udq4xl-think-xhigh_v1/grade.corrected.json new file mode 100644 index 00000000..b65dad0a --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-27b-udq4xl-think-xhigh_v1/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_business_qwen38-27b-udq4xl-think-xhigh_v1", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_business_memo_grade.py:115", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4452, + "ceiling": 700, + "counter_delta": 29, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 723, + "over_ceiling_by_grader_regex": true, + "relname": "memo.md", + "sha256": "faaf12b72f896350630e12de7e4e2f759c10ee4938fa5efa515f84db2fc35fa1", + "stored_grader_count": 723, + "wc_w_count": 694, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 694, + "wc_w_shell": 694, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 5, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_business", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "0415124f5dc5b5e748d2fd70f40c83f1e14e8d07fc2f7abac348b0f6b976eb5e", + "identity": { + "arm": "qwen38-27b-udq4xl-think-xhigh", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "business_memo", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-27b-udq4xl-think-xhigh_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-27b-udq4xl-think-xhigh_v2/grade.corrected.json new file mode 100644 index 00000000..ce8a7ebe --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-27b-udq4xl-think-xhigh_v2/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_business_qwen38-27b-udq4xl-think-xhigh_v2", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_business_memo_grade.py:115", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4552, + "ceiling": 700, + "counter_delta": 48, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 742, + "over_ceiling_by_grader_regex": true, + "relname": "memo.md", + "sha256": "c869d928ab0db8c9d6842b7d9c689c7edbd887efce4749d47e6fc3394fbd4854", + "stored_grader_count": 742, + "wc_w_count": 694, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 694, + "wc_w_shell": 694, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 5, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_business", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "5de758fedb56b91e782df4742de6babaaea4decef98b6fd11dbc135c40e9ad81", + "identity": { + "arm": "qwen38-27b-udq4xl-think-xhigh", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "business_memo", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-27b-udq4xl-think-xhigh_v3/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-27b-udq4xl-think-xhigh_v3/grade.corrected.json new file mode 100644 index 00000000..8da6573a --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-27b-udq4xl-think-xhigh_v3/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_business_qwen38-27b-udq4xl-think-xhigh_v3", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_business_memo_grade.py:115", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4400, + "ceiling": 700, + "counter_delta": 52, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 749, + "over_ceiling_by_grader_regex": true, + "relname": "memo.md", + "sha256": "de65e6468bda6a84615b2b65d93305fca1b0c9d90ccec2bb52e47657fcdc38ae", + "stored_grader_count": 749, + "wc_w_count": 697, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 697, + "wc_w_shell": 697, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 6, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_business", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "e6ea8ced29cb3c11f1fe9fe3bcd1a431b6a941a4eebf5eadd3c30b1cafda1650", + "identity": { + "arm": "qwen38-27b-udq4xl-think-xhigh", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "3", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "business_memo", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-nothink-offspec_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-nothink-offspec_v1/grade.corrected.json new file mode 100644 index 00000000..49e8760b --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-nothink-offspec_v1/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_business_qwen38-nothink-offspec_v1", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_business_memo_grade.py:115", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4368, + "ceiling": 700, + "counter_delta": 39, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 738, + "over_ceiling_by_grader_regex": true, + "relname": "memo.md", + "sha256": "99a81d772d857ae9aae5ff132c4c61f3654c01073ddd8d8c162401960ad210f9", + "stored_grader_count": 738, + "wc_w_count": 699, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 699, + "wc_w_shell": 699, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 10, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_business", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "3eb232ff8cd16c88a8c86d2f640a057724a06fabfe8904124915208c75b79f92", + "identity": { + "arm": "qwen38-nothink-offspec", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "business_memo", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-nothink-offspec_v4/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-nothink-offspec_v4/grade.corrected.json new file mode 100644 index 00000000..56f2ac97 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-nothink-offspec_v4/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_business_qwen38-nothink-offspec_v4", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_business_memo_grade.py:115", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4350, + "ceiling": 700, + "counter_delta": 26, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 723, + "over_ceiling_by_grader_regex": true, + "relname": "memo.md", + "sha256": "c307cdfb8db84a3305c220c02ff22a6f809135a0b43338ed12e7d2050393cf27", + "stored_grader_count": 723, + "wc_w_count": 697, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 697, + "wc_w_shell": 697, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 1, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_business", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "3f9edde22a7b52dda9a4d794ecdd87454c19d8271592888d7ca4b024726041e6", + "identity": { + "arm": "qwen38-nothink-offspec", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "4", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "business_memo", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-nothink-offspec_v5/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-nothink-offspec_v5/grade.corrected.json new file mode 100644 index 00000000..59eaf2a2 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-nothink-offspec_v5/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_business_qwen38-nothink-offspec_v5", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_business_memo_grade.py:115", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4485, + "ceiling": 700, + "counter_delta": 56, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 754, + "over_ceiling_by_grader_regex": true, + "relname": "memo.md", + "sha256": "65e965710821198c61ca611e55127702af1cafeab9b3dd6ec0df14246df6eaa7", + "stored_grader_count": 754, + "wc_w_count": 698, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 698, + "wc_w_shell": 698, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_business", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "723a2011d60ce5f568ad505324d9edb6b2a392a08442b177ae07393b40a916ec", + "identity": { + "arm": "qwen38-nothink-offspec", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "5", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "business_memo", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-nothink-offspec_v6/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-nothink-offspec_v6/grade.corrected.json new file mode 100644 index 00000000..2b45fd41 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-nothink-offspec_v6/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_business_qwen38-nothink-offspec_v6", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_business_memo_grade.py:115", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4337, + "ceiling": 700, + "counter_delta": 33, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 732, + "over_ceiling_by_grader_regex": true, + "relname": "memo.md", + "sha256": "bcfc60ba175fa2c2f98d905f374f1bb83d1e0f7220abece06f860f5a527971c1", + "stored_grader_count": 732, + "wc_w_count": 699, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 699, + "wc_w_shell": 699, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 8, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_business", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "924bdecdd84d963bf57ee17ce9215b9f8300680bcdb00e25b1adfad5e84492c3", + "identity": { + "arm": "qwen38-nothink-offspec", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "6", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "business_memo", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-nothink-offspec_v7/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-nothink-offspec_v7/grade.corrected.json new file mode 100644 index 00000000..8e3e5def --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-nothink-offspec_v7/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_business_qwen38-nothink-offspec_v7", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_business_memo_grade.py:115", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4364, + "ceiling": 700, + "counter_delta": 36, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 731, + "over_ceiling_by_grader_regex": true, + "relname": "memo.md", + "sha256": "59faffc4596f3c3cde50c34729f0a5c1b588b2631c4873f06d613cf04be49b34", + "stored_grader_count": 731, + "wc_w_count": 695, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 695, + "wc_w_shell": 695, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 3, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_business", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "90aa99b98b4f92ede998fb6c5e51a32c8c0794ae65d03377e9691d6759a90b18", + "identity": { + "arm": "qwen38-nothink-offspec", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "7", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "business_memo", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-think-medium-offspec_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-think-medium-offspec_v2/grade.corrected.json new file mode 100644 index 00000000..a9ac63f9 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-think-medium-offspec_v2/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_business_qwen38-think-medium-offspec_v2", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_business_memo_grade.py:115", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4380, + "ceiling": 700, + "counter_delta": 45, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 741, + "over_ceiling_by_grader_regex": true, + "relname": "memo.md", + "sha256": "e9f4b24e4901af246e099c81b39445b27c61ea381d2d937263a8b1b61cc73c7b", + "stored_grader_count": 741, + "wc_w_count": 696, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 696, + "wc_w_shell": 696, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 4, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_business", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "512e382129b5830515de2b7d87b725946eb9079ec76d1ae3bc93dd6d53a580c5", + "identity": { + "arm": "qwen38-think-medium-offspec", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "business_memo", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-think-xhigh-offspec_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-think-xhigh-offspec_v1/grade.corrected.json new file mode 100644 index 00000000..746d1a51 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_business_qwen38-think-xhigh-offspec_v1/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_business_qwen38-think-xhigh-offspec_v1", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_business_memo_grade.py:115", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4343, + "ceiling": 700, + "counter_delta": 34, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 724, + "over_ceiling_by_grader_regex": true, + "relname": "memo.md", + "sha256": "8e0ef51385cc41a27ef863fdc2c3ec4c114096dd7bfe0579eabc0452c2871536", + "stored_grader_count": 724, + "wc_w_count": 690, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 690, + "wc_w_shell": 690, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 5, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_business", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "a923b2900346e5fcb071ed3551b52658e2414cd4e858c64505e03a274a399c2e", + "identity": { + "arm": "qwen38-think-xhigh-offspec", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "business_memo", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_doc_qwen38-27b-udq4xl-nothink_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_doc_qwen38-27b-udq4xl-nothink_v1/grade.corrected.json new file mode 100644 index 00000000..9441e5ac --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_doc_qwen38-27b-udq4xl-nothink_v1/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen38-27b-udq4xl-nothink_v1", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4210, + "ceiling": 700, + "counter_delta": 23, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 702, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "eca686e9657961db217c4724500aa65aea024cb2622be98b8c7d82cb9db4a0cd", + "stored_grader_count": 702, + "wc_w_count": 679, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 679, + "wc_w_shell": 679, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 8, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "ead6f5d24ee0c5b1fefbfb03343181c808f944e5d23fc326b3f894c994aed1f1", + "identity": { + "arm": "qwen38-27b-udq4xl-nothink", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.7/p0.8/pp1.5", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_doc_qwen38-27b-udq4xl-nothink_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_doc_qwen38-27b-udq4xl-nothink_v2/grade.corrected.json new file mode 100644 index 00000000..a4fba112 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_doc_qwen38-27b-udq4xl-nothink_v2/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen38-27b-udq4xl-nothink_v2", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4344, + "ceiling": 700, + "counter_delta": 27, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 727, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "b68ffc2cfc39e94e890f023d6ba34c663e6bfad7f07507d55a36df3ac6c49aab", + "stored_grader_count": 727, + "wc_w_count": 700, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 700, + "wc_w_shell": 700, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 7, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "539788b1a834d67f2a976114e30d72c09b558b6e4f437ddce97d70cf09b65576", + "identity": { + "arm": "qwen38-27b-udq4xl-nothink", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.7/p0.8/pp1.5", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_doc_qwen38-27b-udq4xl-think-xhigh_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_doc_qwen38-27b-udq4xl-think-xhigh_v1/grade.corrected.json new file mode 100644 index 00000000..0a79b5aa --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_doc_qwen38-27b-udq4xl-think-xhigh_v1/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen38-27b-udq4xl-think-xhigh_v1", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4118, + "ceiling": 700, + "counter_delta": 40, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 722, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "f1d6c6fea1094fe6363e4daadd77d76b026c262659d822b9bacac7623110b5cd", + "stored_grader_count": 722, + "wc_w_count": 682, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 682, + "wc_w_shell": 682, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 4, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "367943c288b5f651ec2a5d26ff385c8e34bb7ae292777ebb44a47a1cb44ce44d", + "identity": { + "arm": "qwen38-27b-udq4xl-think-xhigh", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_doc_qwen38-27b-udq4xl-think-xhigh_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_doc_qwen38-27b-udq4xl-think-xhigh_v2/grade.corrected.json new file mode 100644 index 00000000..c8a609b2 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_doc_qwen38-27b-udq4xl-think-xhigh_v2/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen38-27b-udq4xl-think-xhigh_v2", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4220, + "ceiling": 700, + "counter_delta": 30, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 719, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "6a66e60af97a7813ca36e430cd5a632835ac0841f30157979451cacbb11a5cc6", + "stored_grader_count": 719, + "wc_w_count": 689, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 689, + "wc_w_shell": 689, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 18, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "bc954de64249ce81db3b22ca116f8524a85fd9d4294d268bbae9b7c44e090750", + "identity": { + "arm": "qwen38-27b-udq4xl-think-xhigh", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_doc_qwen38-27b-udq4xl-think-xhigh_v3/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_doc_qwen38-27b-udq4xl-think-xhigh_v3/grade.corrected.json new file mode 100644 index 00000000..a3d5f2b4 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_doc_qwen38-27b-udq4xl-think-xhigh_v3/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen38-27b-udq4xl-think-xhigh_v3", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4242, + "ceiling": 700, + "counter_delta": 31, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 725, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "2dd5fc058f7f145d502221b46f2d82d9e5ee77450f07cd12c58f4d745113ee50", + "stored_grader_count": 725, + "wc_w_count": 694, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 694, + "wc_w_shell": 694, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 13, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "105b33795e470e7ec033b41c9d6487d3a58829548f8957cbe1a623e96efc3531", + "identity": { + "arm": "qwen38-27b-udq4xl-think-xhigh", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "3", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_doc_qwen38-think-medium-offspec_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_doc_qwen38-think-medium-offspec_v1/grade.corrected.json new file mode 100644 index 00000000..a97ecfa5 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_doc_qwen38-think-medium-offspec_v1/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen38-think-medium-offspec_v1", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4190, + "ceiling": 700, + "counter_delta": 12, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 708, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "6a190b287e3674c732b26777e8868d3fda006d8a7933f0516594cf8d8aaf0f5c", + "stored_grader_count": 708, + "wc_w_count": 696, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 696, + "wc_w_shell": 696, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 8, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "c92dea0dccb6d6e7b07549ac135286dfb09afad4eaad311025aba492c88fde46", + "identity": { + "arm": "qwen38-think-medium-offspec", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_doc_qwen38-think-xhigh-offspec_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_doc_qwen38-think-xhigh-offspec_v1/grade.corrected.json new file mode 100644 index 00000000..7a47a2d0 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_doc_qwen38-think-xhigh-offspec_v1/grade.corrected.json @@ -0,0 +1,66 @@ +{ + "cell": "p3_doc_qwen38-think-xhigh-offspec_v1", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_doc_synthesis_grade.py:70", + "length_attributable_fail": true, + "measurements": [ + { + "bytes": 4258, + "ceiling": 700, + "counter_delta": 27, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 723, + "over_ceiling_by_grader_regex": true, + "relname": "brief.md", + "sha256": "1bede448fa6f0e895c5ac8b731288bba1c5e3db8639ec2bf0afb1ccfcfd097a8", + "stored_grader_count": 723, + "wc_w_count": 696, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 696, + "wc_w_shell": 696, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 11, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_doc", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "7420a8242a02060b207e70ba63f3608429d1462a4ee5188c3fa8f3519c319930", + "identity": { + "arm": "qwen38-think-xhigh-offspec", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "doc_synthesis", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_pm_qwen38-think-low-offspec_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_pm_qwen38-think-low-offspec_v1/grade.corrected.json new file mode 100644 index 00000000..7f02f4b7 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_pm_qwen38-think-low-offspec_v1/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen38-think-low-offspec_v1", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2902, + "ceiling": 700, + "counter_delta": -42, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 455, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "fbb12b31a5bfb1ff5d5d75c97b8b4f603ae6da3c6bf8032cca54afa002578f5e", + "stored_grader_count": 455, + "wc_w_count": 497, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 497, + "wc_w_shell": 497, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 5, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal dependency** — *med-high.* private-beta contract", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "cut custom-branding", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "native in v2", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "mid-july", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom-branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 455, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "ing. 3. **legal dependency** — *med-high.* private-beta contract draft unanswered (wk 6); needed for beta-customer contracts (wk 6). 4. **no timeline slack** — *med.* \"out of slack\" (w", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1277, + "matched_span": "legal dependency** — *med-high.* private-beta contract", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1223 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "fbb12b31a5bfb1ff5d5d75c97b8b4f603ae6da3c6bf8032cca54afa002578f5e", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "6dee727794dc4d5e439d612a03df29bb9a9e643988c16f9929d24d4239674f93", + "identity": { + "arm": "qwen38-think-low-offspec", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_pm_qwen38-think-medium-offspec_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_pm_qwen38-think-medium-offspec_v1/grade.corrected.json new file mode 100644 index 00000000..5e0197fe --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_pm_qwen38-think-medium-offspec_v1/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen38-think-medium-offspec_v1", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 2869, + "ceiling": 700, + "counter_delta": -45, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 453, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "47386ee68030f9e691634026da82dfc83a8c47d6710fa4747d9a0c7709a6ea4f", + "stored_grader_count": 453, + "wc_w_count": 498, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 498, + "wc_w_shell": 498, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 3, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal not engaged.** private-beta contract", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "cut custom branding", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "responsive in v1", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": null, + "matched": false + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "4/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 453, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "*medium — legal not engaged.** private-beta contract draft unanswered (week 6). 4. **medium — timeline slack exhausted.** query layer re-estimated 4–6 → 8 weeks (week 2); e", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 1445, + "matched_span": "legal not engaged.** private-beta contract", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 1403 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "47386ee68030f9e691634026da82dfc83a8c47d6710fa4747d9a0c7709a6ea4f", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "29daf3f9d7abfd7d3d6e109c46560039fbb872f5811a931ddf324b75667bfe1b", + "identity": { + "arm": "qwen38-think-medium-offspec", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_pm_qwen38-think-medium-offspec_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_pm_qwen38-think-medium-offspec_v2/grade.corrected.json new file mode 100644 index 00000000..454862bb --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_pm_qwen38-think-medium-offspec_v2/grade.corrected.json @@ -0,0 +1,218 @@ +{ + "cell": "p3_pm_qwen38-think-medium-offspec_v2", + "corrected_verdict": "PASS", + "defects_applied": [ + "D2" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": false, + "applies": false, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_project_mgmt_grade.py:95-102", + "length_attributable_fail": false, + "measurements": [ + { + "bytes": 3014, + "ceiling": 700, + "counter_delta": -23, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 475, + "over_ceiling_by_grader_regex": false, + "relname": "status_report.md", + "sha256": "c5f0cc6d51d39274a6f05151f1bc43bc6e9b063ce3d20f136b3e41a44fae1c61", + "stored_grader_count": 475, + "wc_w_count": 498, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 498, + "wc_w_shell": 498, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 2, + "non_length_gates_pass": false, + "title": "word-gate tokenizer mismatch" + }, + "D2": { + "applies": true, + "changes": [ + { + "category": "risks", + "item": "R3", + "matched_span": "legal sign-off", + "pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b", + "reason": "legal/private-beta contract delay uses an equivalent non-contracted phrase" + } + ], + "corrected_details": { + "decisions": { + "D1_branding": { + "keyword": "cut custom branding", + "matched": true + }, + "D2_panel_limit": { + "keyword": "40-panel", + "matched": true + }, + "D3_mobile": { + "keyword": "responsive in v1", + "matched": true + }, + "D4_option_b": { + "keyword": "option b", + "matched": true + } + }, + "milestones": { + "M1": { + "keyword": "mid-may", + "matched": true + }, + "M2": { + "keyword": "mid-may", + "matched": true + }, + "M3": { + "keyword": "july 2026", + "matched": true + }, + "M4": { + "keyword": "v1.1", + "matched": true + }, + "M5": { + "keyword": "deferred", + "matched": true + } + }, + "risks": { + "R1": { + "keyword": null, + "matched": false + }, + "R2": { + "keyword": null, + "matched": false + }, + "R3": { + "keyword": "correction-overlay-semantic-equivalent", + "matched": true + }, + "R4": { + "keyword": "40-panel", + "matched": true + }, + "R5": { + "keyword": "custom branding", + "matched": true + }, + "R6": { + "keyword": null, + "matched": false + } + }, + "workstreams": { + "WS1": { + "keyword": "query layer", + "matched": true + }, + "WS2": { + "keyword": "dashboard refresh", + "matched": true + }, + "WS3": { + "keyword": "panel-density", + "matched": true + }, + "WS4": { + "keyword": "access-control", + "matched": true + }, + "WS5": { + "keyword": "maevia", + "matched": true + }, + "WS6": { + "keyword": "legal", + "matched": true + } + } + }, + "corrected_scores": { + "decision_recall": "4/4", + "milestone_recall": "5/5", + "risk_recall": "3/6", + "sections_present": [ + "workstream", + "risk", + "decision", + "milestone" + ], + "word_count": 475, + "workstream_recall": "6/6" + }, + "defect": "D2", + "evidence": { + "R3": { + "literal_keywords_required_by_grader": [ + "legal unresponsive", + "legal silent", + "legal hasn't", + "blocking on legal" + ], + "matched_phrase_in_report": "items are legal sign-off on beta contracts and the maevia customer commitment (week 6). ## workstreams | workstream | owner | current status | b", + "matched_span_of_upstream_pattern": { + "end_in_normalized_text": 262, + "matched_span": "legal sign-off", + "normalization": "lowercased, whitespace collapsed to single spaces (identical to upstream apply_correction)", + "start_in_normalized_text": 248 + }, + "semantic_rule": "legal/private-beta contract delay uses an equivalent non-contracted phrase", + "upstream_pattern": "\\blegal\\b.{0,200}\\b(?:has\\s+not|not\\s+yet\\s+responded|sign[ -]?off|approval|unresponsive|silent|delay|contract)\\b" + } + }, + "mechanism_note": "the rule actually applied is upstream R3, an adjacency regex (the word legal within 200 characters of a completion term such as has not / sign-off / approval / unresponsive / silent / delay / contract), which matches strictly more than contractions; matched_span in each change records the exact text that satisfied it", + "status_report_sha256": "c5f0cc6d51d39274a6f05151f1bc43bc6e9b063ce3d20f136b3e41a44fae1c61", + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "title": "p3_pm keyword literalism", + "upstream_corrected_verdict": "PASS", + "upstream_module": { + "active_rules": [ + "R3" + ], + "generalised_by": "cell selection driven from the frozen dataset instead of upstream build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names", + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "reused": "read_archived_report + apply_correction (not reimplemented)", + "sha256": "86eeec68b6c7114fb0672566220938e35fa605cd1b89ac4b0863f7cfe305f581" + } + } + }, + "family": "p3_pm", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "grade_json_sha256": "dd26e620a855d291b92d78d29780a88ebb514e5387dea4e33fae12495893c21c", + "identity": { + "arm": "qwen38-think-medium-offspec", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "project_mgmt", + "verdict_changed": true +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_writing_qwen38-27b-udq4xl-nothink_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_writing_qwen38-27b-udq4xl-nothink_v1/grade.corrected.json new file mode 100644 index 00000000..a8680146 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_writing_qwen38-27b-udq4xl-nothink_v1/grade.corrected.json @@ -0,0 +1,107 @@ +{ + "cell": "p3_writing_qwen38-27b-udq4xl-nothink_v1", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + "length_attributable_fail": true, + "measurements": [ + { + "audience": "ceo_brief", + "bytes": 1631, + "ceiling": 250, + "counter_delta": 9, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 255, + "over_ceiling_by_grader_regex": true, + "relname": "ceo_brief.md", + "sha256": "4c01fbc0c16643c922391c4ed57d987bef3c754c5801aa8be1daf67530569c6a", + "stored_grader_count": 255, + "wc_w_count": 246, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 246, + "wc_w_shell": 246, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "customer_email", + "bytes": 2082, + "ceiling": 350, + "counter_delta": 10, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 358, + "over_ceiling_by_grader_regex": true, + "relname": "customer_email.md", + "sha256": "8d614adfb057cdc92ab086bd36677a7936304d8c1e2c27c256145551af083435", + "stored_grader_count": 358, + "wc_w_count": 348, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 348, + "wc_w_shell": 348, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "legal_summary", + "bytes": 2257, + "ceiling": 400, + "counter_delta": 13, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 328, + "over_ceiling_by_grader_regex": false, + "relname": "legal_summary.md", + "sha256": "25eb2754d1221108a848525c41710ad8b356384e757b2d666e9b004a54f3119a", + "stored_grader_count": 328, + "wc_w_count": 315, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 315, + "wc_w_shell": 315, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 7, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_writing", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "ccfb2212c7ad8f42136ad00068ce08eddd3fdffc555906da28e0446f5158ea27", + "identity": { + "arm": "qwen38-27b-udq4xl-nothink", + "mode": "no-think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.7/p0.8/pp1.5", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "writing_editing", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_writing_qwen38-27b-udq4xl-think-xhigh_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_writing_qwen38-27b-udq4xl-think-xhigh_v1/grade.corrected.json new file mode 100644 index 00000000..96372581 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_writing_qwen38-27b-udq4xl-think-xhigh_v1/grade.corrected.json @@ -0,0 +1,107 @@ +{ + "cell": "p3_writing_qwen38-27b-udq4xl-think-xhigh_v1", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + "length_attributable_fail": true, + "measurements": [ + { + "audience": "ceo_brief", + "bytes": 1662, + "ceiling": 250, + "counter_delta": 12, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 258, + "over_ceiling_by_grader_regex": true, + "relname": "ceo_brief.md", + "sha256": "c1169e0c8d292b55b0dd0aa33fee2b0ef2c8a9081ac13dca43a2ac873ec8fa97", + "stored_grader_count": 258, + "wc_w_count": 246, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 246, + "wc_w_shell": 246, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "customer_email", + "bytes": 1982, + "ceiling": 350, + "counter_delta": 6, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 337, + "over_ceiling_by_grader_regex": false, + "relname": "customer_email.md", + "sha256": "f6a4dd3edcb2d2adb37da95e5b78339d7f73411b39cf410274d521ae0975d9ee", + "stored_grader_count": 337, + "wc_w_count": 331, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 331, + "wc_w_shell": 331, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "legal_summary", + "bytes": 2820, + "ceiling": 400, + "counter_delta": 16, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 410, + "over_ceiling_by_grader_regex": true, + "relname": "legal_summary.md", + "sha256": "c559cd76b97b8264b5912497dce06be48e3883c08c55ba8208603207dd0bee9c", + "stored_grader_count": 410, + "wc_w_count": 394, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 394, + "wc_w_shell": 394, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 9, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_writing", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "f9f431ce4a798f712d88a4a0f14d1a49fd9a160146438f339dbdbb319735b9b0", + "identity": { + "arm": "qwen38-27b-udq4xl-think-xhigh", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "writing_editing", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_writing_qwen38-27b-udq4xl-think-xhigh_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_writing_qwen38-27b-udq4xl-think-xhigh_v2/grade.corrected.json new file mode 100644 index 00000000..ce275532 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_writing_qwen38-27b-udq4xl-think-xhigh_v2/grade.corrected.json @@ -0,0 +1,107 @@ +{ + "cell": "p3_writing_qwen38-27b-udq4xl-think-xhigh_v2", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + "length_attributable_fail": true, + "measurements": [ + { + "audience": "ceo_brief", + "bytes": 1708, + "ceiling": 250, + "counter_delta": 9, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 258, + "over_ceiling_by_grader_regex": true, + "relname": "ceo_brief.md", + "sha256": "da31f8272cad55e0a2e42f52dd5c555a6ea65acc50dd6ed43a9a541b8c982592", + "stored_grader_count": 258, + "wc_w_count": 249, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 249, + "wc_w_shell": 249, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "customer_email", + "bytes": 2025, + "ceiling": 350, + "counter_delta": 6, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 351, + "over_ceiling_by_grader_regex": true, + "relname": "customer_email.md", + "sha256": "8c2f13693d7cda992789ec598fa42f42d11a56bc2c7fa36bea4bbaef2de50d89", + "stored_grader_count": 351, + "wc_w_count": 345, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 345, + "wc_w_shell": 345, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "legal_summary", + "bytes": 2751, + "ceiling": 400, + "counter_delta": 6, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 399, + "over_ceiling_by_grader_regex": false, + "relname": "legal_summary.md", + "sha256": "fe7e0c44829fbb2283872259675357c506bc2cfb8cdccbaa0a5420dad3a6f889", + "stored_grader_count": 399, + "wc_w_count": 393, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 393, + "wc_w_shell": 393, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 9, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_writing", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "a4b20f247a9f2e078e2ffc29be280d4850d981761d7257d557e4735d3015174d", + "identity": { + "arm": "qwen38-27b-udq4xl-think-xhigh", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "writing_editing", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_writing_qwen38-27b-udq4xl-think-xhigh_v3/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_writing_qwen38-27b-udq4xl-think-xhigh_v3/grade.corrected.json new file mode 100644 index 00000000..1d327de0 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_writing_qwen38-27b-udq4xl-think-xhigh_v3/grade.corrected.json @@ -0,0 +1,107 @@ +{ + "cell": "p3_writing_qwen38-27b-udq4xl-think-xhigh_v3", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + "length_attributable_fail": true, + "measurements": [ + { + "audience": "ceo_brief", + "bytes": 1666, + "ceiling": 250, + "counter_delta": 14, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 260, + "over_ceiling_by_grader_regex": true, + "relname": "ceo_brief.md", + "sha256": "a834800c2eb76f2a343c6a000aab2acc2de0ee6e33e712aaf1ad5783cb4e9d42", + "stored_grader_count": 260, + "wc_w_count": 246, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 246, + "wc_w_shell": 246, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "customer_email", + "bytes": 1911, + "ceiling": 350, + "counter_delta": 9, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 329, + "over_ceiling_by_grader_regex": false, + "relname": "customer_email.md", + "sha256": "4174c45153c85f361b285b0e91e7bd03986cb77cb05cecf27dc660e053a25dbd", + "stored_grader_count": 329, + "wc_w_count": 320, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 320, + "wc_w_shell": 320, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "legal_summary", + "bytes": 2742, + "ceiling": 400, + "counter_delta": 10, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 401, + "over_ceiling_by_grader_regex": true, + "relname": "legal_summary.md", + "sha256": "d536b4ddba647614c8adac5ff72b30e9c82351504177f5753f392b4a635df030", + "stored_grader_count": 401, + "wc_w_count": 391, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 391, + "wc_w_shell": 391, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 9, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_writing", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "e70d3e35dd6d609e39edcb66f56a4b31178f587fb092dae1d7727b970d5baafb", + "identity": { + "arm": "qwen38-27b-udq4xl-think-xhigh", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T1/p0.95/pp0", + "replicate": "3", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "writing_editing", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_writing_qwen38-think-medium-offspec_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_writing_qwen38-think-medium-offspec_v1/grade.corrected.json new file mode 100644 index 00000000..cf529c94 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_writing_qwen38-think-medium-offspec_v1/grade.corrected.json @@ -0,0 +1,107 @@ +{ + "cell": "p3_writing_qwen38-think-medium-offspec_v1", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + "length_attributable_fail": true, + "measurements": [ + { + "audience": "ceo_brief", + "bytes": 1619, + "ceiling": 250, + "counter_delta": 10, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 260, + "over_ceiling_by_grader_regex": true, + "relname": "ceo_brief.md", + "sha256": "2f857c6e9536d51946d4af893f6b9b9113214dbf6a3a6aa4acebd82343499cc2", + "stored_grader_count": 260, + "wc_w_count": 250, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 250, + "wc_w_shell": 250, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "customer_email", + "bytes": 1754, + "ceiling": 350, + "counter_delta": 13, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 298, + "over_ceiling_by_grader_regex": false, + "relname": "customer_email.md", + "sha256": "aa938a28a518e991de2385c59fdbc4a4ce42f9a001f1b215847cb38e6f958050", + "stored_grader_count": 298, + "wc_w_count": 285, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 285, + "wc_w_shell": 285, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "legal_summary", + "bytes": 1839, + "ceiling": 400, + "counter_delta": 16, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 282, + "over_ceiling_by_grader_regex": false, + "relname": "legal_summary.md", + "sha256": "e8f4abbc2c4161c0a6dd7be83ca31b0dd2f2026c282cd29d9c0f9de5ec72d894", + "stored_grader_count": 282, + "wc_w_count": 266, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 266, + "wc_w_shell": 266, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 7, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_writing", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "ad2d379b46bf6855856c9c3d6d7ab89a188882637792f8f94a82b3f35552cebe", + "identity": { + "arm": "qwen38-think-medium-offspec", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "writing_editing", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_writing_qwen38-think-medium-offspec_v2/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_writing_qwen38-think-medium-offspec_v2/grade.corrected.json new file mode 100644 index 00000000..8805ebd2 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_writing_qwen38-think-medium-offspec_v2/grade.corrected.json @@ -0,0 +1,107 @@ +{ + "cell": "p3_writing_qwen38-think-medium-offspec_v2", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + "length_attributable_fail": true, + "measurements": [ + { + "audience": "ceo_brief", + "bytes": 1596, + "ceiling": 250, + "counter_delta": 10, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 259, + "over_ceiling_by_grader_regex": true, + "relname": "ceo_brief.md", + "sha256": "929e7b94673e76cb3d6973a89bdacc939410131645bf1871f3504ea78fa461f0", + "stored_grader_count": 259, + "wc_w_count": 249, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 249, + "wc_w_shell": 249, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "customer_email", + "bytes": 1762, + "ceiling": 350, + "counter_delta": 13, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 298, + "over_ceiling_by_grader_regex": false, + "relname": "customer_email.md", + "sha256": "3328cb096038aa0d701ab883f567770435e11b1d8529b8febe96ebc1062219e5", + "stored_grader_count": 298, + "wc_w_count": 285, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 285, + "wc_w_shell": 285, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "legal_summary", + "bytes": 1831, + "ceiling": 400, + "counter_delta": 16, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 280, + "over_ceiling_by_grader_regex": false, + "relname": "legal_summary.md", + "sha256": "14d91aa7617d5d2e310f1c63b9369c4dfe2726880f5204fa32e00e4d0e50e40b", + "stored_grader_count": 280, + "wc_w_count": 264, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 264, + "wc_w_shell": 264, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 8, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_writing", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "145b3def196e2fbfeb32f0901e56e361f6f4ae161ae4c3dd69e563fd77a477f0", + "identity": { + "arm": "qwen38-think-medium-offspec", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "2", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "writing_editing", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_writing_qwen38-think-xhigh-offspec_v1/grade.corrected.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_writing_qwen38-think-xhigh-offspec_v1/grade.corrected.json new file mode 100644 index 00000000..09cf5f08 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/mmbt-qwen38-eaaa8ca/logs/p3_writing_qwen38-think-xhigh-offspec_v1/grade.corrected.json @@ -0,0 +1,107 @@ +{ + "cell": "p3_writing_qwen38-think-xhigh-offspec_v1", + "defects_applied": [ + "D1" + ], + "evidence": { + "D1": { + "all_deliverables_within_ceiling_by_wc_w": true, + "any_deliverable_over_ceiling_by_grader_regex": true, + "applies": true, + "defect": "D1", + "extraction_verified": true, + "framing": "neither counter is ground truth: wc -w counts markdown punctuation as words, the grader regex splits hyphenated tokens. Two reasonable counters disagree by more than the margin at a hard threshold.", + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + "length_attributable_fail": true, + "measurements": [ + { + "audience": "ceo_brief", + "bytes": 1679, + "ceiling": 250, + "counter_delta": 8, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 257, + "over_ceiling_by_grader_regex": true, + "relname": "ceo_brief.md", + "sha256": "21938cd1751c215e9c2c158c67512a4dddc0a2ea9cbcf71fd7e6f509defd10e3", + "stored_grader_count": 257, + "wc_w_count": 249, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 249, + "wc_w_shell": 249, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "customer_email", + "bytes": 1893, + "ceiling": 350, + "counter_delta": 1, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 328, + "over_ceiling_by_grader_regex": false, + "relname": "customer_email.md", + "sha256": "3fdeb5e3e8d52cad2b11d97be2d741af8d079089dd483f6048718c042614bab5", + "stored_grader_count": 328, + "wc_w_count": 327, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 327, + "wc_w_shell": 327, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + }, + { + "audience": "legal_summary", + "bytes": 2082, + "ceiling": 400, + "counter_delta": 22, + "grader_recount_matches_stored": true, + "grader_regex_word_count": 318, + "over_ceiling_by_grader_regex": false, + "relname": "legal_summary.md", + "sha256": "927876d0fcb262d841f0675a790e0414dfbf5272a2077d064bf7145db0d1d59b", + "stored_grader_count": 318, + "wc_w_count": 296, + "wc_w_counters_agree": true, + "wc_w_locale": "C.UTF-8", + "wc_w_locale_provenance": "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside that image against an extracted deliverable", + "wc_w_pure_python": 296, + "wc_w_shell": 296, + "wc_w_source": "shell wc -w under LC_ALL=C.UTF-8", + "within_ceiling_by_wc_w": true + } + ], + "model_wc_w_calls_in_transcript": 1, + "non_length_gates_pass": true, + "title": "word-gate tokenizer mismatch" + } + }, + "family": "p3_writing", + "frozen_dataset": { + "graded_at_freeze": true, + "matches_on_disk": true, + "verdict_at_freeze": "FAIL" + }, + "gate_invalidated": true, + "gate_invalidated_meaning": "the original FAIL is attributable solely to the length gate, where two reasonable word counters disagree across the threshold; read this cell as NOT A VALID FAIL, not as a verified PASS (grader-defects doc, D1 correction limits)", + "grade_json_sha256": "da73f6dd151579a7564e30450be17727e996d734c5e14240d6c23805cb4c6220", + "identity": { + "arm": "qwen38-think-xhigh-offspec", + "mode": "think", + "model": "3.8", + "quant": "UD-Q4_K_XL", + "regime": "T0.3/p0.8/pp0", + "replicate": "1", + "source": "cell identity, model and sampler are taken verbatim from the frozen dataset; never recomputed from directory names" + }, + "original_verdict": "FAIL", + "policy": "grade.json, task briefs and ground truth are immutable run evidence; this overlay modifies no run artifact and no raw verdict", + "repo": "mmbt-qwen38-eaaa8ca", + "schema_version": "mmbt-grade-correction-overlay/2", + "task": "writing_editing", + "verdict_without_length_gate": "PASS" +} diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/results-tables.md b/benchmarks/qwen36-vs-qwen38-27b-2026-08/results-tables.md new file mode 100644 index 00000000..17597ff4 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/results-tables.md @@ -0,0 +1,910 @@ +# Qwen3.6-27B vs Qwen3.8-27B — Definitive Results Tables + +All numbers below are computed from the frozen dataset +`/home/michael/mmbt-frozen-dataset-v2.csv` (802 cells, frozen 2026-08-16T14:23:09Z). +No live re-scan was used for any figure. Model, quantization and sampler identity are taken +verbatim from the frozen CSV, which sourced them from each run's `receipt.json` — never from +directory names. + +**Reproduce:** `python3 mmbt_results.py` writes `results.json`; `python3 make_md.py` renders +this file; `python3 test_stats.py` validates the estimators. No third-party dependencies. + +### Reading these tables + +- **Sign convention.** Every delta in this document is arithmetically **3.8 minus 3.6**, + applied identically to every table. Note that this means the *direction of badness flips + with the metric*: for good-direction metrics (terminal rate, pass rate) a **negative** delta + means 3.8 is worse, while for bad-direction metrics (ungraded rate, loop rate) a + **positive** delta means 3.8 is worse. Each table states which metric it reports. +- **Every rate carries its denominator** as `x/n`. +- **CIs** are Wilson 95% score intervals for single proportions and Newcombe 95% hybrid-score + intervals for differences. `p` is a two-sided Fisher exact test. All three are implemented + from scratch in pure python; formulas are in the appendix and in the source comments. +- **\*** marks `p < 0.05`. No multiplicity correction is applied — with 4 comparison + families and 7 scorings each, treat isolated marginal results with suspicion. +- `passed == 1` means verdict in {PASS, STRUCTURAL_PASS}. **Every ungraded cell has + `passed == 0`**, so an all-cells pass rate silently merges "graded and failed" with + "never produced a gradeable artifact". Both components are reported separately. + +--- + +## Headline + +**1. The one large, unambiguous effect is a delivery regression in no-think mode, not a +quality regression.** 3.8's loop rate is +28.0 pp (4.9e-09) at the matched sampler and +29.2 pp (1.1e-11) at each +model's own vendor sampler. This is measured upstream of grading, so no scoring choice, and +none of the three grader defects, can touch it. + +**2. That regression disappears in thinking mode.** Loop rate falls to 2/51 (T0.3) and 0/72 +(T1/p0.95/pp0) for 3.8, neither distinguishable from 3.6. + +**3. Conditional on delivering, 3.8 is not clearly worse at the matched no-think sampler.** +All-cells pass rate is -13.3 pp (0.05052) but graded-only is +7.4 pp (0.2959) — the sign flips. "3.8 is worse at the task" +is **not** a supportable summary of this corpus; "3.8 fails more often by not finishing" is. + +**4. Thinking mode costs ~2.2–3.0× more tokens on 3.8 with no pass-rate gain**, and 3.8's +effort ladder buys tokens rather than quality (no effort step is significant). + +**5. Two things reported here are provisional at best and are labelled as such:** the Q8_0 +quantization arm (8 of 19 cells graded at freeze #2 — enough for provisional loop and +graded-outcome rates with wide intervals, not enough for any quality verdict, §6) and every +`best_of_n` / `first_replicate` row (n = 12, §9). + +Full robustness accounting is in §7; what the data cannot support is in §9. + +--- + +## 0. Group inventory and missing-data structure + +| Group | Definition | Cells | Families | Graded | Graded+failed | Ungraded (looped) | Ungraded (no loop) | Graded despite loop | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| `A36` | 3.6 no-think T0.3/p0.8/pp0 | 121 | 12 | 114 | 33 | 3 | 4 | 0 | +| `A38` | 3.8 no-think T0.3/p0.8/pp0 | 95 | 12 | 65 | 14 | 25 | 5 | 4 | +| `V36` | 3.6 no-think T1/p0.95/pp0 | 120 | 12 | 120 | 25 | 0 | 0 | 0 | +| `V38` | 3.8 no-think T0.7/p0.8/pp1.5 | 96 | 12 | 79 | 31 | 13 | 4 | 15 | +| `B36` | 3.6 think T0.3/p0.8/pp0 | 108 | 12 | 107 | 40 | 1 | 0 | 0 | +| `B38` | 3.8 think T0.3/p0.8/pp0 | 51 | 12 | 48 | 15 | 2 | 1 | 0 | +| `C36` | 3.6 think T1/p0.95/pp0 | 120 | 12 | 120 | 15 | 0 | 0 | 0 | +| `C38` | 3.8 think T1/p0.95/pp0 | 72 | 12 | 72 | 18 | 0 | 0 | 0 | +| `Q8` | 3.8 Q8_0 no-think T0.3/p0.8/pp0 | 19 | 12 | 8 | 3 | 6 | 5 | 0 | + +Notes on group composition, which matter for how much weight each comparison can carry: + +- **`A36`** — 3.6 no-think T0.3/p0.8/pp0. *off-spec for 3.6 (its card is T1/p0.95/pp0); sampler-matched to A38* Arms pooled: `qwen36-27b-udq4xl-t03-nothink`, `qwen36-nothink-offspec`. +- **`A38`** — 3.8 no-think T0.3/p0.8/pp0. *off-spec for 3.8 (its card is T0.7/p0.8/pp1.5); sampler-matched to A36* Arms pooled: `qwen38-27b-udq4xl-t03-nothink`, `qwen38-nothink-offspec`. +- **`V36`** — 3.6 no-think T1/p0.95/pp0. *3.6 VENDOR point* Arms pooled: `qwen36-nothink-card`. +- **`V38`** — 3.8 no-think T0.7/p0.8/pp1.5. *3.8 VENDOR point* Arms pooled: `qwen38-27b-udq4xl-nothink`, `qwen38-nothink-card`. +- **`B36`** — 3.6 think T0.3/p0.8/pp0. *off-spec for 3.6; sampler-matched to B38* Arms pooled: `qwen36-think-offspec`. +- **`B38`** — 3.8 think T0.3/p0.8/pp0. *off-spec for 3.8; sampler-matched to B36; MIXED effort (low/medium/xhigh)* Arms pooled: `qwen38-think-low-offspec`, `qwen38-think-medium-offspec`, `qwen38-think-xhigh-offspec`. +- **`C36`** — 3.6 think T1/p0.95/pp0. *3.6 VENDOR point* Arms pooled: `qwen36-think-card`. +- **`C38`** — 3.8 think T1/p0.95/pp0. *this is 3.6's vendor point, NOT 3.8's; sampler-matched to C36 but off-spec for 3.8; MIXED effort (low/medium/xhigh)* Arms pooled: `qwen38-27b-udq4xl-think-xhigh`, `qwen38-think-low-card`, `qwen38-think-medium-card`, `qwen38-think-xhigh-card`. +- **`Q8`** — 3.8 Q8_0 no-think T0.3/p0.8/pp0. *quantization control; 19 cells over all 12 families (2 replicates on the 7 phase-1/2 families, 1 on the 5 phase-3)* Arms pooled: `qwen38q8-nothink-matched`. + +All 802 runs used **seed 42**. Replicates are therefore repeated runs at a *fixed* seed, not +a seed sweep; within-arm variation reflects server/batching nondeterminism, not sampled seeds. +This limits how far replicate counts can be read as independent draws. + +--- + +## 1. Delivery reliability + +Loop detection is reported under both available metrics. `looped_run30` is strictly nested +inside `looped_freq30` across the whole corpus (47 cells flagged by both, 22 by frequency +only, 0 by run only), so `freq30` is the more inclusive detector and `run30` the stricter one. + +### 1a. Per-group delivery rates (Wilson 95% CI) + +| Group | n | Terminal | Ungraded | Loop (freq30) | Loop (run30) | +|---|---:|---|---|---|---| +| `A36` | 121 | 118/121 97.5% [93.0, 99.2] | 7/121 5.8% [2.8, 11.5] | 3/121 2.5% [0.8, 7.0] | 3/121 2.5% [0.8, 7.0] | +| `A38` | 95 | 93/95 97.9% [92.6, 99.4] | 30/95 31.6% [23.1, 41.5] | 29/95 30.5% [22.2, 40.4] | 23/95 24.2% [16.7, 33.7] | +| `V36` | 120 | 120/120 100.0% [96.9, 100.0] | 0/120 0.0% [0.0, 3.1] | 0/120 0.0% [0.0, 3.1] | 0/120 0.0% [0.0, 3.1] | +| `V38` | 96 | 96/96 100.0% [96.2, 100.0] | 17/96 17.7% [11.4, 26.5] | 28/96 29.2% [21.0, 38.9] | 13/96 13.5% [8.1, 21.8] | +| `B36` | 108 | 108/108 100.0% [96.6, 100.0] | 1/108 0.9% [0.2, 5.1] | 1/108 0.9% [0.2, 5.1] | 1/108 0.9% [0.2, 5.1] | +| `B38` | 51 | 50/51 98.0% [89.7, 99.7] | 3/51 5.9% [2.0, 15.9] | 2/51 3.9% [1.1, 13.2] | 2/51 3.9% [1.1, 13.2] | +| `C36` | 120 | 120/120 100.0% [96.9, 100.0] | 0/120 0.0% [0.0, 3.1] | 0/120 0.0% [0.0, 3.1] | 0/120 0.0% [0.0, 3.1] | +| `C38` | 72 | 72/72 100.0% [94.9, 100.0] | 0/72 0.0% [0.0, 5.1] | 0/72 0.0% [0.0, 5.1] | 0/72 0.0% [0.0, 5.1] | +| `Q8` | 19 | 19/19 100.0% [83.2, 100.0] | 11/19 57.9% [36.3, 76.9] | 6/19 31.6% [15.4, 54.0] | 5/19 26.3% [11.8, 48.8] | + +### 1b. Paired 3.6-vs-3.8 delivery contrasts + +**P1 — No-think, SAMPLER-MATCHED (both T0.3/p0.8/pp0). Off-spec for both models; the cleanest controlled-sampler contrast.** + +| Metric | 3.6 `A36` | 3.8 `A38` | Δ (3.8−3.6) | Newcombe 95% CI | Fisher p | +|---|---|---|---:|---|---:| +| Terminal | 118/121 97.5% [93.0, 99.2] | 93/95 97.9% [92.6, 99.4] | +0.4 pp | [-5.1, 5.2] | 1 | +| Ungraded | 7/121 5.8% [2.8, 11.5] | 30/95 31.6% [23.1, 41.5] | +25.8 pp | [15.6, 36.1] | 8.2e-07 **\*** | +| Loop (freq30) | 3/121 2.5% [0.8, 7.0] | 29/95 30.5% [22.2, 40.4] | +28.0 pp | [18.5, 38.0] | 4.9e-09 **\*** | +| Loop (run30) | 3/121 2.5% [0.8, 7.0] | 23/95 24.2% [16.7, 33.7] | +21.7 pp | [13.0, 31.4] | 8.7e-07 **\*** | + +**P2 — No-think, VENDOR-MATCHED (each model at its own model-card sampler: 3.6 T1/p0.95/pp0, 3.8 T0.7/p0.8/pp1.5). Best-foot-forward contrast; sampler differs by construction.** + +| Metric | 3.6 `V36` | 3.8 `V38` | Δ (3.8−3.6) | Newcombe 95% CI | Fisher p | +|---|---|---|---:|---|---:| +| Terminal | 120/120 100.0% [96.9, 100.0] | 96/96 100.0% [96.2, 100.0] | +0.0 pp | [-3.8, 3.1] | 1 | +| Ungraded | 0/120 0.0% [0.0, 3.1] | 17/96 17.7% [11.4, 26.5] | +17.7 pp | [10.6, 26.5] | 4.4e-07 **\*** | +| Loop (freq30) | 0/120 0.0% [0.0, 3.1] | 28/96 29.2% [21.0, 38.9] | +29.2 pp | [20.4, 38.9] | 1.1e-11 **\*** | +| Loop (run30) | 0/120 0.0% [0.0, 3.1] | 13/96 13.5% [8.1, 21.8] | +13.5 pp | [7.3, 21.8] | 1.6e-05 **\*** | + +**P3 — Thinking, SAMPLER-MATCHED (both T0.3/p0.8/pp0). Off-spec for both. 3.8 side pools three effort levels.** + +| Metric | 3.6 `B36` | 3.8 `B38` | Δ (3.8−3.6) | Newcombe 95% CI | Fisher p | +|---|---|---|---:|---|---:| +| Terminal | 108/108 100.0% [96.6, 100.0] | 50/51 98.0% [89.7, 99.7] | -2.0 pp | [-10.3, 1.8] | 0.3208 | +| Ungraded | 1/108 0.9% [0.2, 5.1] | 3/51 5.9% [2.0, 15.9] | +5.0 pp | [-0.7, 15.0] | 0.09748 | +| Loop (freq30) | 1/108 0.9% [0.2, 5.1] | 2/51 3.9% [1.1, 13.2] | +3.0 pp | [-2.0, 12.3] | 0.2412 | +| Loop (run30) | 1/108 0.9% [0.2, 5.1] | 2/51 3.9% [1.1, 13.2] | +3.0 pp | [-2.0, 12.3] | 0.2412 | + +**P4 — Thinking at T1/p0.95/pp0. Sampler-matched, but that triple is 3.6's vendor point and is NOT 3.8's -- 3.8 is running off-spec here and this pair is biased in 3.6's favour. 3.8 side pools three effort levels.** + +| Metric | 3.6 `C36` | 3.8 `C38` | Δ (3.8−3.6) | Newcombe 95% CI | Fisher p | +|---|---|---|---:|---|---:| +| Terminal | 120/120 100.0% [96.9, 100.0] | 72/72 100.0% [94.9, 100.0] | +0.0 pp | [-5.1, 3.1] | 1 | +| Ungraded | 0/120 0.0% [0.0, 3.1] | 0/72 0.0% [0.0, 5.1] | +0.0 pp | [-3.1, 5.1] | 1 | +| Loop (freq30) | 0/120 0.0% [0.0, 3.1] | 0/72 0.0% [0.0, 5.1] | +0.0 pp | [-3.1, 5.1] | 1 | +| Loop (run30) | 0/120 0.0% [0.0, 3.1] | 0/72 0.0% [0.0, 5.1] | +0.0 pp | [-3.1, 5.1] | 1 | + +--- + +## 2. Quality + +Two pass rates are reported for every pair because they answer different questions: + +- **All-cells** (loops count as failure) — *how often does a run of this model produce a + passing deliverable?* This is the end-to-end number. +- **Graded-only** — *given that a run produced something gradeable, how good was it?* This + discards every delivery failure and is maximally charitable to a model that fails by + not delivering. + +Where these two disagree, the disagreement **is** the finding. + +**P1 — 3.6 no-think T0.3/p0.8/pp0 vs 3.8 no-think T0.3/p0.8/pp0** + +| Scoring | 3.6 | 3.8 | Δ (3.8−3.6) | Newcombe 95% CI | Fisher p | +|---|---|---|---:|---|---:| +| All cells (loops = fail) | 81/121 66.9% [58.2, 74.7] | 51/95 53.7% [43.7, 63.4] | -13.3 pp | [-25.9, -0.2] | 0.05052 | +| Graded only | 81/114 71.1% [62.1, 78.6] | 51/65 78.5% [67.0, 86.7] | +7.4 pp | [-6.3, 19.5] | 0.2959 | + +**P2 — 3.6 no-think T1/p0.95/pp0 vs 3.8 no-think T0.7/p0.8/pp1.5** + +| Scoring | 3.6 | 3.8 | Δ (3.8−3.6) | Newcombe 95% CI | Fisher p | +|---|---|---|---:|---|---:| +| All cells (loops = fail) | 95/120 79.2% [71.1, 85.5] | 48/96 50.0% [40.2, 59.8] | -29.2 pp | [-40.8, -16.4] | 1.1e-05 **\*** | +| Graded only | 95/120 79.2% [71.1, 85.5] | 48/79 60.8% [49.7, 70.8] | -18.4 pp | [-31.1, -5.5] | 0.006102 **\*** | + +**P3 — 3.6 think T0.3/p0.8/pp0 vs 3.8 think T0.3/p0.8/pp0** + +| Scoring | 3.6 | 3.8 | Δ (3.8−3.6) | Newcombe 95% CI | Fisher p | +|---|---|---|---:|---|---:| +| All cells (loops = fail) | 67/108 62.0% [52.6, 70.6] | 33/51 64.7% [51.0, 76.4] | +2.7 pp | [-13.5, 17.7] | 0.8607 | +| Graded only | 67/107 62.6% [53.2, 71.2] | 33/48 68.8% [54.7, 80.1] | +6.1 pp | [-10.4, 20.9] | 0.5863 | + +**P4 — 3.6 think T1/p0.95/pp0 vs 3.8 think T1/p0.95/pp0** + +| Scoring | 3.6 | 3.8 | Δ (3.8−3.6) | Newcombe 95% CI | Fisher p | +|---|---|---|---:|---|---:| +| All cells (loops = fail) | 105/120 87.5% [80.4, 92.3] | 54/72 75.0% [63.9, 83.6] | -12.5 pp | [-24.6, -1.4] | 0.03081 **\*** | +| Graded only | 105/120 87.5% [80.4, 92.3] | 54/72 75.0% [63.9, 83.6] | -12.5 pp | [-24.6, -1.4] | 0.03081 **\*** | + +--- + +## 3. Per-family breakdown + +All-cells pass rate (loops count as failure), with graded and loop counts so any cell can be +re-derived. `g` = graded cells, `L` = cells flagged by `looped_freq30`. + +### P1 — 3.6 no-think T0.3/p0.8/pp0 vs 3.8 no-think T0.3/p0.8/pp0 + +| Family | 3.6 pass/n (g, L) | 3.6 rate | 3.8 pass/n (g, L) | 3.8 rate | Δ (3.8−3.6) | Fisher p | +|---|---|---:|---|---:|---:|---:| +| `p1_bugfix` | 18/19 (g=18, L=0) | 94.7% | 0/12 (g=1, L=10) | 0.0% | -94.7 pp | 9.2e-08 **\*** | +| `p1_refactor` | 5/9 (g=9, L=0) | 55.6% | 3/7 (g=5, L=1) | 42.9% | -12.7 pp | 1 | +| `p1_testwrite` | 9/12 (g=10, L=0) | 75.0% | 0/13 (g=2, L=10) | 0.0% | -75.0 pp | 0.00011 **\*** | +| `p2_ci` | 9/9 (g=9, L=0) | 100.0% | 7/7 (g=7, L=0) | 100.0% | +0.0 pp | 1 | +| `p2_extract` | 9/9 (g=9, L=0) | 100.0% | 7/7 (g=7, L=0) | 100.0% | +0.0 pp | 1 | +| `p2_hallucination` | 9/9 (g=9, L=0) | 100.0% | 7/7 (g=7, L=0) | 100.0% | +0.0 pp | 1 | +| `p2_triage` | 0/9 (g=9, L=0) | 0.0% | 7/7 (g=7, L=0) | 100.0% | +100.0 pp | 8.7e-05 **\*** | +| `p3_business` | 7/9 (g=9, L=0) | 77.8% | 0/7 (g=6, L=1) | 0.0% | -77.8 pp | 0.003234 **\*** | +| `p3_doc` | 8/9 (g=9, L=0) | 88.9% | 6/7 (g=6, L=1) | 85.7% | -3.2 pp | 1 | +| `p3_market` | 4/9 (g=5, L=3) | 44.4% | 0/7 (g=3, L=6) | 0.0% | -44.4 pp | 0.08846 | +| `p3_pm` | 1/9 (g=9, L=0) | 11.1% | 7/7 (g=7, L=0) | 100.0% | +88.9 pp | 0.001399 **\*** | +| `p3_writing` | 2/9 (g=9, L=0) | 22.2% | 7/7 (g=7, L=0) | 100.0% | +77.8 pp | 0.003234 **\*** | + +Per-family n is 7–19 per side, so **no single family row is individually conclusive**; +they are provided so the pooled numbers can be audited, not to support family-level claims. + +### P2 — 3.6 no-think T1/p0.95/pp0 vs 3.8 no-think T0.7/p0.8/pp1.5 + +| Family | 3.6 pass/n (g, L) | 3.6 rate | 3.8 pass/n (g, L) | 3.8 rate | Δ (3.8−3.6) | Fisher p | +|---|---|---:|---|---:|---:|---:| +| `p1_bugfix` | 10/10 (g=10, L=0) | 100.0% | 4/8 (g=4, L=5) | 50.0% | -50.0 pp | 0.02288 **\*** | +| `p1_refactor` | 8/10 (g=10, L=0) | 80.0% | 4/8 (g=8, L=3) | 50.0% | -30.0 pp | 0.3213 | +| `p1_testwrite` | 9/10 (g=10, L=0) | 90.0% | 0/8 (g=7, L=7) | 0.0% | -90.0 pp | 0.00041 **\*** | +| `p2_ci` | 10/10 (g=10, L=0) | 100.0% | 8/8 (g=8, L=0) | 100.0% | +0.0 pp | 1 | +| `p2_extract` | 10/10 (g=10, L=0) | 100.0% | 8/8 (g=8, L=0) | 100.0% | +0.0 pp | 1 | +| `p2_hallucination` | 10/10 (g=10, L=0) | 100.0% | 8/8 (g=8, L=0) | 100.0% | +0.0 pp | 1 | +| `p2_triage` | 7/10 (g=10, L=0) | 70.0% | 5/8 (g=8, L=0) | 62.5% | -7.5 pp | 1 | +| `p3_business` | 10/10 (g=10, L=0) | 100.0% | 2/8 (g=6, L=3) | 25.0% | -75.0 pp | 0.001508 **\*** | +| `p3_doc` | 10/10 (g=10, L=0) | 100.0% | 1/8 (g=8, L=0) | 12.5% | -87.5 pp | 0.00025 **\*** | +| `p3_market` | 10/10 (g=10, L=0) | 100.0% | 0/8 (g=2, L=5) | 0.0% | -100.0 pp | 2.3e-05 **\*** | +| `p3_pm` | 1/10 (g=10, L=0) | 10.0% | 8/8 (g=8, L=0) | 100.0% | +90.0 pp | 0.00041 **\*** | +| `p3_writing` | 0/10 (g=10, L=0) | 0.0% | 0/8 (g=4, L=5) | 0.0% | +0.0 pp | 1 | + +Per-family n is 8–10 per side, so **no single family row is individually conclusive**; +they are provided so the pooled numbers can be audited, not to support family-level claims. + +### P3 — 3.6 think T0.3/p0.8/pp0 vs 3.8 think T0.3/p0.8/pp0 + +| Family | 3.6 pass/n (g, L) | 3.6 rate | 3.8 pass/n (g, L) | 3.8 rate | Δ (3.8−3.6) | Fisher p | +|---|---|---:|---|---:|---:|---:| +| `p1_bugfix` | 9/9 (g=9, L=0) | 100.0% | 4/6 (g=5, L=0) | 66.7% | -33.3 pp | 0.1429 | +| `p1_refactor` | 4/9 (g=9, L=0) | 44.4% | 3/4 (g=4, L=0) | 75.0% | +30.6 pp | 0.5594 | +| `p1_testwrite` | 9/9 (g=9, L=0) | 100.0% | 4/5 (g=5, L=0) | 80.0% | -20.0 pp | 0.3571 | +| `p2_ci` | 9/9 (g=9, L=0) | 100.0% | 4/4 (g=4, L=0) | 100.0% | +0.0 pp | 1 | +| `p2_extract` | 9/9 (g=9, L=0) | 100.0% | 4/4 (g=4, L=0) | 100.0% | +0.0 pp | 1 | +| `p2_hallucination` | 9/9 (g=9, L=0) | 100.0% | 4/4 (g=4, L=0) | 100.0% | +0.0 pp | 1 | +| `p2_triage` | 0/9 (g=9, L=0) | 0.0% | 4/4 (g=4, L=0) | 100.0% | +100.0 pp | 0.001399 **\*** | +| `p3_business` | 8/9 (g=9, L=0) | 88.9% | 2/4 (g=4, L=0) | 50.0% | -38.9 pp | 0.2028 | +| `p3_doc` | 0/9 (g=9, L=0) | 0.0% | 1/4 (g=4, L=0) | 25.0% | +25.0 pp | 0.3077 | +| `p3_market` | 8/9 (g=8, L=1) | 88.9% | 1/4 (g=2, L=2) | 25.0% | -63.9 pp | 0.05175 | +| `p3_pm` | 0/9 (g=9, L=0) | 0.0% | 1/4 (g=4, L=0) | 25.0% | +25.0 pp | 0.3077 | +| `p3_writing` | 2/9 (g=9, L=0) | 22.2% | 1/4 (g=4, L=0) | 25.0% | +2.8 pp | 1 | + +Per-family n is 4–9 per side, so **no single family row is individually conclusive**; +they are provided so the pooled numbers can be audited, not to support family-level claims. + +### P4 — 3.6 think T1/p0.95/pp0 vs 3.8 think T1/p0.95/pp0 + +| Family | 3.6 pass/n (g, L) | 3.6 rate | 3.8 pass/n (g, L) | 3.8 rate | Δ (3.8−3.6) | Fisher p | +|---|---|---:|---|---:|---:|---:| +| `p1_bugfix` | 10/10 (g=10, L=0) | 100.0% | 6/6 (g=6, L=0) | 100.0% | +0.0 pp | 1 | +| `p1_refactor` | 7/10 (g=10, L=0) | 70.0% | 6/6 (g=6, L=0) | 100.0% | +30.0 pp | 0.25 | +| `p1_testwrite` | 9/10 (g=10, L=0) | 90.0% | 6/6 (g=6, L=0) | 100.0% | +10.0 pp | 1 | +| `p2_ci` | 10/10 (g=10, L=0) | 100.0% | 6/6 (g=6, L=0) | 100.0% | +0.0 pp | 1 | +| `p2_extract` | 10/10 (g=10, L=0) | 100.0% | 6/6 (g=6, L=0) | 100.0% | +0.0 pp | 1 | +| `p2_hallucination` | 10/10 (g=10, L=0) | 100.0% | 6/6 (g=6, L=0) | 100.0% | +0.0 pp | 1 | +| `p2_triage` | 10/10 (g=10, L=0) | 100.0% | 6/6 (g=6, L=0) | 100.0% | +0.0 pp | 1 | +| `p3_business` | 10/10 (g=10, L=0) | 100.0% | 0/6 (g=6, L=0) | 0.0% | -100.0 pp | 0.00012 **\*** | +| `p3_doc` | 10/10 (g=10, L=0) | 100.0% | 1/6 (g=6, L=0) | 16.7% | -83.3 pp | 0.001374 **\*** | +| `p3_market` | 10/10 (g=10, L=0) | 100.0% | 6/6 (g=6, L=0) | 100.0% | +0.0 pp | 1 | +| `p3_pm` | 2/10 (g=10, L=0) | 20.0% | 5/6 (g=6, L=0) | 83.3% | +63.3 pp | 0.03497 **\*** | +| `p3_writing` | 7/10 (g=10, L=0) | 70.0% | 0/6 (g=6, L=0) | 0.0% | -70.0 pp | 0.01136 **\*** | + +Per-family n is 6–10 per side, so **no single family row is individually conclusive**; +they are provided so the pooled numbers can be audited, not to support family-level claims. + +--- + +## 4. Cost, matched by family + +**Why family matching is mandatory here.** Task families differ in cost by more than an order +of magnitude (`p2_extract` ~2.5k tokens vs `p1_bugfix` ~35k). Two groups with different family +mixes produce medians that can be ordered either way at will. Earlier in this project exactly +that produced two contradictory cost claims from the same corpus. Every figure below is +computed per family first. + +**A second, subtler trap.** `completion_tokens` is missing for exactly the cells that were +never graded, and looping is the dominant cause of that. Missingness is therefore *not* +random — it is concentrated in the runs that would have been most expensive. A family is +counted as **cost-comparable** only when both sides retain ≥3 cells with cost data *and* +≥50% coverage of that side's cells in the family. Non-comparable families are shown but +excluded from the summary ratio. + +### P1 — 3.6 no-think T0.3/p0.8/pp0 vs 3.8 no-think T0.3/p0.8/pp0 + +Token-data coverage: 3.6 94% (114/121 cells) · 3.8 68% (65/95 cells). Cost-comparable families: **9 of 12**. Excluded: `p1_bugfix`, `p1_testwrite`, `p3_market`. + +| Family | 3.6 med tok (n) | 3.8 med tok (n) | tok ratio | 3.6 med s | 3.8 med s | s ratio | comparable | +|---|---:|---:|---:|---:|---:|---:|:--:| +| `p1_bugfix` | 31,183 (18) | 102,609 (1) | 3.29 | 894.2 (18) | 4,860.3 (1) | 5.44 | **no** | +| `p1_refactor` | 11,033 (9) | 11,468 (5) | 1.04 | 212.7 (9) | 195.0 (5) | 0.92 | yes | +| `p1_testwrite` | 36,840 (10) | 30,596 (2) | 0.83 | 638.1 (10) | 684.0 (2) | 1.07 | **no** | +| `p2_ci` | 4,600 (9) | 3,748 (7) | 0.81 | 82.6 (9) | 71.7 (7) | 0.87 | yes | +| `p2_extract` | 2,871 (9) | 2,666 (7) | 0.93 | 45.5 (9) | 42.8 (7) | 0.94 | yes | +| `p2_hallucination` | 5,101 (9) | 4,377 (7) | 0.86 | 83.1 (9) | 72.4 (7) | 0.87 | yes | +| `p2_triage` | 7,707 (9) | 6,289 (7) | 0.82 | 118.3 (9) | 97.0 (7) | 0.82 | yes | +| `p3_business` | 9,503 (9) | 10,344 (6) | 1.09 | 146.3 (9) | 157.9 (6) | 1.08 | yes | +| `p3_doc` | 7,801 (9) | 7,108 (6) | 0.91 | 120.1 (9) | 108.5 (6) | 0.90 | yes | +| `p3_market` | 40,432 (5) | 103,162 (3) | 2.55 | 943.4 (5) | 2,914.9 (3) | 3.09 | **no** | +| `p3_pm` | 4,634 (9) | 3,308 (7) | 0.71 | 71.5 (9) | 50.7 (7) | 0.71 | yes | +| `p3_writing` | 6,427 (9) | 4,927 (7) | 0.77 | 104.0 (9) | 77.4 (7) | 0.74 | yes | + +**Median of per-family ratios (comparable families only): tokens ×0.86, wall-clock ×0.87.** + +> ⚠️ **The excluded families are the expensive ones.** Median 3.6 cost is 36,840 tokens +> across the excluded families versus 6,427 across the retained ones (×5.7). Because +> cost data goes missing exactly when a run loops, and 3.8 loops most on the +> long-horizon families, the ×0.86 summary ratio above is computed over the *cheap* +> tail of the task set. **It must not be read as "3.8 is cheaper overall"** — the +> comparison is silent on precisely the families where 3.8's cost would be worst. + +| Tokens per successful deliverable | 3.6 | 3.8 | +|---|---:|---:| +| Total completion tokens recorded | 1,680,139 | 815,989 | +| Cells with token data | 114/121 | 65/95 | +| Passing cells | 81 | 51 | +| **Tokens per success** | **20,742** (lower bound) | **16,000** (lower bound) | +| Tokens burned on cells that never got graded | 0 | 0 | + +- 3.6: 7 of 121 cells have no token accounting (all of them ungraded), so total tokens and tokens-per-success are UNDERSTATED for this arm. +- 3.8: 30 of 95 cells have no token accounting (all of them ungraded), so total tokens and tokens-per-success are UNDERSTATED for this arm. + +### P2 — 3.6 no-think T1/p0.95/pp0 vs 3.8 no-think T0.7/p0.8/pp1.5 + +Token-data coverage: 3.6 100% (120/120 cells) · 3.8 82% (79/96 cells). Cost-comparable families: **11 of 12**. Excluded: `p3_market`. + +| Family | 3.6 med tok (n) | 3.8 med tok (n) | tok ratio | 3.6 med s | 3.8 med s | s ratio | comparable | +|---|---:|---:|---:|---:|---:|---:|:--:| +| `p1_bugfix` | 34,135 (10) | 57,052 (4) | 1.67 | 989.0 (10) | 4,126.5 (4) | 4.17 | yes | +| `p1_refactor` | 11,324 (10) | 13,288 (8) | 1.17 | 212.9 (10) | 241.6 (8) | 1.13 | yes | +| `p1_testwrite` | 33,248 (10) | 25,923 (7) | 0.78 | 563.6 (10) | 772.8 (7) | 1.37 | yes | +| `p2_ci` | 5,597 (10) | 4,090 (8) | 0.73 | 98.4 (10) | 76.9 (8) | 0.78 | yes | +| `p2_extract` | 1,502 (10) | 2,980 (8) | 1.98 | 24.9 (10) | 49.7 (8) | 1.99 | yes | +| `p2_hallucination` | 6,214 (10) | 4,240 (8) | 0.68 | 104.1 (10) | 77.1 (8) | 0.74 | yes | +| `p2_triage` | 7,010 (10) | 5,998 (8) | 0.86 | 107.8 (10) | 99.2 (8) | 0.92 | yes | +| `p3_business` | 7,404 (10) | 9,501 (6) | 1.28 | 112.0 (10) | 158.4 (6) | 1.41 | yes | +| `p3_doc` | 8,097 (10) | 10,830 (8) | 1.34 | 124.2 (10) | 177.8 (8) | 1.43 | yes | +| `p3_market` | 32,928 (10) | 52,876 (2) | 1.61 | 785.0 (10) | 1,491.3 (2) | 1.90 | **no** | +| `p3_pm` | 4,182 (10) | 4,530 (8) | 1.08 | 65.2 (10) | 74.8 (8) | 1.15 | yes | +| `p3_writing` | 7,104 (10) | 14,931 (4) | 2.10 | 113.5 (10) | 254.2 (4) | 2.24 | yes | + +**Median of per-family ratios (comparable families only): tokens ×1.17, wall-clock ×1.37.** + +> ⚠️ **The excluded families are the expensive ones.** Median 3.6 cost is 32,928 tokens +> across the excluded families versus 7,104 across the retained ones (×4.6). Because +> cost data goes missing exactly when a run loops, and 3.8 loops most on the +> long-horizon families, the ×1.17 summary ratio above is computed over the *cheap* +> tail of the task set. **It must not be read as "3.8 is cheaper overall"** — the +> comparison is silent on precisely the families where 3.8's cost would be worst. + +| Tokens per successful deliverable | 3.6 | 3.8 | +|---|---:|---:| +| Total completion tokens recorded | 1,628,275 | 1,661,867 | +| Cells with token data | 120/120 | 79/96 | +| Passing cells | 95 | 48 | +| **Tokens per success** | **17,140** | **34,622** (lower bound) | +| Tokens burned on cells that never got graded | 0 | 0 | + +- 3.8: 17 of 96 cells have no token accounting (all of them ungraded), so total tokens and tokens-per-success are UNDERSTATED for this arm. + +### P3 — 3.6 think T0.3/p0.8/pp0 vs 3.8 think T0.3/p0.8/pp0 + +Token-data coverage: 3.6 99% (107/108 cells) · 3.8 94% (48/51 cells). Cost-comparable families: **11 of 12**. Excluded: `p3_market`. + +| Family | 3.6 med tok (n) | 3.8 med tok (n) | tok ratio | 3.6 med s | 3.8 med s | s ratio | comparable | +|---|---:|---:|---:|---:|---:|---:|:--:| +| `p1_bugfix` | 32,044 (9) | 86,053 (5) | 2.69 | 899.6 (9) | 2,378.7 (5) | 2.64 | yes | +| `p1_refactor` | 12,387 (9) | 27,969 (4) | 2.26 | 220.2 (9) | 666.1 (4) | 3.03 | yes | +| `p1_testwrite` | 42,849 (9) | 31,491 (5) | 0.73 | 730.9 (9) | 940.5 (5) | 1.29 | yes | +| `p2_ci` | 6,675 (9) | 6,881 (4) | 1.03 | 115.6 (9) | 113.8 (4) | 0.98 | yes | +| `p2_extract` | 3,865 (9) | 4,870 (4) | 1.26 | 60.7 (9) | 112.5 (4) | 1.85 | yes | +| `p2_hallucination` | 7,234 (9) | 8,621 (4) | 1.19 | 117.8 (9) | 135.8 (4) | 1.15 | yes | +| `p2_triage` | 7,681 (9) | 9,907 (4) | 1.29 | 116.9 (9) | 149.9 (4) | 1.28 | yes | +| `p3_business` | 8,137 (9) | 17,730 (4) | 2.18 | 122.5 (9) | 269.4 (4) | 2.20 | yes | +| `p3_doc` | 7,304 (9) | 42,730 (4) | 5.85 | 111.9 (9) | 686.8 (4) | 6.14 | yes | +| `p3_market` | 26,516 (8) | 78,524 (2) | 2.96 | 529.8 (8) | 1,779.8 (2) | 3.36 | **no** | +| `p3_pm` | 3,729 (9) | 10,202 (4) | 2.74 | 58.0 (9) | 154.9 (4) | 2.67 | yes | +| `p3_writing` | 6,315 (9) | 17,228 (4) | 2.73 | 99.7 (9) | 265.3 (4) | 2.66 | yes | + +**Median of per-family ratios (comparable families only): tokens ×2.18, wall-clock ×2.20.** + +> ⚠️ **The excluded families are the expensive ones.** Median 3.6 cost is 26,516 tokens +> across the excluded families versus 7,304 across the retained ones (×3.6). Because +> cost data goes missing exactly when a run loops, and 3.8 loops most on the +> long-horizon families, the ×2.18 summary ratio above is computed over the *cheap* +> tail of the task set. **It must not be read as "3.8 is cheaper overall"** — the +> comparison is silent on precisely the families where 3.8's cost would be worst. + +| Tokens per successful deliverable | 3.6 | 3.8 | +|---|---:|---:| +| Total completion tokens recorded | 1,540,103 | 1,392,807 | +| Cells with token data | 107/108 | 48/51 | +| Passing cells | 67 | 33 | +| **Tokens per success** | **22,987** (lower bound) | **42,206** (lower bound) | +| Tokens burned on cells that never got graded | 0 | 0 | + +- 3.6: 1 of 108 cell has no token accounting (all of them ungraded), so total tokens and tokens-per-success are UNDERSTATED for this arm. +- 3.8: 3 of 51 cells have no token accounting (all of them ungraded), so total tokens and tokens-per-success are UNDERSTATED for this arm. + +### P4 — 3.6 think T1/p0.95/pp0 vs 3.8 think T1/p0.95/pp0 + +Token-data coverage: 3.6 100% (120/120 cells) · 3.8 100% (72/72 cells). Cost-comparable families: **12 of 12**. + +| Family | 3.6 med tok (n) | 3.8 med tok (n) | tok ratio | 3.6 med s | 3.8 med s | s ratio | comparable | +|---|---:|---:|---:|---:|---:|---:|:--:| +| `p1_bugfix` | 34,634 (10) | 119,290 (6) | 3.44 | 910.8 (10) | 3,073.9 (6) | 3.37 | yes | +| `p1_refactor` | 12,830 (10) | 46,639 (6) | 3.64 | 234.1 (10) | 828.4 (6) | 3.54 | yes | +| `p1_testwrite` | 38,600 (10) | 75,152 (6) | 1.95 | 657.0 (10) | 1,487.0 (6) | 2.26 | yes | +| `p2_ci` | 6,800 (10) | 16,928 (6) | 2.49 | 119.7 (10) | 277.4 (6) | 2.32 | yes | +| `p2_extract` | 4,722 (10) | 10,504 (6) | 2.22 | 73.8 (10) | 162.4 (6) | 2.20 | yes | +| `p2_hallucination` | 7,320 (10) | 16,217 (6) | 2.22 | 120.2 (10) | 259.0 (6) | 2.15 | yes | +| `p2_triage` | 9,285 (10) | 18,494 (6) | 1.99 | 140.7 (10) | 284.9 (6) | 2.02 | yes | +| `p3_business` | 12,102 (10) | 29,340 (6) | 2.42 | 182.9 (10) | 465.1 (6) | 2.54 | yes | +| `p3_doc` | 10,542 (10) | 42,326 (6) | 4.02 | 160.9 (10) | 686.3 (6) | 4.27 | yes | +| `p3_market` | 30,441 (10) | 116,924 (6) | 3.84 | 634.0 (10) | 3,293.9 (6) | 5.20 | yes | +| `p3_pm` | 5,020 (10) | 21,210 (6) | 4.23 | 78.4 (10) | 329.5 (6) | 4.20 | yes | +| `p3_writing` | 8,030 (10) | 36,736 (6) | 4.57 | 127.5 (10) | 592.6 (6) | 4.65 | yes | + +**Median of per-family ratios (comparable families only): tokens ×2.97, wall-clock ×2.96.** + +| Tokens per successful deliverable | 3.6 | 3.8 | +|---|---:|---:| +| Total completion tokens recorded | 1,798,295 | 2,989,297 | +| Cells with token data | 120/120 | 72/72 | +| Passing cells | 105 | 54 | +| **Tokens per success** | **17,127** | **55,357** | +| Tokens burned on cells that never got graded | 0 | 0 | + +--- + +## 5. Effort ladder within 3.8 thinking + +`effort` is populated **only** for 3.8 thinking cells; every 3.6 cell and every no-think cell +has an empty effort field, so there is no 3.6 effort ladder to compare against. The ladder is +broken out by sampler as well as pooled, because the two 3.8 thinking groups have very +different effort mixes (T0.3: 14/24/13 low/med/xhigh; T1: 12/12/48) and pooling them +confounds effort with sampler. + +| Sampler | Effort | Cells | Families | Graded | Loops | Pass (all cells) | Wilson 95% CI | Pass (graded) | Median tokens | Median s | +|---|---|---:|---:|---:|---:|---|---|---|---:|---:| +| ALL (pooled) | low | 26 | 12 | 26 | 0 | 21/26 80.8% | [62.1, 91.5] | 21/26 80.8% | 16,333 | 261.7 | +| ALL (pooled) | medium | 36 | 12 | 34 | 2 | 21/36 58.3% | [42.2, 72.9] | 21/34 61.8% | 13,252 | 223.8 | +| ALL (pooled) | xhigh | 61 | 12 | 60 | 0 | 45/61 73.8% | [61.6, 83.2] | 45/60 75.0% | 36,736 | 592.6 | +| T0.3/p0.8/pp0 | low | 14 | 12 | 14 | 0 | 11/14 78.6% | [52.4, 92.4] | 11/14 78.6% | 17,366 | 265.5 | +| T0.3/p0.8/pp0 | medium | 24 | 12 | 22 | 2 | 13/24 54.2% | [35.1, 72.1] | 13/22 59.1% | 10,898 | 223.8 | +| T0.3/p0.8/pp0 | xhigh | 13 | 12 | 12 | 0 | 9/13 69.2% | [42.4, 87.3] | 9/12 75.0% | 32,094 | 501.9 | +| T1/p0.95/pp0 | low | 12 | 12 | 12 | 0 | 10/12 83.3% | [55.2, 95.3] | 10/12 83.3% | 14,514 | 231.9 | +| T1/p0.95/pp0 | medium | 12 | 12 | 12 | 0 | 8/12 66.7% | [39.1, 86.2] | 8/12 66.7% | 14,358 | 225.6 | +| T1/p0.95/pp0 | xhigh | 48 | 12 | 48 | 0 | 36/48 75.0% | [61.2, 85.1] | 36/48 75.0% | 38,506 | 623.3 | + +### Pairwise effort steps (Δ = higher effort − lower effort, all-cells pass rate) + +| Sampler | Step | Lower | Higher | Δ | Newcombe 95% CI | Fisher p | +|---|---|---|---|---:|---|---:| +| T0.3/p0.8/pp0 | low → medium | 11/14 | 13/24 | -24.4 pp | [-48.0, 7.3] | 0.175 | +| T0.3/p0.8/pp0 | medium → xhigh | 13/24 | 9/13 | +15.1 pp | [-17.2, 41.4] | 0.4908 | +| T0.3/p0.8/pp0 | low → xhigh | 11/14 | 9/13 | -9.3 pp | [-39.6, 22.5] | 0.6776 | +| T1/p0.95/pp0 | low → medium | 10/12 | 8/12 | -16.7 pp | [-46.8, 17.6] | 0.6404 | +| T1/p0.95/pp0 | medium → xhigh | 8/12 | 36/48 | +8.3 pp | [-15.6, 37.7] | 0.7163 | +| T1/p0.95/pp0 | low → xhigh | 10/12 | 36/48 | -8.3 pp | [-26.6, 21.6] | 0.7134 | +| ALL (pooled) | low → medium | 21/26 | 21/36 | -22.4 pp | [-41.8, 1.2] | 0.09799 | +| ALL (pooled) | medium → xhigh | 21/36 | 45/61 | +15.4 pp | [-3.5, 34.1] | 0.1229 | +| ALL (pooled) | low → xhigh | 21/26 | 45/61 | -7.0 pp | [-23.2, 13.9] | 0.5903 | + +**Not one effort step is statistically distinguishable** (every Fisher p ≥ 0.098; every +Newcombe CI straddles zero). Cell counts per rung are 12–48, which cannot resolve the +7.0–24.4 pp differences observed. The apparent non-monotonicity — `medium` scoring lowest at +both samplers — is **not** supported as a real effect and should not be reported as one. + +What the ladder *does* show cleanly is cost: median tokens rise roughly 1.8–2.7× from `low` +to `xhigh` (T0.3: 17,366 → 32,094; T1: 14,514 → 38,506), with no accompanying pass-rate gain. +On this corpus, raising 3.8's thinking effort buys tokens, not quality. + +--- + +## 6. Q8_0 quantization control + +Q8_0 arm at freeze #2 is 19 cells over all 12 families: 2 replicates on the 7 phase-1/phase-2 families, 1 on the 5 phase-3 families. + +> **What changed at freeze #2.** At freeze #1 this arm was 11 cells with zero graded and +> carried no quality information at all. At freeze #2 it is 19 cells with 8 graded +> (5 PASS, 3 FAIL), so it now supports **provisional rates with wide intervals** — +> loop rate 6/19 31.6% [15.4, 54.0] and graded-only pass rate 5/8 62.5% [30.6, 86.3] (Wilson 95%). +> The loop interval excludes zero, so "the no-think loop occurs at Q8_0 at a real, +> non-negligible rate" is now a provisional *rate* claim rather than an existence-only +> observation. The quality numbers stay **descriptive only**: on n=8 graded cells no +> pass-rate comparison against a Q4 arm can separate quantization from noise, and none is +> asserted. 19 of the 19 cells ran to a terminal state and 13 produced full token accounting. + +> **Unscored qualitative evidence, disclosed and excluded.** `p3_doc_qwen38q8-nothink-matched_v2` +> was quarantined in-flight at freeze #2 and is **not a row in the frozen CSV**: it was +> observed in a rewrite loop — 139 iterations rewriting brief.md, context grown to ~228k tokens when quarantined. +> This is a different failure shape from the identical-call loop that `looped_freq30` / +> `looped_run30` count, it is UNSCORED, and it is excluded from every rate and every +> denominator in this document. It is recorded because a second loop subclass at Q8_0 is +> qualitatively relevant to the quantization question even though it carries no +> statistical weight. + +Per-cell detail for all 19 Q8_0 cells as of freeze #2: + +| Family | Rep | Verdict | Graded | Terminal | Loop freq30 | Loop run30 | Tokens | Elapsed s | +|---|---:|---|---:|---:|---:|---:|---:|---:| +| `p1_bugfix` | 1 | *(none)* | 0 | 1 | 1 | 1 | n/a | n/a | +| `p1_bugfix` | 2 | *(none)* | 0 | 1 | 0 | 0 | n/a | n/a | +| `p1_refactor` | 1 | PASS | 1 | 1 | 0 | 0 | 11,435 | 259.2 | +| `p1_refactor` | 2 | *(none)* | 0 | 1 | 1 | 1 | n/a | n/a | +| `p1_testwrite` | 1 | *(none)* | 0 | 1 | 1 | 1 | n/a | n/a | +| `p1_testwrite` | 2 | *(none)* | 0 | 1 | 1 | 0 | 16,665 | 597.3 | +| `p2_ci` | 1 | PASS | 1 | 1 | 0 | 0 | 4,136 | 96.2 | +| `p2_ci` | 2 | *(none)* | 0 | 1 | 0 | 0 | 4,160 | 98.7 | +| `p2_extract` | 1 | PASS | 1 | 1 | 0 | 0 | 2,422 | 53.0 | +| `p2_extract` | 2 | *(none)* | 0 | 1 | 0 | 0 | 2,435 | 54.5 | +| `p2_hallucination` | 1 | PASS | 1 | 1 | 0 | 0 | 4,316 | 97.0 | +| `p2_hallucination` | 2 | *(none)* | 0 | 1 | 0 | 0 | 4,446 | 101.5 | +| `p2_triage` | 1 | PASS | 1 | 1 | 0 | 0 | 6,771 | 144.5 | +| `p2_triage` | 2 | *(none)* | 0 | 1 | 0 | 0 | 7,434 | 159.1 | +| `p3_business` | 1 | FAIL | 1 | 1 | 0 | 0 | 8,334 | 176.6 | +| `p3_doc` | 1 | FAIL | 1 | 1 | 0 | 0 | 15,304 | 326.8 | +| `p3_market` | 1 | *(none)* | 0 | 1 | 1 | 1 | n/a | n/a | +| `p3_pm` | 1 | FAIL | 1 | 1 | 0 | 0 | 2,935 | 65.2 | +| `p3_writing` | 1 | *(none)* | 0 | 1 | 1 | 1 | n/a | n/a | + +Delivery and (provisionally) graded-only quality against Q4 arms restricted to the same +12 families: + +**vs 3.8 Q4 (same sampler, same model — isolates quantization)** + +| Metric | Q4 comparator | Q8_0 | Δ (Q8−Q4) | Newcombe 95% CI | Fisher p | +|---|---|---|---:|---|---:| +| Terminal | 93/95 97.9% | 19/19 100.0% | +2.1 pp | [-14.8, 7.4] | 1 | +| Loop (freq30) | 29/95 30.5% | 6/19 31.6% | +1.1 pp | [-17.9, 25.0] | 1 | +| Loop (run30) | 23/95 24.2% | 5/19 26.3% | +2.1 pp | [-15.2, 25.8] | 1 | +| Pass, graded only (PROVISIONAL) | 51/65 78.5% | 5/8 62.5% | -16.0 pp | [-48.9, 10.5] | 0.3782 | + +Token cost, descriptive only (Q8_0 carries 1–2 replicates per family, so the +≥3-cells-per-side comparability rule of §4 is generally not satisfied and no summary +ratio is claimed): 9 families have data on both sides; median per-family token ratio Q8/Q4 = **×1.00** (range ×0.54–×2.15). + +**vs 3.6 Q4 (same sampler, different model)** + +| Metric | Q4 comparator | Q8_0 | Δ (Q8−Q4) | Newcombe 95% CI | Fisher p | +|---|---|---|---:|---|---:| +| Terminal | 118/121 97.5% | 19/19 100.0% | +2.5 pp | [-14.4, 7.0] | 1 | +| Loop (freq30) | 3/121 2.5% | 6/19 31.6% | +29.1 pp | [12.3, 51.6] | 0.00019 **\*** | +| Loop (run30) | 3/121 2.5% | 5/19 26.3% | +23.8 pp | [8.6, 46.4] | 0.00119 **\*** | + +Token cost, descriptive only (Q8_0 carries 1–2 replicates per family, so the +≥3-cells-per-side comparability rule of §4 is generally not satisfied and no summary +ratio is claimed): 9 families have data on both sides; median per-family token ratio Q8/Q4 = **×0.88** (range ×0.45–×1.96). + +> **Any loop difference in this table is a model effect, not a quantization +> effect.** This row compares 3.8-Q8_0 against 3.6-Q4 — the models differ as well as the +> quantization — so it simply reproduces the no-think looping gap already established in +> §1. Only the preceding table (3.8-Q8_0 vs 3.8-Q4, same model and sampler) isolates +> quantization. + +With 19 cells and 1–2 replicates per family, **every interval here still spans tens of +percentage points**. What the arm now establishes: the identical-call loop occurs at Q8_0 at +a rate whose Wilson interval excludes zero, so the loop is not an artifact of the UD-Q4_K_XL +quantization. What it still cannot do: support any quality verdict, or attribute (or +exonerate) the delivery regression as a quantization effect — there is still no matched 3.6 +Q8_0 arm, and the graded subset is n=8. The graded outcomes above are published as data +for the next freeze to build on, not as findings. + +--- + +## 7. Sensitivity: does the 3.6-vs-3.8 delta survive rescoring? + +Seven scorings per comparison. The first six are the requested set; `graded_only` is added +because it is the natural upper bound on charity toward a model whose failures are delivery +failures. + +- **`loops_as_failure`** — All cells in the group; passed==1 is success. Loops and ungraded cells score 0. This is the headline scoring. +- **`loops_excluded`** — Cells with looped_freq30==1 dropped entirely, then pass rate over the remainder. Charitable to whichever model loops more. NOTE: cells that are ungraded WITHOUT having looped are retained here and score 0. +- **`loops_half_credit`** — Score 1.0 for pass, 0.5 for a non-passing looped cell, 0.0 otherwise; compare MEAN scores. Scores are not 0/1 so Wilson and Fisher do not apply; CI is a z-interval on the difference of means. +- **`graded_only`** — Only graded==1 cells. Removes every delivery failure (loops and ungraded alike) and measures output quality conditional on delivery. Maximally charitable to a model that fails by not delivering. +- **`best_of_n_per_family`** — One Bernoulli trial per SHARED family: 1 if any cell in that group/family passed. Measures 'can the model ever do this task', not reliability. n = number of families, so it is structurally low-powered. +- **`first_replicate_only`** — Only rows with replicate==1 (one run per family per arm). Removes any weighting from unequal repeat depth; retains one row per arm, so n is families x arms, not families. +- **`depth_matched`** — Per shared family, the first k=min(n_a,n_b) cells from each side ordered by (arm, replicate, cell). Both sides then have identical family x depth composition. + +### P1 — 3.6 no-think T0.3/p0.8/pp0 vs 3.8 no-think T0.3/p0.8/pp0 + +| Scoring | 3.6 | 3.8 | Δ (3.8−3.6) | 95% CI on Δ | Fisher p | +|---|---|---|---:|---|---:| +| `loops_as_failure` | 81/121 66.9% | 51/95 53.7% | -13.3 pp | [-25.9, -0.2] | 0.05052 | +| `loops_excluded` | 81/118 68.6% | 51/66 77.3% | +8.6 pp | [-5.1, 20.9] | 0.2358 | +| `loops_half_credit` | mean 0.682 (n=121) | mean 0.689 (n=95) | +0.8 pp | [-10.4, 11.9] | n/a | +| `graded_only` | 81/114 71.1% | 51/65 78.5% | +7.4 pp | [-6.3, 19.5] | 0.2959 | +| `best_of_n_per_family` | 11/12 91.7% | 8/12 66.7% | -25.0 pp | [-53.4, 8.4] | 0.3168 | +| `first_replicate_only` | 8/13 61.5% | 8/13 61.5% | +0.0 pp | [-33.3, 33.3] | 1 | +| `depth_matched` | 62/94 66.0% | 51/94 54.3% | -11.7 pp | [-25.0, 2.3] | 0.1361 | + +Range of Δ across the seven scorings: **-25.0 pp to +8.6 pp**. Sign is **NOT consistent — it flips**. Significant in **0 of 6** scorings that admit a Fisher test. + +### P2 — 3.6 no-think T1/p0.95/pp0 vs 3.8 no-think T0.7/p0.8/pp1.5 + +| Scoring | 3.6 | 3.8 | Δ (3.8−3.6) | 95% CI on Δ | Fisher p | +|---|---|---|---:|---|---:| +| `loops_as_failure` | 95/120 79.2% | 48/96 50.0% | -29.2 pp | [-40.8, -16.4] | 1.1e-05 **\*** | +| `loops_excluded` | 95/120 79.2% | 46/68 67.6% | -11.5 pp | [-24.9, 1.3] | 0.114 | +| `loops_half_credit` | mean 0.792 (n=120) | mean 0.635 (n=96) | -15.6 pp | [-26.6, -4.7] | n/a | +| `graded_only` | 95/120 79.2% | 48/79 60.8% | -18.4 pp | [-31.1, -5.5] | 0.006102 **\*** | +| `best_of_n_per_family` | 11/12 91.7% | 9/12 75.0% | -16.7 pp | [-45.7, 14.8] | 0.5901 | +| `first_replicate_only` | 9/12 75.0% | 12/24 50.0% | -25.0 pp | [-49.6, 8.8] | 0.2821 | +| `depth_matched` | 77/96 80.2% | 48/96 50.0% | -30.2 pp | [-42.1, -16.9] | 1.8e-05 **\*** | + +Range of Δ across the seven scorings: **-30.2 pp to -11.5 pp**. Sign is consistently negative (3.8 worse). Significant in **3 of 6** scorings that admit a Fisher test. + +### P3 — 3.6 think T0.3/p0.8/pp0 vs 3.8 think T0.3/p0.8/pp0 + +| Scoring | 3.6 | 3.8 | Δ (3.8−3.6) | 95% CI on Δ | Fisher p | +|---|---|---|---:|---|---:| +| `loops_as_failure` | 67/108 62.0% | 33/51 64.7% | +2.7 pp | [-13.5, 17.7] | 0.8607 | +| `loops_excluded` | 67/107 62.6% | 33/49 67.3% | +4.7 pp | [-11.7, 19.6] | 0.5949 | +| `loops_half_credit` | mean 0.625 (n=108) | mean 0.667 (n=51) | +4.2 pp | [-11.5, 19.9] | n/a | +| `graded_only` | 67/107 62.6% | 33/48 68.8% | +6.1 pp | [-10.4, 20.9] | 0.5863 | +| `best_of_n_per_family` | 9/12 75.0% | 12/12 100.0% | +25.0 pp | [-4.1, 53.2] | 0.2174 | +| `first_replicate_only` | 8/12 66.7% | 26/36 72.2% | +5.6 pp | [-19.8, 35.6] | 0.726 | +| `depth_matched` | 34/51 66.7% | 33/51 64.7% | -2.0 pp | [-19.8, 16.0] | 1 | + +Range of Δ across the seven scorings: **-2.0 pp to +25.0 pp**. Sign is **NOT consistent — it flips**. Significant in **0 of 6** scorings that admit a Fisher test. + +### P4 — 3.6 think T1/p0.95/pp0 vs 3.8 think T1/p0.95/pp0 + +| Scoring | 3.6 | 3.8 | Δ (3.8−3.6) | 95% CI on Δ | Fisher p | +|---|---|---|---:|---|---:| +| `loops_as_failure` | 105/120 87.5% | 54/72 75.0% | -12.5 pp | [-24.6, -1.4] | 0.03081 **\*** | +| `loops_excluded` | 105/120 87.5% | 54/72 75.0% | -12.5 pp | [-24.6, -1.4] | 0.03081 **\*** | +| `loops_half_credit` | mean 0.875 (n=120) | mean 0.750 (n=72) | -12.5 pp | [-24.2, -0.8] | n/a | +| `graded_only` | 105/120 87.5% | 54/72 75.0% | -12.5 pp | [-24.6, -1.4] | 0.03081 **\*** | +| `best_of_n_per_family` | 12/12 100.0% | 10/12 83.3% | -16.7 pp | [-44.8, 10.4] | 0.4783 | +| `first_replicate_only` | 12/12 100.0% | 36/48 75.0% | -25.0 pp | [-38.8, 1.3] | 0.1006 | +| `depth_matched` | 63/72 87.5% | 54/72 75.0% | -12.5 pp | [-25.0, 0.3] | 0.08639 | + +Range of Δ across the seven scorings: **-25.0 pp to -12.5 pp**. Sign is consistently negative (3.8 worse). Significant in **3 of 6** scorings that admit a Fisher test. + +### What survives every scoring + +Computed directly from the table above rather than asserted: + +- **P1** (3.6 no-think T0.3/p0.8/pp0 vs 3.8 no-think T0.3/p0.8/pp0): **Not robust — the sign flips with the scoring rule.** No quality claim in either direction survives. Significant in 0 of 6. +- **P2** (3.6 no-think T1/p0.95/pp0 vs 3.8 no-think T0.7/p0.8/pp1.5): **Directionally robust, not uniformly significant.** 3.8 is worse under all seven scorings, significant in 3 of 6. +- **P3** (3.6 think T0.3/p0.8/pp0 vs 3.8 think T0.3/p0.8/pp0): **Not robust — the sign flips with the scoring rule.** No quality claim in either direction survives. Significant in 0 of 6. +- **P4** (3.6 think T1/p0.95/pp0 vs 3.8 think T1/p0.95/pp0): **Directionally robust, not uniformly significant.** 3.8 is worse under all seven scorings, significant in 3 of 6. + +**A confound in P2, and why it cannot carry the effect.** P2's vendor-matched design ties +model identity to `presence_penalty`: the 3.8 vendor sampler carries pp=1.5 while 3.6's +carries pp=0, so any P2 delta could in principle be a presence-penalty artifact rather than +a model difference. That objection fails against the delivery finding: P1 holds pp=0 on +**both** sides and still shows the +28.0 pp loop delta (4.9e-09), so the looping regression +appears with the confound removed entirely. P2's *quality* deltas have no pp-free +replication at the same (no-think) mode — P1, the pp=0 no-think pair, flips sign across +scorings — so the confound caveat does apply to P2's pass rates. One more reason the loop +finding, not the pass-rate finding, is this document's headline. + +Three statements survive **every scoring rule**, in every comparison where they can be +measured at all: + +1. **3.8's no-think looping is real and large.** Loop rate (freq30) is +28.0 pp in P1 (4.9e-09) and +29.2 pp in + P2 (1.1e-11), with CIs excluding zero. It is the single largest and most reliable effect in the + corpus, and it is not sensitive to any scoring choice because it is measured upstream of grading. +2. **The looping is confined to no-think mode.** In thinking mode the loop rate collapses to + 2/51 and 0/72 for 3.8 (P3, P4) against 1/108 and 0/120 for 3.6 — no significant difference + and, at T1/p0.95/pp0, zero loops on either side. +3. **3.8 costs more in thinking mode.** Median per-family token ratio is ×2.18 (P3) and ×2.97 + (P4), with wall-clock tracking it (×2.20, ×2.96), and no pass-rate gain to show for it. + +The claim that does **not** survive is any clean statement that 3.8 produces worse *output*. +In P1 the sign flips: 3.8 is 13.3 pp worse end-to-end but 7.4 pp **better** among graded +cells. Both cannot be summarized as "3.8 is worse at the task" — the honest reading is +that 3.8 fails more often by not finishing, not by finishing badly. + +--- + +## 8. Grader-defect exposure (why these tables are labelled *as-graded*) + +Every pass rate in this document is **as-graded**: it uses the `passed` bit exactly as the +original graders produced it. Three grader defects are verified in this corpus. Per repo +convention they are corrected in a **separate non-destructive overlay**, never by editing +`grade.json` or the task briefs, so longitudinal comparability is preserved. The tables here +are the baseline that overlay will be applied to. + +Diagnostics below are computed from raw `grade.json` files (the frozen CSV carries only the +final pass/fail bit, so diagnosing *why* a cell failed requires the grader's score fields). +Cell→model identity is joined from the frozen CSV by cell id, and cells that were ungraded +at the freeze are excluded even where a post-freeze `grade.json` now exists on disk (see the +drift note in §6). **No headline number above depends on this section.** + +### Exposure: cells sitting in a defect-affected family + +| Group | n | D1 length-gated (4 families) | D2 `p3_pm` | D3 `p2_triage` | +|---|---:|---|---|---| +| `A36` | 121 | n=36, graded=36, pass=18 | n=9, graded=9, pass=1 | n=9, graded=9, pass=0 | +| `A38` | 95 | n=28, graded=26, pass=20 | n=7, graded=7, pass=7 | n=7, graded=7, pass=7 | +| `V36` | 120 | n=40, graded=40, pass=21 | n=10, graded=10, pass=1 | n=10, graded=10, pass=7 | +| `V38` | 96 | n=32, graded=26, pass=11 | n=8, graded=8, pass=8 | n=8, graded=8, pass=5 | +| `B36` | 108 | n=36, graded=36, pass=10 | n=9, graded=9, pass=0 | n=9, graded=9, pass=0 | +| `B38` | 51 | n=16, graded=16, pass=5 | n=4, graded=4, pass=1 | n=4, graded=4, pass=4 | +| `C36` | 120 | n=40, graded=40, pass=29 | n=10, graded=10, pass=2 | n=10, graded=10, pass=10 | +| `C38` | 72 | n=24, graded=24, pass=6 | n=6, graded=6, pass=5 | n=6, graded=6, pass=6 | +| `Q8` | 19 | n=4, graded=3, pass=0 | n=1, graded=1, pass=0 | n=2, graded=1, pass=1 | + +### D1 — word-gate tokenizer mismatch · correction favours **3.8** + +| Family | 3.6 pass/n | 3.6 over word limit | 3.8 pass/n | 3.8 over word limit | +|---|---:|---:|---:|---:| +| `p3_business` | 35/38 | 3 | 4/23 | 18 | +| `p3_doc` | 28/38 | 10 | 9/25 | 16 | + +In `p3_business` **every** FAIL on both sides is a word-limit FAIL (3.6: 3/3, 3.8: 18/18); +the substantive `stance_pushback` criterion was met by every graded cell of both models. + +Overshoot sizes in `p3_doc` against its 700-word ceiling, i.e. how far over the grader's own +counter each failing deliverable landed: + +| Model | Over-limit cells | Word counts | Overshoot range | Median overshoot | +|---|---:|---|---:|---:| +| 3.6 | 10 | 707, 707, 707, 707, 707, 707, 707, 707, 709, 713 | 7–13 words | 7.0 words | +| 3.8 | 16 | 701, 702, 708, 718, 719, 720, 722, 722, 722, 723, 725, 726, 727, 730, 733, 772 | 1–72 words | 22.0 words | + +The bulk of these are single-digit to low-double-digit overruns on a 700-word budget — 3.6's +largest is 13 words over and its median is 7 — which is well inside the disagreement between +the two counters. That is the substance of D1: the graders count `\b\w+\b` while both models +budgeted with shell `wc -w`, and neither counter is ground truth. **The tail runs longer on +3.8's side** — its worst `p3_doc` overshoot is 72 words (772 against a 700 ceiling) versus +3.6's 13 — so the overlay must report per-cell outcomes rather than assume a blanket +reversal; correcting D1 should be expected to rescue most, though not necessarily all, of +3.8's length failures. + +### D2 — `p3_pm` risk-keyword literalism · correction favours **3.6** + +| Model | Pass/n | FAILs blocked by risk_recall < 3 | risk_recall distribution | +|---|---:|---:|---| +| 3.6 | 4/38 | 34 | 2 risks: 34 cells, 3 risks: 4 cells | +| 3.8 | 21/26 | 5 | 2 risks: 5 cells, 3 risks: 16 cells, 4 risks: 5 cells | + +**34 of 38** 3.6 cells land on exactly `2/6` risks — one short of the `min_risks = 3` +threshold — and every one of the 34 3.6 FAILs is blocked by that rule alone. This is the +signature of a single unmatched keyword, consistent with the verified R3 literalism +(3.6 writes "legal has not responded"; the rule requires a contracted form). + +### D3 — `p2_triage` brief/ground-truth contradiction · correction favours **3.6** + +| Quantity | Value | +|---|---:| +| Graded triage cells examined | 64 | +| Cells answering `n/a` on all three spam tickets (004, 009, 021) | **64/64** | +| FAIL cells | 24 | +| FAIL cells blocked by urgency accuracy *alone* | **24/24** | +| FAIL cells that flip to PASS once the spam penalty is removed | **24/24** | +| …of which 3.6 | 21 | +| …of which 3.8 | 3 | + +> **Correction to the briefed characterization of D3.** The defect brief states that +> `p2_triage` "has zero discriminating power as graded". The frozen data does not support +> that as written: **as graded** the family splits 40 PASS / 24 FAIL and does discriminate +> (3.6 17/38 vs 3.8 23/26). The accurate statement is the reverse in time: all 64 cells take +> the identical 0.100 urgency penalty, and because the threshold is 0.700 that penalty decides +> the verdict purely by where each cell already sat — cells at 0.767–0.800 survive, cells at +> 0.633–0.667 do not. The observed split is threshold noise, not signal. It is **after** the +> correction that the family becomes uniformly passing and loses all discriminating power. + +### Net direction of the pending overlay + +The three defects **do not push the same way**: D1 materially helps 3.8, while D2 and D3 help +3.6. Anyone assuming the corrections will uniformly move the headline in one direction is +wrong. The as-graded numbers in this document should not be read as biased against either +model until the overlay is computed. + +As one bound on how much of the quality signal is defect-contaminated, restricting to the +seven families with **no** verified grader defect (`p1_bugfix`, `p1_refactor`, `p1_testwrite`, `p2_ci`, `p2_extract`, `p2_hallucination`, `p3_market`): + +| Comparison | 3.6 | 3.8 | Δ (3.8−3.6) | Newcombe 95% CI | Fisher p | +|---|---|---|---:|---|---:| +| P1 | 63/76 82.9% | 24/60 40.0% | -42.9 pp | [-56.2, -26.8] | 2.9e-07 **\*** | +| P2 | 67/70 95.7% | 32/56 57.1% | -38.6 pp | [-51.9, -24.3] | 1.2e-07 **\*** | +| P3 | 57/63 90.5% | 24/31 77.4% | -13.1 pp | [-31.0, 1.8] | 0.1133 | +| P4 | 66/70 94.3% | 42/42 100.0% | +5.7 pp | [-3.4, 13.8] | 0.295 | + +This subset is **not** a substitute for the overlay: it changes the family mix (and in P1/P2 it +is heavily confounded by 3.8's looping, which is concentrated in `p1_bugfix` and `p1_testwrite`). +It is reported only to show that the defect families are load-bearing for the as-graded totals. + +--- + +## 9. Comparisons too underpowered to report + +Screening rule, applied uniformly: a comparison is flagged **UNDERPOWERED** if either arm has +n < 30, or the Newcombe 95% CI on the difference is wider than 30 pp — i.e. the interval +cannot separate a small effect from a large one in either direction. + +This flag is about **precision, not significance**, and the two are independent. A result can +be flagged here and still be statistically significant. Read such rows as "the direction +is probably real, the magnitude +is not pinned down". Conversely, P3's non-significant rows are flagged because the data cannot +distinguish "no difference" from "a difference of 18 pp in either direction" — **P3 is not +evidence of equivalence.** + +| Comparison | Metric | Δ | 95% CI | Width | Verdict | +|---|---|---:|---|---:|---| +| P1 | `best_of_n_per_family` | -25.0 pp | [-53.4, 8.4] | 61.8 pp | UNDERPOWERED — n<30 in an arm (n_a=12, n_b=12); Newcombe CI width 61.8 pp | +| P1 | `first_replicate_only` | +0.0 pp | [-33.3, 33.3] | 66.6 pp | UNDERPOWERED — n<30 in an arm (n_a=13, n_b=13); Newcombe CI width 66.6 pp | +| P2 | `best_of_n_per_family` | -16.7 pp | [-45.7, 14.8] | 60.5 pp | UNDERPOWERED — n<30 in an arm (n_a=12, n_b=12); Newcombe CI width 60.5 pp | +| P2 | `first_replicate_only` | -25.0 pp | [-49.6, 8.8] | 58.4 pp | UNDERPOWERED — n<30 in an arm (n_a=12, n_b=24); Newcombe CI width 58.4 pp | +| P3 | `pass_all` | +2.7 pp | [-13.5, 17.7] | 31.2 pp | UNDERPOWERED — Newcombe CI width 31.2 pp | +| P3 | `pass_graded` | +6.1 pp | [-10.4, 20.9] | 31.2 pp | UNDERPOWERED — Newcombe CI width 31.2 pp | +| P3 | `loops_as_failure` | +2.7 pp | [-13.5, 17.7] | 31.2 pp | UNDERPOWERED — Newcombe CI width 31.2 pp | +| P3 | `loops_excluded` | +4.7 pp | [-11.7, 19.6] | 31.2 pp | UNDERPOWERED — Newcombe CI width 31.2 pp | +| P3 | `graded_only` | +6.1 pp | [-10.4, 20.9] | 31.2 pp | UNDERPOWERED — Newcombe CI width 31.2 pp | +| P3 | `loops_half_credit` | +4.2 pp | [-11.5, 19.9] | 31.4 pp | UNDERPOWERED — Newcombe CI width 31.4 pp | +| P3 | `best_of_n_per_family` | +25.0 pp | [-4.1, 53.2] | 57.3 pp | UNDERPOWERED — n<30 in an arm (n_a=12, n_b=12); Newcombe CI width 57.3 pp | +| P3 | `first_replicate_only` | +5.6 pp | [-19.8, 35.6] | 55.4 pp | UNDERPOWERED — n<30 in an arm (n_a=12, n_b=36); Newcombe CI width 55.4 pp | +| P3 | `depth_matched` | -2.0 pp | [-19.8, 16.0] | 35.8 pp | UNDERPOWERED — Newcombe CI width 35.8 pp | +| P4 | `best_of_n_per_family` | -16.7 pp | [-44.8, 10.4] | 55.2 pp | UNDERPOWERED — n<30 in an arm (n_a=12, n_b=12); Newcombe CI width 55.2 pp | +| P4 | `first_replicate_only` | -25.0 pp | [-38.8, 1.3] | 40.0 pp | UNDERPOWERED — n<30 in an arm (n_a=12, n_b=48); Newcombe CI width 40.0 pp | + +Explicitly **not** reportable: + +- **The Q8_0 quantization arm for any quality *verdict*.** 8 of 19 cells graded at freeze #2; + the graded-only rate 5/8 is published in §6 as provisional data with a Wilson interval + spanning tens of points — direction-finding for the next freeze, not evidence. +- **Every `best_of_n_per_family` result.** n = 12 families per side by construction; CI widths + are 55.2–61.8 pp. These rows are shown for completeness of the sensitivity grid, not as evidence. +- **Every `first_replicate_only` result** for the same reason (n = 12–48). +- **Every effort-ladder step.** All nine pairwise steps are non-significant with CI widths + of 37.1–64.3 pp. +- **All per-family rows in §3** individually (n = 4–19 per side). +- **P4's quality delta** is borderline and, more importantly, structurally biased: the shared + T1/p0.95/pp0 sampler is 3.6's vendor point and not 3.8's, so 3.8 runs off-spec by construction. + +--- + +## 10. Method appendix + +### Estimators (pure python, no scipy) + +**Wilson score interval** for a proportion — inverts the score test rather than using the +Wald form, so it stays inside [0,1] and behaves at x = 0 and x = n (several arms here are 0/n): + +``` +center = (p + z^2/(2n)) / (1 + z^2/n) +half = z * sqrt( p(1-p)/n + z^2/(4n^2) ) / (1 + z^2/n) +CI = center -/+ half z = 1.959963984540054 +``` + +**Fisher exact test**, two-sided. Conditioning on both margins, the top-left cell follows the +hypergeometric law; the p-value is the Fisher/Irwin sum-of-small-probabilities form used by +R's `fisher.test`: + +``` +P(a) = C(r1,a) * C(r2,c1-a) / C(r1+r2,c1) +p = sum of P(x) over all tables x with the same margins where P(x) <= P(observed) +``` + +Computed in exact rational arithmetic (`fractions.Fraction`), so no floating-point tie-breaking. + +**Newcombe method 10** (hybrid score) CI for a difference of proportions — propagates the two +Wilson intervals instead of a pooled Wald SE, which is what keeps it sane at boundary counts: + +``` +lower = (p1-p2) - sqrt( (p1-l1)^2 + (u2-p2)^2 ) +upper = (p1-p2) + sqrt( (u1-p1)^2 + (p2-l2)^2 ) +``` + +**Half-credit scoring** produces per-cell scores in {0, 0.5, 1}, which are not Bernoulli. +Wilson/Newcombe/Fisher are therefore **not** applied to it; that row uses a z-interval on +the difference of means (`z_diff_mean` — a normal approximation with critical value z, not +a Welch-t interval) and is labelled as such. + +### Validation + +`test_stats.py` checks each estimator against its defining equation or an exact independent +enumeration rather than against remembered published values: + +- Wilson limits are verified to be exact roots of the score equation (the score statistic + evaluates to z at both limits to 1e-9), and to bracket p̂ for every (x, n) with n ≤ 199. +- The hypergeometric pmf is verified to sum to exactly 1 in rational arithmetic, and to match + brute-force enumeration over all C(11,4) subsets. +- Fisher reproduces the tea-tasting table `[[3,1],[1,3]]` as exactly 17/35, is symmetric under + row and column swaps, and agrees with the chi-square approximation at large n. +- Newcombe is verified against its construction, verified antisymmetric under group swap, and + its zero-exclusion agrees with Fisher p < 0.05 on 98.2% of a 441-table grid. + +One real defect was found and fixed by this suite: at x = 0 and x = n the Wilson closed form +missed the exact boundary by ~1e-16, leaving the interval not quite containing p̂. Those two +cases are now snapped to exactly 0 and 1. + +### Known limitations of this dataset + +1. **Single seed.** All 802 runs use seed 42; replicates are repeat runs, not independent + seeds. Binomial CIs treat cells as independent draws, which is optimistic. +2. **Unequal and non-random depth.** Replicate counts vary by family and arm (1–19 per + family-group). The `depth_matched` scoring exists to bound the effect of this. +3. **Non-random missing cost data.** 64 of 802 cells lack `completion_tokens`; all 64 are + ungraded and most looped. Cost figures for affected arms are lower bounds. +4. **Pooled effort in 3.8 thinking.** Groups `B38` and `C38` each pool three effort levels; + their aggregate pass rates are effort-mix-weighted, not single-configuration numbers. +5. **As-graded only.** Three verified grader defects remain uncorrected here by design. +6. **Arm pooling.** Several groups pool two or more arms that share model, quant, mode and + sampler triple but were run as separate sweeps (listed in §0). + diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/results.json b/benchmarks/qwen36-vs-qwen38-27b-2026-08/results.json new file mode 100644 index 00000000..7d7c0f3f --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/results.json @@ -0,0 +1,7182 @@ +{ + "provenance": { + "source_csv": "/home/michael/mmbt-frozen-dataset-v2.csv", + "frozen_at": "2026-08-16T14:23:09Z", + "n_cells": 802, + "seeds": [ + "42" + ], + "note": "Model/quant/sampler identity taken verbatim from the frozen CSV, which sourced it from receipt.json. No live re-scan. No identity inferred from directory names.", + "columns_used": [ + "family", + "model", + "quant", + "mode", + "effort", + "regime", + "replicate", + "arm", + "graded", + "passed", + "terminal", + "looped_freq30", + "looped_run30", + "completion_tokens", + "elapsed_s" + ], + "passed_semantics": "passed==1 corresponds to verdict in {PASS, STRUCTURAL_PASS}; every ungraded cell (graded==0) has passed==0.", + "loop_nesting": "looped_run30==1 implies looped_freq30==1 (47 cells both, 22 freq-only, 0 run-only).", + "loop_nesting_counts": { + "both": 47, + "freq_only": 22, + "run_only": 0 + }, + "cost_missing": { + "n": 64, + "all_ungraded": true + }, + "depth_range": { + "min": 1, + "max": 19 + } + }, + "groups": { + "A36": { + "label": "3.6 no-think T0.3/p0.8/pp0", + "note": "off-spec for 3.6 (its card is T1/p0.95/pp0); sampler-matched to A38", + "key": { + "model": "3.6", + "quant": "UD-Q4_K_XL", + "mode": "no-think", + "regime": "T0.3/p0.8/pp0" + }, + "n": 121, + "missing": { + "graded_fail": 33, + "ungraded_looped": 3, + "ungraded_not_looped": 4, + "graded_but_looped": 0 + }, + "terminal": { + "x": 118, + "n": 121, + "p": 0.9752066115702479, + "lo": 0.9296356592312203, + "hi": 0.9915326856947302 + }, + "ungraded": { + "x": 7, + "n": 121, + "p": 0.05785123966942149, + "lo": 0.02830275348227581, + "hi": 0.11461017776053556 + }, + "loop_freq30": { + "x": 3, + "n": 121, + "p": 0.024793388429752067, + "lo": 0.008467314305269862, + "hi": 0.07036434076877973 + }, + "loop_run30": { + "x": 3, + "n": 121, + "p": 0.024793388429752067, + "lo": 0.008467314305269862, + "hi": 0.07036434076877973 + }, + "pass_graded": { + "x": 81, + "n": 114, + "p": 0.7105263157894737, + "lo": 0.6214933876201677, + "hi": 0.7858335453655833 + }, + "pass_all": { + "x": 81, + "n": 121, + "p": 0.6694214876033058, + "lo": 0.5815244079594877, + "hi": 0.7468921324054163 + }, + "n_graded": 114, + "families": 12, + "arms": [ + "qwen36-27b-udq4xl-t03-nothink", + "qwen36-nothink-offspec" + ], + "cost_missing": 7 + }, + "A38": { + "label": "3.8 no-think T0.3/p0.8/pp0", + "note": "off-spec for 3.8 (its card is T0.7/p0.8/pp1.5); sampler-matched to A36", + "key": { + "model": "3.8", + "quant": "UD-Q4_K_XL", + "mode": "no-think", + "regime": "T0.3/p0.8/pp0" + }, + "n": 95, + "missing": { + "graded_fail": 14, + "ungraded_looped": 25, + "ungraded_not_looped": 5, + "graded_but_looped": 4 + }, + "terminal": { + "x": 93, + "n": 95, + "p": 0.9789473684210527, + "lo": 0.9264588050529105, + "hi": 0.9942074931355945 + }, + "ungraded": { + "x": 30, + "n": 95, + "p": 0.3157894736842105, + "lo": 0.23103226162043158, + "hi": 0.41486531599937426 + }, + "loop_freq30": { + "x": 29, + "n": 95, + "p": 0.30526315789473685, + "lo": 0.22172919278467978, + "hi": 0.40393396069911497 + }, + "loop_run30": { + "x": 23, + "n": 95, + "p": 0.24210526315789474, + "lo": 0.16708840091876592, + "hi": 0.3371682077489622 + }, + "pass_graded": { + "x": 51, + "n": 65, + "p": 0.7846153846153846, + "lo": 0.6703343975431146, + "hi": 0.8671324230884898 + }, + "pass_all": { + "x": 51, + "n": 95, + "p": 0.5368421052631579, + "lo": 0.4370969126824784, + "hi": 0.6337235717935605 + }, + "n_graded": 65, + "families": 12, + "arms": [ + "qwen38-27b-udq4xl-t03-nothink", + "qwen38-nothink-offspec" + ], + "cost_missing": 30 + }, + "V36": { + "label": "3.6 no-think T1/p0.95/pp0", + "note": "3.6 VENDOR point", + "key": { + "model": "3.6", + "quant": "UD-Q4_K_XL", + "mode": "no-think", + "regime": "T1/p0.95/pp0" + }, + "n": 120, + "missing": { + "graded_fail": 25, + "ungraded_looped": 0, + "ungraded_not_looped": 0, + "graded_but_looped": 0 + }, + "terminal": { + "x": 120, + "n": 120, + "p": 1.0, + "lo": 0.9689808335812966, + "hi": 1.0 + }, + "ungraded": { + "x": 0, + "n": 120, + "p": 0.0, + "lo": 0.0, + "hi": 0.031019166418703486 + }, + "loop_freq30": { + "x": 0, + "n": 120, + "p": 0.0, + "lo": 0.0, + "hi": 0.031019166418703486 + }, + "loop_run30": { + "x": 0, + "n": 120, + "p": 0.0, + "lo": 0.0, + "hi": 0.031019166418703486 + }, + "pass_graded": { + "x": 95, + "n": 120, + "p": 0.7916666666666666, + "lo": 0.7105232156701865, + "hi": 0.8547156039189031 + }, + "pass_all": { + "x": 95, + "n": 120, + "p": 0.7916666666666666, + "lo": 0.7105232156701865, + "hi": 0.8547156039189031 + }, + "n_graded": 120, + "families": 12, + "arms": [ + "qwen36-nothink-card" + ], + "cost_missing": 0 + }, + "V38": { + "label": "3.8 no-think T0.7/p0.8/pp1.5", + "note": "3.8 VENDOR point", + "key": { + "model": "3.8", + "quant": "UD-Q4_K_XL", + "mode": "no-think", + "regime": "T0.7/p0.8/pp1.5" + }, + "n": 96, + "missing": { + "graded_fail": 31, + "ungraded_looped": 13, + "ungraded_not_looped": 4, + "graded_but_looped": 15 + }, + "terminal": { + "x": 96, + "n": 96, + "p": 1.0, + "lo": 0.9615244121423243, + "hi": 1.0 + }, + "ungraded": { + "x": 17, + "n": 96, + "p": 0.17708333333333334, + "lo": 0.11360507865666794, + "hi": 0.265410405168081 + }, + "loop_freq30": { + "x": 28, + "n": 96, + "p": 0.2916666666666667, + "lo": 0.21016592525216696, + "hi": 0.38919890302186466 + }, + "loop_run30": { + "x": 13, + "n": 96, + "p": 0.13541666666666666, + "lo": 0.0808770644581112, + "hi": 0.21801138502144396 + }, + "pass_graded": { + "x": 48, + "n": 79, + "p": 0.6075949367088608, + "lo": 0.4973399088270549, + "hi": 0.7078713494246327 + }, + "pass_all": { + "x": 48, + "n": 96, + "p": 0.5, + "lo": 0.401924024529863, + "hi": 0.598075975470137 + }, + "n_graded": 79, + "families": 12, + "arms": [ + "qwen38-27b-udq4xl-nothink", + "qwen38-nothink-card" + ], + "cost_missing": 17 + }, + "B36": { + "label": "3.6 think T0.3/p0.8/pp0", + "note": "off-spec for 3.6; sampler-matched to B38", + "key": { + "model": "3.6", + "quant": "UD-Q4_K_XL", + "mode": "think", + "regime": "T0.3/p0.8/pp0" + }, + "n": 108, + "missing": { + "graded_fail": 40, + "ungraded_looped": 1, + "ungraded_not_looped": 0, + "graded_but_looped": 0 + }, + "terminal": { + "x": 108, + "n": 108, + "p": 1.0, + "lo": 0.9656526402534428, + "hi": 1.0 + }, + "ungraded": { + "x": 1, + "n": 108, + "p": 0.009259259259259259, + "lo": 0.0016363608538834916, + "hi": 0.05059345519366322 + }, + "loop_freq30": { + "x": 1, + "n": 108, + "p": 0.009259259259259259, + "lo": 0.0016363608538834916, + "hi": 0.05059345519366322 + }, + "loop_run30": { + "x": 1, + "n": 108, + "p": 0.009259259259259259, + "lo": 0.0016363608538834916, + "hi": 0.05059345519366322 + }, + "pass_graded": { + "x": 67, + "n": 107, + "p": 0.6261682242990654, + "lo": 0.5316194156172028, + "hi": 0.7119717486210567 + }, + "pass_all": { + "x": 67, + "n": 108, + "p": 0.6203703703703703, + "lo": 0.5262011202247348, + "hi": 0.7062708116881311 + }, + "n_graded": 107, + "families": 12, + "arms": [ + "qwen36-think-offspec" + ], + "cost_missing": 1 + }, + "B38": { + "label": "3.8 think T0.3/p0.8/pp0", + "note": "off-spec for 3.8; sampler-matched to B36; MIXED effort (low/medium/xhigh)", + "key": { + "model": "3.8", + "quant": "UD-Q4_K_XL", + "mode": "think", + "regime": "T0.3/p0.8/pp0" + }, + "n": 51, + "missing": { + "graded_fail": 15, + "ungraded_looped": 2, + "ungraded_not_looped": 1, + "graded_but_looped": 0 + }, + "terminal": { + "x": 50, + "n": 51, + "p": 0.9803921568627451, + "lo": 0.8969543127385502, + "hi": 0.9965303073589391 + }, + "ungraded": { + "x": 3, + "n": 51, + "p": 0.058823529411764705, + "lo": 0.020206639422464673, + "hi": 0.15924626048800425 + }, + "loop_freq30": { + "x": 2, + "n": 51, + "p": 0.0392156862745098, + "lo": 0.010821083352867085, + "hi": 0.13216305655362268 + }, + "loop_run30": { + "x": 2, + "n": 51, + "p": 0.0392156862745098, + "lo": 0.010821083352867085, + "hi": 0.13216305655362268 + }, + "pass_graded": { + "x": 33, + "n": 48, + "p": 0.6875, + "lo": 0.5466692858410841, + "hi": 0.8005431655341219 + }, + "pass_all": { + "x": 33, + "n": 51, + "p": 0.6470588235294118, + "lo": 0.5098605845450523, + "hi": 0.7636551154847915 + }, + "n_graded": 48, + "families": 12, + "arms": [ + "qwen38-think-low-offspec", + "qwen38-think-medium-offspec", + "qwen38-think-xhigh-offspec" + ], + "cost_missing": 3 + }, + "C36": { + "label": "3.6 think T1/p0.95/pp0", + "note": "3.6 VENDOR point", + "key": { + "model": "3.6", + "quant": "UD-Q4_K_XL", + "mode": "think", + "regime": "T1/p0.95/pp0" + }, + "n": 120, + "missing": { + "graded_fail": 15, + "ungraded_looped": 0, + "ungraded_not_looped": 0, + "graded_but_looped": 0 + }, + "terminal": { + "x": 120, + "n": 120, + "p": 1.0, + "lo": 0.9689808335812966, + "hi": 1.0 + }, + "ungraded": { + "x": 0, + "n": 120, + "p": 0.0, + "lo": 0.0, + "hi": 0.031019166418703486 + }, + "loop_freq30": { + "x": 0, + "n": 120, + "p": 0.0, + "lo": 0.0, + "hi": 0.031019166418703486 + }, + "loop_run30": { + "x": 0, + "n": 120, + "p": 0.0, + "lo": 0.0, + "hi": 0.031019166418703486 + }, + "pass_graded": { + "x": 105, + "n": 120, + "p": 0.875, + "lo": 0.8039705979199262, + "hi": 0.9227650272660463 + }, + "pass_all": { + "x": 105, + "n": 120, + "p": 0.875, + "lo": 0.8039705979199262, + "hi": 0.9227650272660463 + }, + "n_graded": 120, + "families": 12, + "arms": [ + "qwen36-think-card" + ], + "cost_missing": 0 + }, + "C38": { + "label": "3.8 think T1/p0.95/pp0", + "note": "this is 3.6's vendor point, NOT 3.8's; sampler-matched to C36 but off-spec for 3.8; MIXED effort (low/medium/xhigh)", + "key": { + "model": "3.8", + "quant": "UD-Q4_K_XL", + "mode": "think", + "regime": "T1/p0.95/pp0" + }, + "n": 72, + "missing": { + "graded_fail": 18, + "ungraded_looped": 0, + "ungraded_not_looped": 0, + "graded_but_looped": 0 + }, + "terminal": { + "x": 72, + "n": 72, + "p": 1.0, + "lo": 0.9493488274035422, + "hi": 1.0 + }, + "ungraded": { + "x": 0, + "n": 72, + "p": 0.0, + "lo": 0.0, + "hi": 0.05065117259645781 + }, + "loop_freq30": { + "x": 0, + "n": 72, + "p": 0.0, + "lo": 0.0, + "hi": 0.05065117259645781 + }, + "loop_run30": { + "x": 0, + "n": 72, + "p": 0.0, + "lo": 0.0, + "hi": 0.05065117259645781 + }, + "pass_graded": { + "x": 54, + "n": 72, + "p": 0.75, + "lo": 0.6390649259565606, + "hi": 0.8356094877452105 + }, + "pass_all": { + "x": 54, + "n": 72, + "p": 0.75, + "lo": 0.6390649259565606, + "hi": 0.8356094877452105 + }, + "n_graded": 72, + "families": 12, + "arms": [ + "qwen38-27b-udq4xl-think-xhigh", + "qwen38-think-low-card", + "qwen38-think-medium-card", + "qwen38-think-xhigh-card" + ], + "cost_missing": 0 + }, + "Q8": { + "label": "3.8 Q8_0 no-think T0.3/p0.8/pp0", + "note": "quantization control; 19 cells over all 12 families (2 replicates on the 7 phase-1/2 families, 1 on the 5 phase-3)", + "key": { + "model": "3.8", + "quant": "Q8_0", + "mode": "no-think", + "regime": "T0.3/p0.8/pp0" + }, + "n": 19, + "missing": { + "graded_fail": 3, + "ungraded_looped": 6, + "ungraded_not_looped": 5, + "graded_but_looped": 0 + }, + "terminal": { + "x": 19, + "n": 19, + "p": 1.0, + "lo": 0.8318207759473837, + "hi": 1.0 + }, + "ungraded": { + "x": 11, + "n": 19, + "p": 0.5789473684210527, + "lo": 0.3627590353425504, + "hi": 0.7685810871754576 + }, + "loop_freq30": { + "x": 6, + "n": 19, + "p": 0.3157894736842105, + "lo": 0.1536437908779228, + "hi": 0.5398959232467253 + }, + "loop_run30": { + "x": 5, + "n": 19, + "p": 0.2631578947368421, + "lo": 0.11806412346782508, + "hi": 0.487915508978151 + }, + "pass_graded": { + "x": 5, + "n": 8, + "p": 0.625, + "lo": 0.30574239460262737, + "hi": 0.8631557141764026 + }, + "pass_all": { + "x": 5, + "n": 19, + "p": 0.2631578947368421, + "lo": 0.11806412346782508, + "hi": 0.487915508978151 + }, + "n_graded": 8, + "families": 12, + "arms": [ + "qwen38q8-nothink-matched" + ], + "cost_missing": 6 + } + }, + "comparisons": { + "P1": { + "pair": [ + "A36", + "A38" + ], + "description": "No-think, SAMPLER-MATCHED (both T0.3/p0.8/pp0). Off-spec for both models; the cleanest controlled-sampler contrast.", + "label_a": "3.6 no-think T0.3/p0.8/pp0", + "label_b": "3.8 no-think T0.3/p0.8/pp0", + "delivery": { + "terminal": { + "a": { + "x": 118, + "n": 121, + "p": 0.9752066115702479, + "lo": 0.9296356592312203, + "hi": 0.9915326856947302 + }, + "b": { + "x": 93, + "n": 95, + "p": 0.9789473684210527, + "lo": 0.9264588050529105, + "hi": 0.9942074931355945 + }, + "fisher_p": 1.0, + "diff": 0.0037407568508047717, + "diff_lo": -0.0512282342329833, + "diff_hi": 0.05179887898837169 + }, + "ungraded": { + "a": { + "x": 7, + "n": 121, + "p": 0.05785123966942149, + "lo": 0.02830275348227581, + "hi": 0.11461017776053556 + }, + "b": { + "x": 30, + "n": 95, + "p": 0.3157894736842105, + "lo": 0.23103226162043158, + "hi": 0.41486531599937426 + }, + "fisher_p": 8.212383305482713e-07, + "diff": 0.257938234014789, + "diff_lo": 0.1559315575173395, + "diff_hi": 0.36132650978342307 + }, + "loop_freq30": { + "a": { + "x": 3, + "n": 121, + "p": 0.024793388429752067, + "lo": 0.008467314305269862, + "hi": 0.07036434076877973 + }, + "b": { + "x": 29, + "n": 95, + "p": 0.30526315789473685, + "lo": 0.22172919278467978, + "hi": 0.40393396069911497 + }, + "fisher_p": 4.914929259256003e-09, + "diff": 0.2804697694649848, + "diff_lo": 0.18531392348538056, + "diff_hi": 0.38048210881557976 + }, + "loop_run30": { + "a": { + "x": 3, + "n": 121, + "p": 0.024793388429752067, + "lo": 0.008467314305269862, + "hi": 0.07036434076877973 + }, + "b": { + "x": 23, + "n": 95, + "p": 0.24210526315789474, + "lo": 0.16708840091876592, + "hi": 0.3371682077489622 + }, + "fisher_p": 8.658883118990303e-07, + "diff": 0.21731187472814267, + "diff_lo": 0.12953806703307705, + "diff_hi": 0.3137665486980569 + } + }, + "quality": { + "pass_graded": { + "a": { + "x": 81, + "n": 114, + "p": 0.7105263157894737, + "lo": 0.6214933876201677, + "hi": 0.7858335453655833 + }, + "b": { + "x": 51, + "n": 65, + "p": 0.7846153846153846, + "lo": 0.6703343975431146, + "hi": 0.8671324230884898 + }, + "fisher_p": 0.2959020637202259, + "diff": 0.07408906882591093, + "diff_lo": -0.0627733541858172, + "diff_hi": 0.1954806827733856 + }, + "pass_all": { + "a": { + "x": 81, + "n": 121, + "p": 0.6694214876033058, + "lo": 0.5815244079594877, + "hi": 0.7468921324054163 + }, + "b": { + "x": 51, + "n": 95, + "p": 0.5368421052631579, + "lo": 0.4370969126824784, + "hi": 0.6337235717935605 + }, + "fisher_p": 0.05051759578841104, + "diff": -0.13257938234014788, + "diff_lo": -0.2588758757554902, + "diff_hi": -0.0017668631877623409 + } + }, + "family_breakdown": [ + { + "family": "p1_bugfix", + "n_a": 19, + "pass_a": 18, + "graded_a": 18, + "loop_a": 0, + "n_b": 12, + "pass_b": 0, + "graded_b": 1, + "loop_b": 10, + "rate_a": 0.9473684210526315, + "rate_b": 0.0, + "delta": -0.9473684210526315, + "fisher_p": 9.211983869816244e-08 + }, + { + "family": "p1_refactor", + "n_a": 9, + "pass_a": 5, + "graded_a": 9, + "loop_a": 0, + "n_b": 7, + "pass_b": 3, + "graded_b": 5, + "loop_b": 1, + "rate_a": 0.5555555555555556, + "rate_b": 0.42857142857142855, + "delta": -0.12698412698412703, + "fisher_p": 1.0 + }, + { + "family": "p1_testwrite", + "n_a": 12, + "pass_a": 9, + "graded_a": 10, + "loop_a": 0, + "n_b": 13, + "pass_b": 0, + "graded_b": 2, + "loop_b": 10, + "rate_a": 0.75, + "rate_b": 0.0, + "delta": -0.75, + "fisher_p": 0.00010768609503297887 + }, + { + "family": "p2_ci", + "n_a": 9, + "pass_a": 9, + "graded_a": 9, + "loop_a": 0, + "n_b": 7, + "pass_b": 7, + "graded_b": 7, + "loop_b": 0, + "rate_a": 1.0, + "rate_b": 1.0, + "delta": 0.0, + "fisher_p": 1.0 + }, + { + "family": "p2_extract", + "n_a": 9, + "pass_a": 9, + "graded_a": 9, + "loop_a": 0, + "n_b": 7, + "pass_b": 7, + "graded_b": 7, + "loop_b": 0, + "rate_a": 1.0, + "rate_b": 1.0, + "delta": 0.0, + "fisher_p": 1.0 + }, + { + "family": "p2_hallucination", + "n_a": 9, + "pass_a": 9, + "graded_a": 9, + "loop_a": 0, + "n_b": 7, + "pass_b": 7, + "graded_b": 7, + "loop_b": 0, + "rate_a": 1.0, + "rate_b": 1.0, + "delta": 0.0, + "fisher_p": 1.0 + }, + { + "family": "p2_triage", + "n_a": 9, + "pass_a": 0, + "graded_a": 9, + "loop_a": 0, + "n_b": 7, + "pass_b": 7, + "graded_b": 7, + "loop_b": 0, + "rate_a": 0.0, + "rate_b": 1.0, + "delta": 1.0, + "fisher_p": 8.741258741258741e-05 + }, + { + "family": "p3_business", + "n_a": 9, + "pass_a": 7, + "graded_a": 9, + "loop_a": 0, + "n_b": 7, + "pass_b": 0, + "graded_b": 6, + "loop_b": 1, + "rate_a": 0.7777777777777778, + "rate_b": 0.0, + "delta": -0.7777777777777778, + "fisher_p": 0.003234265734265734 + }, + { + "family": "p3_doc", + "n_a": 9, + "pass_a": 8, + "graded_a": 9, + "loop_a": 0, + "n_b": 7, + "pass_b": 6, + "graded_b": 6, + "loop_b": 1, + "rate_a": 0.8888888888888888, + "rate_b": 0.8571428571428571, + "delta": -0.031746031746031744, + "fisher_p": 1.0 + }, + { + "family": "p3_market", + "n_a": 9, + "pass_a": 4, + "graded_a": 5, + "loop_a": 3, + "n_b": 7, + "pass_b": 0, + "graded_b": 3, + "loop_b": 6, + "rate_a": 0.4444444444444444, + "rate_b": 0.0, + "delta": -0.4444444444444444, + "fisher_p": 0.08846153846153847 + }, + { + "family": "p3_pm", + "n_a": 9, + "pass_a": 1, + "graded_a": 9, + "loop_a": 0, + "n_b": 7, + "pass_b": 7, + "graded_b": 7, + "loop_b": 0, + "rate_a": 0.1111111111111111, + "rate_b": 1.0, + "delta": 0.8888888888888888, + "fisher_p": 0.0013986013986013986 + }, + { + "family": "p3_writing", + "n_a": 9, + "pass_a": 2, + "graded_a": 9, + "loop_a": 0, + "n_b": 7, + "pass_b": 7, + "graded_b": 7, + "loop_b": 0, + "rate_a": 0.2222222222222222, + "rate_b": 1.0, + "delta": 0.7777777777777778, + "fisher_p": 0.003234265734265734 + } + ], + "cost": { + "tokens": { + "shared_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_market", + "p3_pm", + "p3_writing" + ], + "per_family": [ + { + "family": "p1_bugfix", + "n_a": 18, + "n_b": 1, + "cells_a": 19, + "cells_b": 12, + "coverage_a": 0.9473684210526315, + "coverage_b": 0.08333333333333333, + "comparable": false, + "median_a": 31183.0, + "median_b": 102609.0, + "delta": 71426.0, + "ratio": 3.2905429240291184 + }, + { + "family": "p1_refactor", + "n_a": 9, + "n_b": 5, + "cells_a": 9, + "cells_b": 7, + "coverage_a": 1.0, + "coverage_b": 0.7142857142857143, + "comparable": true, + "median_a": 11033.0, + "median_b": 11468.0, + "delta": 435.0, + "ratio": 1.0394271730263753 + }, + { + "family": "p1_testwrite", + "n_a": 10, + "n_b": 2, + "cells_a": 12, + "cells_b": 13, + "coverage_a": 0.8333333333333334, + "coverage_b": 0.15384615384615385, + "comparable": false, + "median_a": 36840.0, + "median_b": 30595.5, + "delta": -6244.5, + "ratio": 0.8304967426710098 + }, + { + "family": "p2_ci", + "n_a": 9, + "n_b": 7, + "cells_a": 9, + "cells_b": 7, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 4600.0, + "median_b": 3748.0, + "delta": -852.0, + "ratio": 0.8147826086956522 + }, + { + "family": "p2_extract", + "n_a": 9, + "n_b": 7, + "cells_a": 9, + "cells_b": 7, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 2871.0, + "median_b": 2666.0, + "delta": -205.0, + "ratio": 0.9285963079066527 + }, + { + "family": "p2_hallucination", + "n_a": 9, + "n_b": 7, + "cells_a": 9, + "cells_b": 7, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 5101.0, + "median_b": 4377.0, + "delta": -724.0, + "ratio": 0.8580670456773182 + }, + { + "family": "p2_triage", + "n_a": 9, + "n_b": 7, + "cells_a": 9, + "cells_b": 7, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 7707.0, + "median_b": 6289.0, + "delta": -1418.0, + "ratio": 0.8160114181912547 + }, + { + "family": "p3_business", + "n_a": 9, + "n_b": 6, + "cells_a": 9, + "cells_b": 7, + "coverage_a": 1.0, + "coverage_b": 0.8571428571428571, + "comparable": true, + "median_a": 9503.0, + "median_b": 10344.5, + "delta": 841.5, + "ratio": 1.088550983899821 + }, + { + "family": "p3_doc", + "n_a": 9, + "n_b": 6, + "cells_a": 9, + "cells_b": 7, + "coverage_a": 1.0, + "coverage_b": 0.8571428571428571, + "comparable": true, + "median_a": 7801.0, + "median_b": 7108.0, + "delta": -693.0, + "ratio": 0.911165235226253 + }, + { + "family": "p3_market", + "n_a": 5, + "n_b": 3, + "cells_a": 9, + "cells_b": 7, + "coverage_a": 0.5555555555555556, + "coverage_b": 0.42857142857142855, + "comparable": false, + "median_a": 40432.0, + "median_b": 103162.0, + "delta": 62730.0, + "ratio": 2.551493866244559 + }, + { + "family": "p3_pm", + "n_a": 9, + "n_b": 7, + "cells_a": 9, + "cells_b": 7, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 4634.0, + "median_b": 3308.0, + "delta": -1326.0, + "ratio": 0.7138541217091066 + }, + { + "family": "p3_writing", + "n_a": 9, + "n_b": 7, + "cells_a": 9, + "cells_b": 7, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 6427.0, + "median_b": 4927.0, + "delta": -1500.0, + "ratio": 0.7666096156838338 + } + ], + "n_comparable_families": 9, + "comparable_families": [ + "p1_refactor", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_pm", + "p3_writing" + ], + "excluded_families": [ + "p1_bugfix", + "p1_testwrite", + "p3_market" + ], + "median_of_family_deltas": -724.0, + "median_of_family_ratios": 0.8580670456773182, + "median_of_family_ratios_ALL": 0.8846161404517856, + "pooled_median_a": 7816.0, + "pooled_median_b": 5264.0, + "pooled_is_censored": true, + "coverage_a": 0.9421487603305785, + "coverage_b": 0.6842105263157895, + "cells_a": 121, + "cells_b": 95, + "with_data_a": 114, + "with_data_b": 65, + "n_families": 12 + }, + "elapsed": { + "shared_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_market", + "p3_pm", + "p3_writing" + ], + "per_family": [ + { + "family": "p1_bugfix", + "n_a": 18, + "n_b": 1, + "cells_a": 19, + "cells_b": 12, + "coverage_a": 0.9473684210526315, + "coverage_b": 0.08333333333333333, + "comparable": false, + "median_a": 894.25, + "median_b": 4860.3, + "delta": 3966.05, + "ratio": 5.435057310595472 + }, + { + "family": "p1_refactor", + "n_a": 9, + "n_b": 5, + "cells_a": 9, + "cells_b": 7, + "coverage_a": 1.0, + "coverage_b": 0.7142857142857143, + "comparable": true, + "median_a": 212.7, + "median_b": 195.0, + "delta": -17.69999999999999, + "ratio": 0.9167842031029619 + }, + { + "family": "p1_testwrite", + "n_a": 10, + "n_b": 2, + "cells_a": 12, + "cells_b": 13, + "coverage_a": 0.8333333333333334, + "coverage_b": 0.15384615384615385, + "comparable": false, + "median_a": 638.15, + "median_b": 684.05, + "delta": 45.89999999999998, + "ratio": 1.071926663010264 + }, + { + "family": "p2_ci", + "n_a": 9, + "n_b": 7, + "cells_a": 9, + "cells_b": 7, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 82.6, + "median_b": 71.7, + "delta": -10.899999999999991, + "ratio": 0.8680387409200969 + }, + { + "family": "p2_extract", + "n_a": 9, + "n_b": 7, + "cells_a": 9, + "cells_b": 7, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 45.5, + "median_b": 42.8, + "delta": -2.700000000000003, + "ratio": 0.9406593406593406 + }, + { + "family": "p2_hallucination", + "n_a": 9, + "n_b": 7, + "cells_a": 9, + "cells_b": 7, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 83.1, + "median_b": 72.4, + "delta": -10.699999999999989, + "ratio": 0.871239470517449 + }, + { + "family": "p2_triage", + "n_a": 9, + "n_b": 7, + "cells_a": 9, + "cells_b": 7, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 118.3, + "median_b": 97.0, + "delta": -21.299999999999997, + "ratio": 0.819949281487743 + }, + { + "family": "p3_business", + "n_a": 9, + "n_b": 6, + "cells_a": 9, + "cells_b": 7, + "coverage_a": 1.0, + "coverage_b": 0.8571428571428571, + "comparable": true, + "median_a": 146.3, + "median_b": 157.95, + "delta": 11.649999999999977, + "ratio": 1.079630895420369 + }, + { + "family": "p3_doc", + "n_a": 9, + "n_b": 6, + "cells_a": 9, + "cells_b": 7, + "coverage_a": 1.0, + "coverage_b": 0.8571428571428571, + "comparable": true, + "median_a": 120.1, + "median_b": 108.5, + "delta": -11.599999999999994, + "ratio": 0.9034138218151541 + }, + { + "family": "p3_market", + "n_a": 5, + "n_b": 3, + "cells_a": 9, + "cells_b": 7, + "coverage_a": 0.5555555555555556, + "coverage_b": 0.42857142857142855, + "comparable": false, + "median_a": 943.4, + "median_b": 2914.9, + "delta": 1971.5, + "ratio": 3.089781640873437 + }, + { + "family": "p3_pm", + "n_a": 9, + "n_b": 7, + "cells_a": 9, + "cells_b": 7, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 71.5, + "median_b": 50.7, + "delta": -20.799999999999997, + "ratio": 0.7090909090909091 + }, + { + "family": "p3_writing", + "n_a": 9, + "n_b": 7, + "cells_a": 9, + "cells_b": 7, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 104.0, + "median_b": 77.4, + "delta": -26.599999999999994, + "ratio": 0.7442307692307693 + } + ], + "n_comparable_families": 9, + "comparable_families": [ + "p1_refactor", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_pm", + "p3_writing" + ], + "excluded_families": [ + "p1_bugfix", + "p1_testwrite", + "p3_market" + ], + "median_of_family_deltas": -11.599999999999994, + "median_of_family_ratios": 0.871239470517449, + "median_of_family_ratios_ALL": 0.910099012459058, + "pooled_median_a": 120.6, + "pooled_median_b": 83.9, + "pooled_is_censored": true, + "coverage_a": 0.9421487603305785, + "coverage_b": 0.6842105263157895, + "cells_a": 121, + "cells_b": 95, + "with_data_a": 114, + "with_data_b": 65, + "n_families": 12 + }, + "tokens_per_success_a": { + "total_tokens": 1680139.0, + "n_cells_with_tokens": 114, + "n_cells": 121, + "n_missing_tokens": 7, + "token_coverage": 0.9421487603305785, + "passes": 81, + "tokens_per_success": 20742.456790123455, + "tokens_on_ungraded": 0, + "is_lower_bound": true, + "lower_bound_note": "7 of 121 cells have no token accounting (all of them ungraded), so total tokens and tokens-per-success are UNDERSTATED for this arm." + }, + "tokens_per_success_b": { + "total_tokens": 815989.0, + "n_cells_with_tokens": 65, + "n_cells": 95, + "n_missing_tokens": 30, + "token_coverage": 0.6842105263157895, + "passes": 51, + "tokens_per_success": 15999.78431372549, + "tokens_on_ungraded": 0, + "is_lower_bound": true, + "lower_bound_note": "30 of 95 cells have no token accounting (all of them ungraded), so total tokens and tokens-per-success are UNDERSTATED for this arm." + } + }, + "sensitivity": { + "loops_as_failure": { + "definition": "All cells in the group; passed==1 is success. Loops and ungraded cells score 0. This is the headline scoring.", + "stat": "binomial (Wilson/Newcombe/Fisher)", + "a": { + "x": 81, + "n": 121, + "p": 0.6694214876033058, + "lo": 0.5815244079594877, + "hi": 0.7468921324054163 + }, + "b": { + "x": 51, + "n": 95, + "p": 0.5368421052631579, + "lo": 0.4370969126824784, + "hi": 0.6337235717935605 + }, + "fisher_p": 0.05051759578841104, + "diff": -0.13257938234014788, + "diff_lo": -0.2588758757554902, + "diff_hi": -0.0017668631877623409 + }, + "loops_excluded": { + "definition": "Cells with looped_freq30==1 dropped entirely, then pass rate over the remainder. Charitable to whichever model loops more. NOTE: cells that are ungraded WITHOUT having looped are retained here and score 0.", + "stat": "binomial (Wilson/Newcombe/Fisher)", + "a": { + "x": 81, + "n": 118, + "p": 0.6864406779661016, + "lo": 0.5979749321048041, + "hi": 0.7631500940224013 + }, + "b": { + "x": 51, + "n": 66, + "p": 0.7727272727272727, + "lo": 0.6583054463536169, + "hi": 0.8571477043662696 + }, + "fisher_p": 0.23583670324940778, + "diff": 0.08628659476117106, + "diff_lo": -0.05146930842407005, + "diff_hi": 0.2085690443151043 + }, + "graded_only": { + "definition": "Only graded==1 cells. Removes every delivery failure (loops and ungraded alike) and measures output quality conditional on delivery. Maximally charitable to a model that fails by not delivering.", + "stat": "binomial (Wilson/Newcombe/Fisher)", + "a": { + "x": 81, + "n": 114, + "p": 0.7105263157894737, + "lo": 0.6214933876201677, + "hi": 0.7858335453655833 + }, + "b": { + "x": 51, + "n": 65, + "p": 0.7846153846153846, + "lo": 0.6703343975431146, + "hi": 0.8671324230884898 + }, + "fisher_p": 0.2959020637202259, + "diff": 0.07408906882591093, + "diff_lo": -0.0627733541858172, + "diff_hi": 0.1954806827733856 + }, + "loops_half_credit": { + "definition": "Score 1.0 for pass, 0.5 for a non-passing looped cell, 0.0 otherwise; compare MEAN scores. Scores are not 0/1 so Wilson and Fisher do not apply; CI is a z-interval on the difference of means.", + "stat": "z-interval on difference of means -- NOT Wilson/Fisher, NOT Welch-t", + "a": { + "mean": 0.6818181818181818, + "n": 121 + }, + "b": { + "mean": 0.6894736842105263, + "n": 95 + }, + "diff": 0.007655502392344582, + "diff_lo": -0.10359701314760159, + "diff_hi": 0.11890801793229075, + "fisher_p": null + }, + "best_of_n_per_family": { + "definition": "One Bernoulli trial per SHARED family: 1 if any cell in that group/family passed. Measures 'can the model ever do this task', not reliability. n = number of families, so it is structurally low-powered.", + "stat": "binomial over families (Wilson/Newcombe/Fisher)", + "a": { + "x": 11, + "n": 12, + "p": 0.9166666666666666, + "lo": 0.646120088858883, + "hi": 0.9851349055950829 + }, + "b": { + "x": 8, + "n": 12, + "p": 0.6666666666666666, + "lo": 0.39062208887279953, + "hi": 0.8618799089087867 + }, + "fisher_p": 0.3167701863354037, + "diff": -0.25, + "diff_lo": -0.5344090516691637, + "diff_hi": 0.08362203271094643, + "per_family": { + "p1_bugfix": { + "a": 1, + "b": 0 + }, + "p1_refactor": { + "a": 1, + "b": 1 + }, + "p1_testwrite": { + "a": 1, + "b": 0 + }, + "p2_ci": { + "a": 1, + "b": 1 + }, + "p2_extract": { + "a": 1, + "b": 1 + }, + "p2_hallucination": { + "a": 1, + "b": 1 + }, + "p2_triage": { + "a": 0, + "b": 1 + }, + "p3_business": { + "a": 1, + "b": 0 + }, + "p3_doc": { + "a": 1, + "b": 1 + }, + "p3_market": { + "a": 1, + "b": 0 + }, + "p3_pm": { + "a": 1, + "b": 1 + }, + "p3_writing": { + "a": 1, + "b": 1 + } + } + }, + "first_replicate_only": { + "definition": "Only rows with replicate==1 (one run per family per arm). Removes any weighting from unequal repeat depth; retains one row per arm, so n is families x arms, not families.", + "stat": "binomial (Wilson/Newcombe/Fisher)", + "a": { + "x": 8, + "n": 13, + "p": 0.6153846153846154, + "lo": 0.355228915126657, + "hi": 0.8229029220223743 + }, + "b": { + "x": 8, + "n": 13, + "p": 0.6153846153846154, + "lo": 0.355228915126657, + "hi": 0.8229029220223743 + }, + "fisher_p": 1.0, + "diff": 0.0, + "diff_lo": -0.33278346708710094, + "diff_hi": 0.33278346708710094 + }, + "depth_matched": { + "definition": "Per shared family, the first k=min(n_a,n_b) cells from each side ordered by (arm, replicate, cell). Both sides then have identical family x depth composition.", + "stat": "binomial (Wilson/Newcombe/Fisher)", + "a": { + "x": 62, + "n": 94, + "p": 0.6595744680851063, + "lo": 0.5592082220071468, + "hi": 0.7474102642827856 + }, + "b": { + "x": 51, + "n": 94, + "p": 0.5425531914893617, + "lo": 0.44215447666238733, + "hi": 0.6396104530149279 + }, + "fisher_p": 0.13614297940833214, + "diff": -0.11702127659574468, + "diff_lo": -0.25041928644817224, + "diff_hi": 0.02259783157582218 + } + }, + "power": { + "pass_all": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": false + }, + "pass_graded": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": true + }, + "loop_freq30": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": false + }, + "loop_run30": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": false + }, + "ungraded": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": false + }, + "sens_loops_as_failure": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": false + }, + "sens_loops_excluded": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": true + }, + "sens_graded_only": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": true + }, + "sens_loops_half_credit": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": true + }, + "sens_best_of_n_per_family": { + "underpowered": true, + "reasons": [ + "n<30 in an arm (n_a=12, n_b=12)", + "Newcombe CI width 61.8 pp" + ], + "ci_straddles_zero": true + }, + "sens_first_replicate_only": { + "underpowered": true, + "reasons": [ + "n<30 in an arm (n_a=13, n_b=13)", + "Newcombe CI width 66.6 pp" + ], + "ci_straddles_zero": true + }, + "sens_depth_matched": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": true + } + } + }, + "P2": { + "pair": [ + "V36", + "V38" + ], + "description": "No-think, VENDOR-MATCHED (each model at its own model-card sampler: 3.6 T1/p0.95/pp0, 3.8 T0.7/p0.8/pp1.5). Best-foot-forward contrast; sampler differs by construction.", + "label_a": "3.6 no-think T1/p0.95/pp0", + "label_b": "3.8 no-think T0.7/p0.8/pp1.5", + "delivery": { + "terminal": { + "a": { + "x": 120, + "n": 120, + "p": 1.0, + "lo": 0.9689808335812966, + "hi": 1.0 + }, + "b": { + "x": 96, + "n": 96, + "p": 1.0, + "lo": 0.9615244121423243, + "hi": 1.0 + }, + "fisher_p": 1.0, + "diff": 0.0, + "diff_lo": -0.038475587857675686, + "diff_hi": 0.031019166418703392 + }, + "ungraded": { + "a": { + "x": 0, + "n": 120, + "p": 0.0, + "lo": 0.0, + "hi": 0.031019166418703486 + }, + "b": { + "x": 17, + "n": 96, + "p": 0.17708333333333334, + "lo": 0.11360507865666794, + "hi": 0.265410405168081 + }, + "fisher_p": 4.3621946422937455e-07, + "diff": 0.17708333333333334, + "diff_lo": 0.10643152867055905, + "diff_hi": 0.265410405168081 + }, + "loop_freq30": { + "a": { + "x": 0, + "n": 120, + "p": 0.0, + "lo": 0.0, + "hi": 0.031019166418703486 + }, + "b": { + "x": 28, + "n": 96, + "p": 0.2916666666666667, + "lo": 0.21016592525216696, + "hi": 0.38919890302186466 + }, + "fisher_p": 1.0757234970285007e-11, + "diff": 0.2916666666666667, + "diff_lo": 0.20446254098201588, + "diff_hi": 0.38919890302186466 + }, + "loop_run30": { + "a": { + "x": 0, + "n": 120, + "p": 0.0, + "lo": 0.0, + "hi": 0.031019166418703486 + }, + "b": { + "x": 13, + "n": 96, + "p": 0.13541666666666666, + "lo": 0.0808770644581112, + "hi": 0.21801138502144396 + }, + "fisher_p": 1.6304652600411294e-05, + "diff": 0.13541666666666666, + "diff_lo": 0.07267308616332999, + "diff_hi": 0.21801138502144396 + } + }, + "quality": { + "pass_graded": { + "a": { + "x": 95, + "n": 120, + "p": 0.7916666666666666, + "lo": 0.7105232156701865, + "hi": 0.8547156039189031 + }, + "b": { + "x": 48, + "n": 79, + "p": 0.6075949367088608, + "lo": 0.4973399088270549, + "hi": 0.7078713494246327 + }, + "fisher_p": 0.006101960568202006, + "diff": -0.18407172995780585, + "diff_lo": -0.3110809408909277, + "diff_hi": -0.0550770843838054 + }, + "pass_all": { + "a": { + "x": 95, + "n": 120, + "p": 0.7916666666666666, + "lo": 0.7105232156701865, + "hi": 0.8547156039189031 + }, + "b": { + "x": 48, + "n": 96, + "p": 0.5, + "lo": 0.401924024529863, + "hi": 0.598075975470137 + }, + "fisher_p": 1.1315180666199763e-05, + "diff": -0.29166666666666663, + "diff_lo": -0.408260257627128, + "diff_hi": -0.16437504634530734 + } + }, + "family_breakdown": [ + { + "family": "p1_bugfix", + "n_a": 10, + "pass_a": 10, + "graded_a": 10, + "loop_a": 0, + "n_b": 8, + "pass_b": 4, + "graded_b": 4, + "loop_b": 5, + "rate_a": 1.0, + "rate_b": 0.5, + "delta": -0.5, + "fisher_p": 0.02287581699346405 + }, + { + "family": "p1_refactor", + "n_a": 10, + "pass_a": 8, + "graded_a": 10, + "loop_a": 0, + "n_b": 8, + "pass_b": 4, + "graded_b": 8, + "loop_b": 3, + "rate_a": 0.8, + "rate_b": 0.5, + "delta": -0.30000000000000004, + "fisher_p": 0.3212669683257919 + }, + { + "family": "p1_testwrite", + "n_a": 10, + "pass_a": 9, + "graded_a": 10, + "loop_a": 0, + "n_b": 8, + "pass_b": 0, + "graded_b": 7, + "loop_b": 7, + "rate_a": 0.9, + "rate_b": 0.0, + "delta": -0.9, + "fisher_p": 0.00041135335252982314 + }, + { + "family": "p2_ci", + "n_a": 10, + "pass_a": 10, + "graded_a": 10, + "loop_a": 0, + "n_b": 8, + "pass_b": 8, + "graded_b": 8, + "loop_b": 0, + "rate_a": 1.0, + "rate_b": 1.0, + "delta": 0.0, + "fisher_p": 1.0 + }, + { + "family": "p2_extract", + "n_a": 10, + "pass_a": 10, + "graded_a": 10, + "loop_a": 0, + "n_b": 8, + "pass_b": 8, + "graded_b": 8, + "loop_b": 0, + "rate_a": 1.0, + "rate_b": 1.0, + "delta": 0.0, + "fisher_p": 1.0 + }, + { + "family": "p2_hallucination", + "n_a": 10, + "pass_a": 10, + "graded_a": 10, + "loop_a": 0, + "n_b": 8, + "pass_b": 8, + "graded_b": 8, + "loop_b": 0, + "rate_a": 1.0, + "rate_b": 1.0, + "delta": 0.0, + "fisher_p": 1.0 + }, + { + "family": "p2_triage", + "n_a": 10, + "pass_a": 7, + "graded_a": 10, + "loop_a": 0, + "n_b": 8, + "pass_b": 5, + "graded_b": 8, + "loop_b": 0, + "rate_a": 0.7, + "rate_b": 0.625, + "delta": -0.07499999999999996, + "fisher_p": 1.0 + }, + { + "family": "p3_business", + "n_a": 10, + "pass_a": 10, + "graded_a": 10, + "loop_a": 0, + "n_b": 8, + "pass_b": 2, + "graded_b": 6, + "loop_b": 3, + "rate_a": 1.0, + "rate_b": 0.25, + "delta": -0.75, + "fisher_p": 0.0015082956259426848 + }, + { + "family": "p3_doc", + "n_a": 10, + "pass_a": 10, + "graded_a": 10, + "loop_a": 0, + "n_b": 8, + "pass_b": 1, + "graded_b": 8, + "loop_b": 0, + "rate_a": 1.0, + "rate_b": 0.125, + "delta": -0.875, + "fisher_p": 0.00025138260432378077 + }, + { + "family": "p3_market", + "n_a": 10, + "pass_a": 10, + "graded_a": 10, + "loop_a": 0, + "n_b": 8, + "pass_b": 0, + "graded_b": 2, + "loop_b": 5, + "rate_a": 1.0, + "rate_b": 0.0, + "delta": -1.0, + "fisher_p": 2.2852964029434616e-05 + }, + { + "family": "p3_pm", + "n_a": 10, + "pass_a": 1, + "graded_a": 10, + "loop_a": 0, + "n_b": 8, + "pass_b": 8, + "graded_b": 8, + "loop_b": 0, + "rate_a": 0.1, + "rate_b": 1.0, + "delta": 0.9, + "fisher_p": 0.00041135335252982314 + }, + { + "family": "p3_writing", + "n_a": 10, + "pass_a": 0, + "graded_a": 10, + "loop_a": 0, + "n_b": 8, + "pass_b": 0, + "graded_b": 4, + "loop_b": 5, + "rate_a": 0.0, + "rate_b": 0.0, + "delta": 0.0, + "fisher_p": 1.0 + } + ], + "cost": { + "tokens": { + "shared_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_market", + "p3_pm", + "p3_writing" + ], + "per_family": [ + { + "family": "p1_bugfix", + "n_a": 10, + "n_b": 4, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 0.5, + "comparable": true, + "median_a": 34135.0, + "median_b": 57052.5, + "delta": 22917.5, + "ratio": 1.6713783506664714 + }, + { + "family": "p1_refactor", + "n_a": 10, + "n_b": 8, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 11324.0, + "median_b": 13288.5, + "delta": 1964.5, + "ratio": 1.1734811020840692 + }, + { + "family": "p1_testwrite", + "n_a": 10, + "n_b": 7, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 0.875, + "comparable": true, + "median_a": 33248.0, + "median_b": 25923.0, + "delta": -7325.0, + "ratio": 0.7796859961501443 + }, + { + "family": "p2_ci", + "n_a": 10, + "n_b": 8, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 5597.0, + "median_b": 4090.5, + "delta": -1506.5, + "ratio": 0.730837948901197 + }, + { + "family": "p2_extract", + "n_a": 10, + "n_b": 8, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 1502.5, + "median_b": 2980.0, + "delta": 1477.5, + "ratio": 1.983361064891847 + }, + { + "family": "p2_hallucination", + "n_a": 10, + "n_b": 8, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 6213.5, + "median_b": 4240.5, + "delta": -1973.0, + "ratio": 0.6824655990987366 + }, + { + "family": "p2_triage", + "n_a": 10, + "n_b": 8, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 7010.5, + "median_b": 5997.5, + "delta": -1013.0, + "ratio": 0.855502460594822 + }, + { + "family": "p3_business", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 0.75, + "comparable": true, + "median_a": 7404.5, + "median_b": 9501.0, + "delta": 2096.5, + "ratio": 1.2831386319130258 + }, + { + "family": "p3_doc", + "n_a": 10, + "n_b": 8, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 8097.0, + "median_b": 10829.5, + "delta": 2732.5, + "ratio": 1.337470668148697 + }, + { + "family": "p3_market", + "n_a": 10, + "n_b": 2, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 0.25, + "comparable": false, + "median_a": 32928.0, + "median_b": 52876.0, + "delta": 19948.0, + "ratio": 1.6058066083576288 + }, + { + "family": "p3_pm", + "n_a": 10, + "n_b": 8, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 4181.5, + "median_b": 4529.5, + "delta": 348.0, + "ratio": 1.0832237235441826 + }, + { + "family": "p3_writing", + "n_a": 10, + "n_b": 4, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 0.5, + "comparable": true, + "median_a": 7103.5, + "median_b": 14931.0, + "delta": 7827.5, + "ratio": 2.1019215879496023 + } + ], + "n_comparable_families": 11, + "comparable_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_pm", + "p3_writing" + ], + "excluded_families": [ + "p3_market" + ], + "median_of_family_deltas": 1477.5, + "median_of_family_ratios": 1.1734811020840692, + "median_of_family_ratios_ALL": 1.2283098669985475, + "pooled_median_a": 7404.5, + "pooled_median_b": 7029.0, + "pooled_is_censored": true, + "coverage_a": 1.0, + "coverage_b": 0.8229166666666666, + "cells_a": 120, + "cells_b": 96, + "with_data_a": 120, + "with_data_b": 79, + "n_families": 12 + }, + "elapsed": { + "shared_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_market", + "p3_pm", + "p3_writing" + ], + "per_family": [ + { + "family": "p1_bugfix", + "n_a": 10, + "n_b": 4, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 0.5, + "comparable": true, + "median_a": 988.95, + "median_b": 4126.5, + "delta": 3137.55, + "ratio": 4.172607310784165 + }, + { + "family": "p1_refactor", + "n_a": 10, + "n_b": 8, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 212.95, + "median_b": 241.64999999999998, + "delta": 28.69999999999999, + "ratio": 1.1347734209908429 + }, + { + "family": "p1_testwrite", + "n_a": 10, + "n_b": 7, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 0.875, + "comparable": true, + "median_a": 563.6, + "median_b": 772.8, + "delta": 209.19999999999993, + "ratio": 1.3711852377572746 + }, + { + "family": "p2_ci", + "n_a": 10, + "n_b": 8, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 98.44999999999999, + "median_b": 76.94999999999999, + "delta": -21.5, + "ratio": 0.781615033011681 + }, + { + "family": "p2_extract", + "n_a": 10, + "n_b": 8, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 24.9, + "median_b": 49.650000000000006, + "delta": 24.750000000000007, + "ratio": 1.9939759036144582 + }, + { + "family": "p2_hallucination", + "n_a": 10, + "n_b": 8, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 104.1, + "median_b": 77.1, + "delta": -27.0, + "ratio": 0.7406340057636888 + }, + { + "family": "p2_triage", + "n_a": 10, + "n_b": 8, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 107.75, + "median_b": 99.25, + "delta": -8.5, + "ratio": 0.9211136890951276 + }, + { + "family": "p3_business", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 0.75, + "comparable": true, + "median_a": 111.95, + "median_b": 158.35000000000002, + "delta": 46.40000000000002, + "ratio": 1.4144707458686916 + }, + { + "family": "p3_doc", + "n_a": 10, + "n_b": 8, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 124.2, + "median_b": 177.75, + "delta": 53.55, + "ratio": 1.431159420289855 + }, + { + "family": "p3_market", + "n_a": 10, + "n_b": 2, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 0.25, + "comparable": false, + "median_a": 785.05, + "median_b": 1491.3000000000002, + "delta": 706.2500000000002, + "ratio": 1.899624227756194 + }, + { + "family": "p3_pm", + "n_a": 10, + "n_b": 8, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 65.15, + "median_b": 74.80000000000001, + "delta": 9.650000000000006, + "ratio": 1.148119723714505 + }, + { + "family": "p3_writing", + "n_a": 10, + "n_b": 4, + "cells_a": 10, + "cells_b": 8, + "coverage_a": 1.0, + "coverage_b": 0.5, + "comparable": true, + "median_a": 113.45, + "median_b": 254.25, + "delta": 140.8, + "ratio": 2.241075363596298 + } + ], + "n_comparable_families": 11, + "comparable_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_pm", + "p3_writing" + ], + "excluded_families": [ + "p3_market" + ], + "median_of_family_deltas": 28.69999999999999, + "median_of_family_ratios": 1.3711852377572746, + "median_of_family_ratios_ALL": 1.3928279918129831, + "pooled_median_a": 114.4, + "pooled_median_b": 115.1, + "pooled_is_censored": true, + "coverage_a": 1.0, + "coverage_b": 0.8229166666666666, + "cells_a": 120, + "cells_b": 96, + "with_data_a": 120, + "with_data_b": 79, + "n_families": 12 + }, + "tokens_per_success_a": { + "total_tokens": 1628275.0, + "n_cells_with_tokens": 120, + "n_cells": 120, + "n_missing_tokens": 0, + "token_coverage": 1.0, + "passes": 95, + "tokens_per_success": 17139.736842105263, + "tokens_on_ungraded": 0, + "is_lower_bound": false, + "lower_bound_note": null + }, + "tokens_per_success_b": { + "total_tokens": 1661867.0, + "n_cells_with_tokens": 79, + "n_cells": 96, + "n_missing_tokens": 17, + "token_coverage": 0.8229166666666666, + "passes": 48, + "tokens_per_success": 34622.229166666664, + "tokens_on_ungraded": 0, + "is_lower_bound": true, + "lower_bound_note": "17 of 96 cells have no token accounting (all of them ungraded), so total tokens and tokens-per-success are UNDERSTATED for this arm." + } + }, + "sensitivity": { + "loops_as_failure": { + "definition": "All cells in the group; passed==1 is success. Loops and ungraded cells score 0. This is the headline scoring.", + "stat": "binomial (Wilson/Newcombe/Fisher)", + "a": { + "x": 95, + "n": 120, + "p": 0.7916666666666666, + "lo": 0.7105232156701865, + "hi": 0.8547156039189031 + }, + "b": { + "x": 48, + "n": 96, + "p": 0.5, + "lo": 0.401924024529863, + "hi": 0.598075975470137 + }, + "fisher_p": 1.1315180666199763e-05, + "diff": -0.29166666666666663, + "diff_lo": -0.408260257627128, + "diff_hi": -0.16437504634530734 + }, + "loops_excluded": { + "definition": "Cells with looped_freq30==1 dropped entirely, then pass rate over the remainder. Charitable to whichever model loops more. NOTE: cells that are ungraded WITHOUT having looped are retained here and score 0.", + "stat": "binomial (Wilson/Newcombe/Fisher)", + "a": { + "x": 95, + "n": 120, + "p": 0.7916666666666666, + "lo": 0.7105232156701865, + "hi": 0.8547156039189031 + }, + "b": { + "x": 46, + "n": 68, + "p": 0.6764705882352942, + "lo": 0.5584449810050397, + "hi": 0.7756239590467001 + }, + "fisher_p": 0.11397284852144018, + "diff": -0.11519607843137247, + "diff_lo": -0.24900643845893816, + "diff_hi": 0.012927497006102517 + }, + "graded_only": { + "definition": "Only graded==1 cells. Removes every delivery failure (loops and ungraded alike) and measures output quality conditional on delivery. Maximally charitable to a model that fails by not delivering.", + "stat": "binomial (Wilson/Newcombe/Fisher)", + "a": { + "x": 95, + "n": 120, + "p": 0.7916666666666666, + "lo": 0.7105232156701865, + "hi": 0.8547156039189031 + }, + "b": { + "x": 48, + "n": 79, + "p": 0.6075949367088608, + "lo": 0.4973399088270549, + "hi": 0.7078713494246327 + }, + "fisher_p": 0.006101960568202006, + "diff": -0.18407172995780585, + "diff_lo": -0.3110809408909277, + "diff_hi": -0.0550770843838054 + }, + "loops_half_credit": { + "definition": "Score 1.0 for pass, 0.5 for a non-passing looped cell, 0.0 otherwise; compare MEAN scores. Scores are not 0/1 so Wilson and Fisher do not apply; CI is a z-interval on the difference of means.", + "stat": "z-interval on difference of means -- NOT Wilson/Fisher, NOT Welch-t", + "a": { + "mean": 0.7916666666666666, + "n": 120 + }, + "b": { + "mean": 0.6354166666666666, + "n": 96 + }, + "diff": -0.15625, + "diff_lo": -0.2655837157329279, + "diff_hi": -0.0469162842670721, + "fisher_p": null + }, + "best_of_n_per_family": { + "definition": "One Bernoulli trial per SHARED family: 1 if any cell in that group/family passed. Measures 'can the model ever do this task', not reliability. n = number of families, so it is structurally low-powered.", + "stat": "binomial over families (Wilson/Newcombe/Fisher)", + "a": { + "x": 11, + "n": 12, + "p": 0.9166666666666666, + "lo": 0.646120088858883, + "hi": 0.9851349055950829 + }, + "b": { + "x": 9, + "n": 12, + "p": 0.75, + "lo": 0.46769466506643426, + "hi": 0.9110583316059453 + }, + "fisher_p": 0.5900621118012422, + "diff": -0.16666666666666663, + "diff_lo": -0.45715625662457104, + "diff_hi": 0.14819082005263273, + "per_family": { + "p1_bugfix": { + "a": 1, + "b": 1 + }, + "p1_refactor": { + "a": 1, + "b": 1 + }, + "p1_testwrite": { + "a": 1, + "b": 0 + }, + "p2_ci": { + "a": 1, + "b": 1 + }, + "p2_extract": { + "a": 1, + "b": 1 + }, + "p2_hallucination": { + "a": 1, + "b": 1 + }, + "p2_triage": { + "a": 1, + "b": 1 + }, + "p3_business": { + "a": 1, + "b": 1 + }, + "p3_doc": { + "a": 1, + "b": 1 + }, + "p3_market": { + "a": 1, + "b": 0 + }, + "p3_pm": { + "a": 1, + "b": 1 + }, + "p3_writing": { + "a": 0, + "b": 0 + } + } + }, + "first_replicate_only": { + "definition": "Only rows with replicate==1 (one run per family per arm). Removes any weighting from unequal repeat depth; retains one row per arm, so n is families x arms, not families.", + "stat": "binomial (Wilson/Newcombe/Fisher)", + "a": { + "x": 9, + "n": 12, + "p": 0.75, + "lo": 0.46769466506643426, + "hi": 0.9110583316059453 + }, + "b": { + "x": 12, + "n": 24, + "p": 0.5, + "lo": 0.3142742581957335, + "hi": 0.6857257418042665 + }, + "fisher_p": 0.282128290693363, + "diff": -0.25, + "diff_lo": -0.4958329460191121, + "diff_hi": 0.0879206316588228 + }, + "depth_matched": { + "definition": "Per shared family, the first k=min(n_a,n_b) cells from each side ordered by (arm, replicate, cell). Both sides then have identical family x depth composition.", + "stat": "binomial (Wilson/Newcombe/Fisher)", + "a": { + "x": 77, + "n": 96, + "p": 0.8020833333333334, + "lo": 0.7114483305401257, + "hi": 0.869472668462529 + }, + "b": { + "x": 48, + "n": 96, + "p": 0.5, + "lo": 0.401924024529863, + "hi": 0.598075975470137 + }, + "fisher_p": 1.8192133762657005e-05, + "diff": -0.30208333333333337, + "diff_lo": -0.4210800536813075, + "diff_hi": -0.1685408281308935 + } + }, + "power": { + "pass_all": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": false + }, + "pass_graded": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": false + }, + "loop_freq30": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": false + }, + "loop_run30": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": false + }, + "ungraded": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": false + }, + "sens_loops_as_failure": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": false + }, + "sens_loops_excluded": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": true + }, + "sens_graded_only": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": false + }, + "sens_loops_half_credit": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": false + }, + "sens_best_of_n_per_family": { + "underpowered": true, + "reasons": [ + "n<30 in an arm (n_a=12, n_b=12)", + "Newcombe CI width 60.5 pp" + ], + "ci_straddles_zero": true + }, + "sens_first_replicate_only": { + "underpowered": true, + "reasons": [ + "n<30 in an arm (n_a=12, n_b=24)", + "Newcombe CI width 58.4 pp" + ], + "ci_straddles_zero": true + }, + "sens_depth_matched": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": false + } + } + }, + "P3": { + "pair": [ + "B36", + "B38" + ], + "description": "Thinking, SAMPLER-MATCHED (both T0.3/p0.8/pp0). Off-spec for both. 3.8 side pools three effort levels.", + "label_a": "3.6 think T0.3/p0.8/pp0", + "label_b": "3.8 think T0.3/p0.8/pp0", + "delivery": { + "terminal": { + "a": { + "x": 108, + "n": 108, + "p": 1.0, + "lo": 0.9656526402534428, + "hi": 1.0 + }, + "b": { + "x": 50, + "n": 51, + "p": 0.9803921568627451, + "lo": 0.8969543127385502, + "hi": 0.9965303073589391 + }, + "fisher_p": 0.32075471698113206, + "diff": -0.019607843137254943, + "diff_lo": -0.1030456872614498, + "diff_hi": 0.018341873897226718 + }, + "ungraded": { + "a": { + "x": 1, + "n": 108, + "p": 0.009259259259259259, + "lo": 0.0016363608538834916, + "hi": 0.05059345519366322 + }, + "b": { + "x": 3, + "n": 51, + "p": 0.058823529411764705, + "lo": 0.020206639422464673, + "hi": 0.15924626048800425 + }, + "fisher_p": 0.09747629997752078, + "diff": 0.049564270152505446, + "diff_lo": -0.007002327287939897, + "diff_hi": 0.15027590551262365 + }, + "loop_freq30": { + "a": { + "x": 1, + "n": 108, + "p": 0.009259259259259259, + "lo": 0.0016363608538834916, + "hi": 0.05059345519366322 + }, + "b": { + "x": 2, + "n": 51, + "p": 0.0392156862745098, + "lo": 0.010821083352867085, + "hi": 0.13216305655362268 + }, + "fisher_p": 0.24115437683214194, + "diff": 0.029956427015250545, + "diff_lo": -0.02019104778280343, + "diff_hi": 0.12321586204410538 + }, + "loop_run30": { + "a": { + "x": 1, + "n": 108, + "p": 0.009259259259259259, + "lo": 0.0016363608538834916, + "hi": 0.05059345519366322 + }, + "b": { + "x": 2, + "n": 51, + "p": 0.0392156862745098, + "lo": 0.010821083352867085, + "hi": 0.13216305655362268 + }, + "fisher_p": 0.24115437683214194, + "diff": 0.029956427015250545, + "diff_lo": -0.02019104778280343, + "diff_hi": 0.12321586204410538 + } + }, + "quality": { + "pass_graded": { + "a": { + "x": 67, + "n": 107, + "p": 0.6261682242990654, + "lo": 0.5316194156172028, + "hi": 0.7119717486210567 + }, + "b": { + "x": 33, + "n": 48, + "p": 0.6875, + "lo": 0.5466692858410841, + "hi": 0.8005431655341219 + }, + "fisher_p": 0.5862827308283327, + "diff": 0.06133177570093462, + "diff_lo": -0.10357891175316836, + "diff_hi": 0.20870285324715576 + }, + "pass_all": { + "a": { + "x": 67, + "n": 108, + "p": 0.6203703703703703, + "lo": 0.5262011202247348, + "hi": 0.7062708116881311 + }, + "b": { + "x": 33, + "n": 51, + "p": 0.6470588235294118, + "lo": 0.5098605845450523, + "hi": 0.7636551154847915 + }, + "fisher_p": 0.8606634142133319, + "diff": 0.026688453159041448, + "diff_lo": -0.13518261466654458, + "diff_hi": 0.17656354438748884 + } + }, + "family_breakdown": [ + { + "family": "p1_bugfix", + "n_a": 9, + "pass_a": 9, + "graded_a": 9, + "loop_a": 0, + "n_b": 6, + "pass_b": 4, + "graded_b": 5, + "loop_b": 0, + "rate_a": 1.0, + "rate_b": 0.6666666666666666, + "delta": -0.33333333333333337, + "fisher_p": 0.14285714285714285 + }, + { + "family": "p1_refactor", + "n_a": 9, + "pass_a": 4, + "graded_a": 9, + "loop_a": 0, + "n_b": 4, + "pass_b": 3, + "graded_b": 4, + "loop_b": 0, + "rate_a": 0.4444444444444444, + "rate_b": 0.75, + "delta": 0.3055555555555556, + "fisher_p": 0.5594405594405595 + }, + { + "family": "p1_testwrite", + "n_a": 9, + "pass_a": 9, + "graded_a": 9, + "loop_a": 0, + "n_b": 5, + "pass_b": 4, + "graded_b": 5, + "loop_b": 0, + "rate_a": 1.0, + "rate_b": 0.8, + "delta": -0.19999999999999996, + "fisher_p": 0.35714285714285715 + }, + { + "family": "p2_ci", + "n_a": 9, + "pass_a": 9, + "graded_a": 9, + "loop_a": 0, + "n_b": 4, + "pass_b": 4, + "graded_b": 4, + "loop_b": 0, + "rate_a": 1.0, + "rate_b": 1.0, + "delta": 0.0, + "fisher_p": 1.0 + }, + { + "family": "p2_extract", + "n_a": 9, + "pass_a": 9, + "graded_a": 9, + "loop_a": 0, + "n_b": 4, + "pass_b": 4, + "graded_b": 4, + "loop_b": 0, + "rate_a": 1.0, + "rate_b": 1.0, + "delta": 0.0, + "fisher_p": 1.0 + }, + { + "family": "p2_hallucination", + "n_a": 9, + "pass_a": 9, + "graded_a": 9, + "loop_a": 0, + "n_b": 4, + "pass_b": 4, + "graded_b": 4, + "loop_b": 0, + "rate_a": 1.0, + "rate_b": 1.0, + "delta": 0.0, + "fisher_p": 1.0 + }, + { + "family": "p2_triage", + "n_a": 9, + "pass_a": 0, + "graded_a": 9, + "loop_a": 0, + "n_b": 4, + "pass_b": 4, + "graded_b": 4, + "loop_b": 0, + "rate_a": 0.0, + "rate_b": 1.0, + "delta": 1.0, + "fisher_p": 0.0013986013986013986 + }, + { + "family": "p3_business", + "n_a": 9, + "pass_a": 8, + "graded_a": 9, + "loop_a": 0, + "n_b": 4, + "pass_b": 2, + "graded_b": 4, + "loop_b": 0, + "rate_a": 0.8888888888888888, + "rate_b": 0.5, + "delta": -0.38888888888888884, + "fisher_p": 0.20279720279720279 + }, + { + "family": "p3_doc", + "n_a": 9, + "pass_a": 0, + "graded_a": 9, + "loop_a": 0, + "n_b": 4, + "pass_b": 1, + "graded_b": 4, + "loop_b": 0, + "rate_a": 0.0, + "rate_b": 0.25, + "delta": 0.25, + "fisher_p": 0.3076923076923077 + }, + { + "family": "p3_market", + "n_a": 9, + "pass_a": 8, + "graded_a": 8, + "loop_a": 1, + "n_b": 4, + "pass_b": 1, + "graded_b": 2, + "loop_b": 2, + "rate_a": 0.8888888888888888, + "rate_b": 0.25, + "delta": -0.6388888888888888, + "fisher_p": 0.05174825174825175 + }, + { + "family": "p3_pm", + "n_a": 9, + "pass_a": 0, + "graded_a": 9, + "loop_a": 0, + "n_b": 4, + "pass_b": 1, + "graded_b": 4, + "loop_b": 0, + "rate_a": 0.0, + "rate_b": 0.25, + "delta": 0.25, + "fisher_p": 0.3076923076923077 + }, + { + "family": "p3_writing", + "n_a": 9, + "pass_a": 2, + "graded_a": 9, + "loop_a": 0, + "n_b": 4, + "pass_b": 1, + "graded_b": 4, + "loop_b": 0, + "rate_a": 0.2222222222222222, + "rate_b": 0.25, + "delta": 0.02777777777777779, + "fisher_p": 1.0 + } + ], + "cost": { + "tokens": { + "shared_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_market", + "p3_pm", + "p3_writing" + ], + "per_family": [ + { + "family": "p1_bugfix", + "n_a": 9, + "n_b": 5, + "cells_a": 9, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 0.8333333333333334, + "comparable": true, + "median_a": 32044.0, + "median_b": 86053.0, + "delta": 54009.0, + "ratio": 2.6854637373611285 + }, + { + "family": "p1_refactor", + "n_a": 9, + "n_b": 4, + "cells_a": 9, + "cells_b": 4, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 12387.0, + "median_b": 27969.0, + "delta": 15582.0, + "ratio": 2.2579317025914265 + }, + { + "family": "p1_testwrite", + "n_a": 9, + "n_b": 5, + "cells_a": 9, + "cells_b": 5, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 42849.0, + "median_b": 31491.0, + "delta": -11358.0, + "ratio": 0.7349296366309599 + }, + { + "family": "p2_ci", + "n_a": 9, + "n_b": 4, + "cells_a": 9, + "cells_b": 4, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 6675.0, + "median_b": 6881.0, + "delta": 206.0, + "ratio": 1.0308614232209738 + }, + { + "family": "p2_extract", + "n_a": 9, + "n_b": 4, + "cells_a": 9, + "cells_b": 4, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 3865.0, + "median_b": 4869.5, + "delta": 1004.5, + "ratio": 1.2598965071151358 + }, + { + "family": "p2_hallucination", + "n_a": 9, + "n_b": 4, + "cells_a": 9, + "cells_b": 4, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 7234.0, + "median_b": 8621.0, + "delta": 1387.0, + "ratio": 1.1917334807851812 + }, + { + "family": "p2_triage", + "n_a": 9, + "n_b": 4, + "cells_a": 9, + "cells_b": 4, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 7681.0, + "median_b": 9907.0, + "delta": 2226.0, + "ratio": 1.2898060148418176 + }, + { + "family": "p3_business", + "n_a": 9, + "n_b": 4, + "cells_a": 9, + "cells_b": 4, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 8137.0, + "median_b": 17730.0, + "delta": 9593.0, + "ratio": 2.1789357256974315 + }, + { + "family": "p3_doc", + "n_a": 9, + "n_b": 4, + "cells_a": 9, + "cells_b": 4, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 7304.0, + "median_b": 42729.5, + "delta": 35425.5, + "ratio": 5.850150602409639 + }, + { + "family": "p3_market", + "n_a": 8, + "n_b": 2, + "cells_a": 9, + "cells_b": 4, + "coverage_a": 0.8888888888888888, + "coverage_b": 0.5, + "comparable": false, + "median_a": 26516.0, + "median_b": 78524.0, + "delta": 52008.0, + "ratio": 2.9613818072107407 + }, + { + "family": "p3_pm", + "n_a": 9, + "n_b": 4, + "cells_a": 9, + "cells_b": 4, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 3729.0, + "median_b": 10202.0, + "delta": 6473.0, + "ratio": 2.73585411638509 + }, + { + "family": "p3_writing", + "n_a": 9, + "n_b": 4, + "cells_a": 9, + "cells_b": 4, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 6315.0, + "median_b": 17227.5, + "delta": 10912.5, + "ratio": 2.7280285035629452 + } + ], + "n_comparable_families": 11, + "comparable_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_pm", + "p3_writing" + ], + "excluded_families": [ + "p3_market" + ], + "median_of_family_deltas": 6473.0, + "median_of_family_ratios": 2.1789357256974315, + "median_of_family_ratios_ALL": 2.218433714144429, + "pooled_median_a": 7685.0, + "pooled_median_b": 17107.0, + "pooled_is_censored": true, + "coverage_a": 0.9907407407407407, + "coverage_b": 0.9411764705882353, + "cells_a": 108, + "cells_b": 51, + "with_data_a": 107, + "with_data_b": 48, + "n_families": 12 + }, + "elapsed": { + "shared_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_market", + "p3_pm", + "p3_writing" + ], + "per_family": [ + { + "family": "p1_bugfix", + "n_a": 9, + "n_b": 5, + "cells_a": 9, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 0.8333333333333334, + "comparable": true, + "median_a": 899.6, + "median_b": 2378.7, + "delta": 1479.1, + "ratio": 2.6441751889728766 + }, + { + "family": "p1_refactor", + "n_a": 9, + "n_b": 4, + "cells_a": 9, + "cells_b": 4, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 220.2, + "median_b": 666.15, + "delta": 445.95, + "ratio": 3.0252043596730247 + }, + { + "family": "p1_testwrite", + "n_a": 9, + "n_b": 5, + "cells_a": 9, + "cells_b": 5, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 730.9, + "median_b": 940.5, + "delta": 209.60000000000002, + "ratio": 1.2867697359419894 + }, + { + "family": "p2_ci", + "n_a": 9, + "n_b": 4, + "cells_a": 9, + "cells_b": 4, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 115.6, + "median_b": 113.85, + "delta": -1.75, + "ratio": 0.9848615916955017 + }, + { + "family": "p2_extract", + "n_a": 9, + "n_b": 4, + "cells_a": 9, + "cells_b": 4, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 60.7, + "median_b": 112.5, + "delta": 51.8, + "ratio": 1.8533772652388796 + }, + { + "family": "p2_hallucination", + "n_a": 9, + "n_b": 4, + "cells_a": 9, + "cells_b": 4, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 117.8, + "median_b": 135.75, + "delta": 17.950000000000003, + "ratio": 1.152376910016978 + }, + { + "family": "p2_triage", + "n_a": 9, + "n_b": 4, + "cells_a": 9, + "cells_b": 4, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 116.9, + "median_b": 149.9, + "delta": 33.0, + "ratio": 1.2822925577416595 + }, + { + "family": "p3_business", + "n_a": 9, + "n_b": 4, + "cells_a": 9, + "cells_b": 4, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 122.5, + "median_b": 269.4, + "delta": 146.89999999999998, + "ratio": 2.1991836734693875 + }, + { + "family": "p3_doc", + "n_a": 9, + "n_b": 4, + "cells_a": 9, + "cells_b": 4, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 111.9, + "median_b": 686.8, + "delta": 574.9, + "ratio": 6.137622877569258 + }, + { + "family": "p3_market", + "n_a": 8, + "n_b": 2, + "cells_a": 9, + "cells_b": 4, + "coverage_a": 0.8888888888888888, + "coverage_b": 0.5, + "comparable": false, + "median_a": 529.75, + "median_b": 1779.8000000000002, + "delta": 1250.0500000000002, + "ratio": 3.359697970740916 + }, + { + "family": "p3_pm", + "n_a": 9, + "n_b": 4, + "cells_a": 9, + "cells_b": 4, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 58.0, + "median_b": 154.9, + "delta": 96.9, + "ratio": 2.670689655172414 + }, + { + "family": "p3_writing", + "n_a": 9, + "n_b": 4, + "cells_a": 9, + "cells_b": 4, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 99.7, + "median_b": 265.29999999999995, + "delta": 165.59999999999997, + "ratio": 2.6609829488465393 + } + ], + "n_comparable_families": 11, + "comparable_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_pm", + "p3_writing" + ], + "excluded_families": [ + "p3_market" + ], + "median_of_family_deltas": 146.89999999999998, + "median_of_family_ratios": 2.1991836734693875, + "median_of_family_ratios_ALL": 2.421679431221132, + "pooled_median_a": 120.5, + "pooled_median_b": 265.5, + "pooled_is_censored": true, + "coverage_a": 0.9907407407407407, + "coverage_b": 0.9411764705882353, + "cells_a": 108, + "cells_b": 51, + "with_data_a": 107, + "with_data_b": 48, + "n_families": 12 + }, + "tokens_per_success_a": { + "total_tokens": 1540103.0, + "n_cells_with_tokens": 107, + "n_cells": 108, + "n_missing_tokens": 1, + "token_coverage": 0.9907407407407407, + "passes": 67, + "tokens_per_success": 22986.611940298506, + "tokens_on_ungraded": 0, + "is_lower_bound": true, + "lower_bound_note": "1 of 108 cell has no token accounting (all of them ungraded), so total tokens and tokens-per-success are UNDERSTATED for this arm." + }, + "tokens_per_success_b": { + "total_tokens": 1392807.0, + "n_cells_with_tokens": 48, + "n_cells": 51, + "n_missing_tokens": 3, + "token_coverage": 0.9411764705882353, + "passes": 33, + "tokens_per_success": 42206.27272727273, + "tokens_on_ungraded": 0, + "is_lower_bound": true, + "lower_bound_note": "3 of 51 cells have no token accounting (all of them ungraded), so total tokens and tokens-per-success are UNDERSTATED for this arm." + } + }, + "sensitivity": { + "loops_as_failure": { + "definition": "All cells in the group; passed==1 is success. Loops and ungraded cells score 0. This is the headline scoring.", + "stat": "binomial (Wilson/Newcombe/Fisher)", + "a": { + "x": 67, + "n": 108, + "p": 0.6203703703703703, + "lo": 0.5262011202247348, + "hi": 0.7062708116881311 + }, + "b": { + "x": 33, + "n": 51, + "p": 0.6470588235294118, + "lo": 0.5098605845450523, + "hi": 0.7636551154847915 + }, + "fisher_p": 0.8606634142133319, + "diff": 0.026688453159041448, + "diff_lo": -0.13518261466654458, + "diff_hi": 0.17656354438748884 + }, + "loops_excluded": { + "definition": "Cells with looped_freq30==1 dropped entirely, then pass rate over the remainder. Charitable to whichever model loops more. NOTE: cells that are ungraded WITHOUT having looped are retained here and score 0.", + "stat": "binomial (Wilson/Newcombe/Fisher)", + "a": { + "x": 67, + "n": 107, + "p": 0.6261682242990654, + "lo": 0.5316194156172028, + "hi": 0.7119717486210567 + }, + "b": { + "x": 33, + "n": 49, + "p": 0.673469387755102, + "lo": 0.5337920531457014, + "hi": 0.7879250299818812 + }, + "fisher_p": 0.5948568541571034, + "diff": 0.04730116345603663, + "diff_lo": -0.11662565738076544, + "diff_hi": 0.19575846796040894 + }, + "graded_only": { + "definition": "Only graded==1 cells. Removes every delivery failure (loops and ungraded alike) and measures output quality conditional on delivery. Maximally charitable to a model that fails by not delivering.", + "stat": "binomial (Wilson/Newcombe/Fisher)", + "a": { + "x": 67, + "n": 107, + "p": 0.6261682242990654, + "lo": 0.5316194156172028, + "hi": 0.7119717486210567 + }, + "b": { + "x": 33, + "n": 48, + "p": 0.6875, + "lo": 0.5466692858410841, + "hi": 0.8005431655341219 + }, + "fisher_p": 0.5862827308283327, + "diff": 0.06133177570093462, + "diff_lo": -0.10357891175316836, + "diff_hi": 0.20870285324715576 + }, + "loops_half_credit": { + "definition": "Score 1.0 for pass, 0.5 for a non-passing looped cell, 0.0 otherwise; compare MEAN scores. Scores are not 0/1 so Wilson and Fisher do not apply; CI is a z-interval on the difference of means.", + "stat": "z-interval on difference of means -- NOT Wilson/Fisher, NOT Welch-t", + "a": { + "mean": 0.625, + "n": 108 + }, + "b": { + "mean": 0.6666666666666666, + "n": 51 + }, + "diff": 0.04166666666666663, + "diff_lo": -0.11534042932208707, + "diff_hi": 0.19867376265542033, + "fisher_p": null + }, + "best_of_n_per_family": { + "definition": "One Bernoulli trial per SHARED family: 1 if any cell in that group/family passed. Measures 'can the model ever do this task', not reliability. n = number of families, so it is structurally low-powered.", + "stat": "binomial over families (Wilson/Newcombe/Fisher)", + "a": { + "x": 9, + "n": 12, + "p": 0.75, + "lo": 0.46769466506643426, + "hi": 0.9110583316059453 + }, + "b": { + "x": 12, + "n": 12, + "p": 1.0, + "lo": 0.7575059933447592, + "hi": 1.0 + }, + "fisher_p": 0.21739130434782608, + "diff": 0.25, + "diff_lo": -0.041106732047547445, + "diff_hi": 0.5323053349335658, + "per_family": { + "p1_bugfix": { + "a": 1, + "b": 1 + }, + "p1_refactor": { + "a": 1, + "b": 1 + }, + "p1_testwrite": { + "a": 1, + "b": 1 + }, + "p2_ci": { + "a": 1, + "b": 1 + }, + "p2_extract": { + "a": 1, + "b": 1 + }, + "p2_hallucination": { + "a": 1, + "b": 1 + }, + "p2_triage": { + "a": 0, + "b": 1 + }, + "p3_business": { + "a": 1, + "b": 1 + }, + "p3_doc": { + "a": 0, + "b": 1 + }, + "p3_market": { + "a": 1, + "b": 1 + }, + "p3_pm": { + "a": 0, + "b": 1 + }, + "p3_writing": { + "a": 1, + "b": 1 + } + } + }, + "first_replicate_only": { + "definition": "Only rows with replicate==1 (one run per family per arm). Removes any weighting from unequal repeat depth; retains one row per arm, so n is families x arms, not families.", + "stat": "binomial (Wilson/Newcombe/Fisher)", + "a": { + "x": 8, + "n": 12, + "p": 0.6666666666666666, + "lo": 0.39062208887279953, + "hi": 0.8618799089087867 + }, + "b": { + "x": 26, + "n": 36, + "p": 0.7222222222222222, + "lo": 0.5600751000260289, + "hi": 0.841516620598654 + }, + "fisher_p": 0.7259558087479303, + "diff": 0.05555555555555558, + "diff_lo": -0.19821579661641026, + "diff_hi": 0.3562742985919502 + }, + "depth_matched": { + "definition": "Per shared family, the first k=min(n_a,n_b) cells from each side ordered by (arm, replicate, cell). Both sides then have identical family x depth composition.", + "stat": "binomial (Wilson/Newcombe/Fisher)", + "a": { + "x": 34, + "n": 51, + "p": 0.6666666666666666, + "lo": 0.529683778569708, + "hi": 0.7803006814641148 + }, + "b": { + "x": 33, + "n": 51, + "p": 0.6470588235294118, + "lo": 0.5098605845450523, + "hi": 0.7636551154847915 + }, + "fisher_p": 1.0, + "diff": -0.019607843137254832, + "diff_lo": -0.19775398005690875, + "diff_hi": 0.16027825118114436 + } + }, + "power": { + "pass_all": { + "underpowered": true, + "reasons": [ + "Newcombe CI width 31.2 pp" + ], + "ci_straddles_zero": true + }, + "pass_graded": { + "underpowered": true, + "reasons": [ + "Newcombe CI width 31.2 pp" + ], + "ci_straddles_zero": true + }, + "loop_freq30": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": true + }, + "loop_run30": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": true + }, + "ungraded": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": true + }, + "sens_loops_as_failure": { + "underpowered": true, + "reasons": [ + "Newcombe CI width 31.2 pp" + ], + "ci_straddles_zero": true + }, + "sens_loops_excluded": { + "underpowered": true, + "reasons": [ + "Newcombe CI width 31.2 pp" + ], + "ci_straddles_zero": true + }, + "sens_graded_only": { + "underpowered": true, + "reasons": [ + "Newcombe CI width 31.2 pp" + ], + "ci_straddles_zero": true + }, + "sens_loops_half_credit": { + "underpowered": true, + "reasons": [ + "Newcombe CI width 31.4 pp" + ], + "ci_straddles_zero": true + }, + "sens_best_of_n_per_family": { + "underpowered": true, + "reasons": [ + "n<30 in an arm (n_a=12, n_b=12)", + "Newcombe CI width 57.3 pp" + ], + "ci_straddles_zero": true + }, + "sens_first_replicate_only": { + "underpowered": true, + "reasons": [ + "n<30 in an arm (n_a=12, n_b=36)", + "Newcombe CI width 55.4 pp" + ], + "ci_straddles_zero": true + }, + "sens_depth_matched": { + "underpowered": true, + "reasons": [ + "Newcombe CI width 35.8 pp" + ], + "ci_straddles_zero": true + } + } + }, + "P4": { + "pair": [ + "C36", + "C38" + ], + "description": "Thinking at T1/p0.95/pp0. Sampler-matched, but that triple is 3.6's vendor point and is NOT 3.8's -- 3.8 is running off-spec here and this pair is biased in 3.6's favour. 3.8 side pools three effort levels.", + "label_a": "3.6 think T1/p0.95/pp0", + "label_b": "3.8 think T1/p0.95/pp0", + "delivery": { + "terminal": { + "a": { + "x": 120, + "n": 120, + "p": 1.0, + "lo": 0.9689808335812966, + "hi": 1.0 + }, + "b": { + "x": 72, + "n": 72, + "p": 1.0, + "lo": 0.9493488274035422, + "hi": 1.0 + }, + "fisher_p": 1.0, + "diff": 0.0, + "diff_lo": -0.050651172596457794, + "diff_hi": 0.031019166418703392 + }, + "ungraded": { + "a": { + "x": 0, + "n": 120, + "p": 0.0, + "lo": 0.0, + "hi": 0.031019166418703486 + }, + "b": { + "x": 0, + "n": 72, + "p": 0.0, + "lo": 0.0, + "hi": 0.05065117259645781 + }, + "fisher_p": 1.0, + "diff": 0.0, + "diff_lo": -0.031019166418703486, + "diff_hi": 0.05065117259645781 + }, + "loop_freq30": { + "a": { + "x": 0, + "n": 120, + "p": 0.0, + "lo": 0.0, + "hi": 0.031019166418703486 + }, + "b": { + "x": 0, + "n": 72, + "p": 0.0, + "lo": 0.0, + "hi": 0.05065117259645781 + }, + "fisher_p": 1.0, + "diff": 0.0, + "diff_lo": -0.031019166418703486, + "diff_hi": 0.05065117259645781 + }, + "loop_run30": { + "a": { + "x": 0, + "n": 120, + "p": 0.0, + "lo": 0.0, + "hi": 0.031019166418703486 + }, + "b": { + "x": 0, + "n": 72, + "p": 0.0, + "lo": 0.0, + "hi": 0.05065117259645781 + }, + "fisher_p": 1.0, + "diff": 0.0, + "diff_lo": -0.031019166418703486, + "diff_hi": 0.05065117259645781 + } + }, + "quality": { + "pass_graded": { + "a": { + "x": 105, + "n": 120, + "p": 0.875, + "lo": 0.8039705979199262, + "hi": 0.9227650272660463 + }, + "b": { + "x": 54, + "n": 72, + "p": 0.75, + "lo": 0.6390649259565606, + "hi": 0.8356094877452105 + }, + "fisher_p": 0.030812294128087756, + "diff": -0.125, + "diff_lo": -0.24578115946930434, + "diff_hi": -0.013760796695364008 + }, + "pass_all": { + "a": { + "x": 105, + "n": 120, + "p": 0.875, + "lo": 0.8039705979199262, + "hi": 0.9227650272660463 + }, + "b": { + "x": 54, + "n": 72, + "p": 0.75, + "lo": 0.6390649259565606, + "hi": 0.8356094877452105 + }, + "fisher_p": 0.030812294128087756, + "diff": -0.125, + "diff_lo": -0.24578115946930434, + "diff_hi": -0.013760796695364008 + } + }, + "family_breakdown": [ + { + "family": "p1_bugfix", + "n_a": 10, + "pass_a": 10, + "graded_a": 10, + "loop_a": 0, + "n_b": 6, + "pass_b": 6, + "graded_b": 6, + "loop_b": 0, + "rate_a": 1.0, + "rate_b": 1.0, + "delta": 0.0, + "fisher_p": 1.0 + }, + { + "family": "p1_refactor", + "n_a": 10, + "pass_a": 7, + "graded_a": 10, + "loop_a": 0, + "n_b": 6, + "pass_b": 6, + "graded_b": 6, + "loop_b": 0, + "rate_a": 0.7, + "rate_b": 1.0, + "delta": 0.30000000000000004, + "fisher_p": 0.25 + }, + { + "family": "p1_testwrite", + "n_a": 10, + "pass_a": 9, + "graded_a": 10, + "loop_a": 0, + "n_b": 6, + "pass_b": 6, + "graded_b": 6, + "loop_b": 0, + "rate_a": 0.9, + "rate_b": 1.0, + "delta": 0.09999999999999998, + "fisher_p": 1.0 + }, + { + "family": "p2_ci", + "n_a": 10, + "pass_a": 10, + "graded_a": 10, + "loop_a": 0, + "n_b": 6, + "pass_b": 6, + "graded_b": 6, + "loop_b": 0, + "rate_a": 1.0, + "rate_b": 1.0, + "delta": 0.0, + "fisher_p": 1.0 + }, + { + "family": "p2_extract", + "n_a": 10, + "pass_a": 10, + "graded_a": 10, + "loop_a": 0, + "n_b": 6, + "pass_b": 6, + "graded_b": 6, + "loop_b": 0, + "rate_a": 1.0, + "rate_b": 1.0, + "delta": 0.0, + "fisher_p": 1.0 + }, + { + "family": "p2_hallucination", + "n_a": 10, + "pass_a": 10, + "graded_a": 10, + "loop_a": 0, + "n_b": 6, + "pass_b": 6, + "graded_b": 6, + "loop_b": 0, + "rate_a": 1.0, + "rate_b": 1.0, + "delta": 0.0, + "fisher_p": 1.0 + }, + { + "family": "p2_triage", + "n_a": 10, + "pass_a": 10, + "graded_a": 10, + "loop_a": 0, + "n_b": 6, + "pass_b": 6, + "graded_b": 6, + "loop_b": 0, + "rate_a": 1.0, + "rate_b": 1.0, + "delta": 0.0, + "fisher_p": 1.0 + }, + { + "family": "p3_business", + "n_a": 10, + "pass_a": 10, + "graded_a": 10, + "loop_a": 0, + "n_b": 6, + "pass_b": 0, + "graded_b": 6, + "loop_b": 0, + "rate_a": 1.0, + "rate_b": 0.0, + "delta": -1.0, + "fisher_p": 0.00012487512487512488 + }, + { + "family": "p3_doc", + "n_a": 10, + "pass_a": 10, + "graded_a": 10, + "loop_a": 0, + "n_b": 6, + "pass_b": 1, + "graded_b": 6, + "loop_b": 0, + "rate_a": 1.0, + "rate_b": 0.16666666666666666, + "delta": -0.8333333333333334, + "fisher_p": 0.0013736263736263737 + }, + { + "family": "p3_market", + "n_a": 10, + "pass_a": 10, + "graded_a": 10, + "loop_a": 0, + "n_b": 6, + "pass_b": 6, + "graded_b": 6, + "loop_b": 0, + "rate_a": 1.0, + "rate_b": 1.0, + "delta": 0.0, + "fisher_p": 1.0 + }, + { + "family": "p3_pm", + "n_a": 10, + "pass_a": 2, + "graded_a": 10, + "loop_a": 0, + "n_b": 6, + "pass_b": 5, + "graded_b": 6, + "loop_b": 0, + "rate_a": 0.2, + "rate_b": 0.8333333333333334, + "delta": 0.6333333333333333, + "fisher_p": 0.03496503496503497 + }, + { + "family": "p3_writing", + "n_a": 10, + "pass_a": 7, + "graded_a": 10, + "loop_a": 0, + "n_b": 6, + "pass_b": 0, + "graded_b": 6, + "loop_b": 0, + "rate_a": 0.7, + "rate_b": 0.0, + "delta": -0.7, + "fisher_p": 0.011363636363636364 + } + ], + "cost": { + "tokens": { + "shared_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_market", + "p3_pm", + "p3_writing" + ], + "per_family": [ + { + "family": "p1_bugfix", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 34634.0, + "median_b": 119290.5, + "delta": 84656.5, + "ratio": 3.44431772246925 + }, + { + "family": "p1_refactor", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 12829.5, + "median_b": 46639.0, + "delta": 33809.5, + "ratio": 3.6352936591449394 + }, + { + "family": "p1_testwrite", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 38600.0, + "median_b": 75151.5, + "delta": 36551.5, + "ratio": 1.9469300518134716 + }, + { + "family": "p2_ci", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 6799.5, + "median_b": 16928.5, + "delta": 10129.0, + "ratio": 2.4896683579674974 + }, + { + "family": "p2_extract", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 4722.0, + "median_b": 10503.5, + "delta": 5781.5, + "ratio": 2.22437526471834 + }, + { + "family": "p2_hallucination", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 7319.5, + "median_b": 16217.0, + "delta": 8897.5, + "ratio": 2.2155884964820003 + }, + { + "family": "p2_triage", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 9285.0, + "median_b": 18494.5, + "delta": 9209.5, + "ratio": 1.9918686052773291 + }, + { + "family": "p3_business", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 12102.0, + "median_b": 29339.5, + "delta": 17237.5, + "ratio": 2.4243513468848126 + }, + { + "family": "p3_doc", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 10542.0, + "median_b": 42326.5, + "delta": 31784.5, + "ratio": 4.015035097704421 + }, + { + "family": "p3_market", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 30441.0, + "median_b": 116923.5, + "delta": 86482.5, + "ratio": 3.8409874839854146 + }, + { + "family": "p3_pm", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 5020.0, + "median_b": 21210.0, + "delta": 16190.0, + "ratio": 4.225099601593626 + }, + { + "family": "p3_writing", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 8030.0, + "median_b": 36735.5, + "delta": 28705.5, + "ratio": 4.5747820672478205 + } + ], + "n_comparable_families": 12, + "comparable_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_market", + "p3_pm", + "p3_writing" + ], + "excluded_families": [], + "median_of_family_deltas": 22971.5, + "median_of_family_ratios": 2.966993040218374, + "median_of_family_ratios_ALL": 2.966993040218374, + "pooled_median_a": 9659.5, + "pooled_median_b": 22975.5, + "pooled_is_censored": false, + "coverage_a": 1.0, + "coverage_b": 1.0, + "cells_a": 120, + "cells_b": 72, + "with_data_a": 120, + "with_data_b": 72, + "n_families": 12 + }, + "elapsed": { + "shared_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_market", + "p3_pm", + "p3_writing" + ], + "per_family": [ + { + "family": "p1_bugfix", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 910.8, + "median_b": 3073.9, + "delta": 2163.1000000000004, + "ratio": 3.374945103205973 + }, + { + "family": "p1_refactor", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 234.05, + "median_b": 828.35, + "delta": 594.3, + "ratio": 3.539201025421918 + }, + { + "family": "p1_testwrite", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 657.0, + "median_b": 1486.95, + "delta": 829.95, + "ratio": 2.26324200913242 + }, + { + "family": "p2_ci", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 119.69999999999999, + "median_b": 277.45, + "delta": 157.75, + "ratio": 2.3178780284043445 + }, + { + "family": "p2_extract", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 73.75, + "median_b": 162.45, + "delta": 88.69999999999999, + "ratio": 2.2027118644067794 + }, + { + "family": "p2_hallucination", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 120.25, + "median_b": 258.95000000000005, + "delta": 138.70000000000005, + "ratio": 2.1534303534303536 + }, + { + "family": "p2_triage", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 140.7, + "median_b": 284.9, + "delta": 144.2, + "ratio": 2.0248756218905473 + }, + { + "family": "p3_business", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 182.95, + "median_b": 465.1, + "delta": 282.15000000000003, + "ratio": 2.5422246515441382 + }, + { + "family": "p3_doc", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 160.9, + "median_b": 686.3, + "delta": 525.4, + "ratio": 4.265382224984462 + }, + { + "family": "p3_market", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 634.05, + "median_b": 3293.9, + "delta": 2659.8500000000004, + "ratio": 5.195016165917515 + }, + { + "family": "p3_pm", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 78.4, + "median_b": 329.54999999999995, + "delta": 251.14999999999995, + "ratio": 4.2034438775510194 + }, + { + "family": "p3_writing", + "n_a": 10, + "n_b": 6, + "cells_a": 10, + "cells_b": 6, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": true, + "median_a": 127.55, + "median_b": 592.6, + "delta": 465.05, + "ratio": 4.646021168169345 + } + ], + "n_comparable_families": 12, + "comparable_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_market", + "p3_pm", + "p3_writing" + ], + "excluded_families": [], + "median_of_family_deltas": 373.6, + "median_of_family_ratios": 2.9585848773750554, + "median_of_family_ratios_ALL": 2.9585848773750554, + "pooled_median_a": 152.0, + "pooled_median_b": 386.15, + "pooled_is_censored": false, + "coverage_a": 1.0, + "coverage_b": 1.0, + "cells_a": 120, + "cells_b": 72, + "with_data_a": 120, + "with_data_b": 72, + "n_families": 12 + }, + "tokens_per_success_a": { + "total_tokens": 1798295.0, + "n_cells_with_tokens": 120, + "n_cells": 120, + "n_missing_tokens": 0, + "token_coverage": 1.0, + "passes": 105, + "tokens_per_success": 17126.619047619046, + "tokens_on_ungraded": 0, + "is_lower_bound": false, + "lower_bound_note": null + }, + "tokens_per_success_b": { + "total_tokens": 2989297.0, + "n_cells_with_tokens": 72, + "n_cells": 72, + "n_missing_tokens": 0, + "token_coverage": 1.0, + "passes": 54, + "tokens_per_success": 55357.351851851854, + "tokens_on_ungraded": 0, + "is_lower_bound": false, + "lower_bound_note": null + } + }, + "sensitivity": { + "loops_as_failure": { + "definition": "All cells in the group; passed==1 is success. Loops and ungraded cells score 0. This is the headline scoring.", + "stat": "binomial (Wilson/Newcombe/Fisher)", + "a": { + "x": 105, + "n": 120, + "p": 0.875, + "lo": 0.8039705979199262, + "hi": 0.9227650272660463 + }, + "b": { + "x": 54, + "n": 72, + "p": 0.75, + "lo": 0.6390649259565606, + "hi": 0.8356094877452105 + }, + "fisher_p": 0.030812294128087756, + "diff": -0.125, + "diff_lo": -0.24578115946930434, + "diff_hi": -0.013760796695364008 + }, + "loops_excluded": { + "definition": "Cells with looped_freq30==1 dropped entirely, then pass rate over the remainder. Charitable to whichever model loops more. NOTE: cells that are ungraded WITHOUT having looped are retained here and score 0.", + "stat": "binomial (Wilson/Newcombe/Fisher)", + "a": { + "x": 105, + "n": 120, + "p": 0.875, + "lo": 0.8039705979199262, + "hi": 0.9227650272660463 + }, + "b": { + "x": 54, + "n": 72, + "p": 0.75, + "lo": 0.6390649259565606, + "hi": 0.8356094877452105 + }, + "fisher_p": 0.030812294128087756, + "diff": -0.125, + "diff_lo": -0.24578115946930434, + "diff_hi": -0.013760796695364008 + }, + "graded_only": { + "definition": "Only graded==1 cells. Removes every delivery failure (loops and ungraded alike) and measures output quality conditional on delivery. Maximally charitable to a model that fails by not delivering.", + "stat": "binomial (Wilson/Newcombe/Fisher)", + "a": { + "x": 105, + "n": 120, + "p": 0.875, + "lo": 0.8039705979199262, + "hi": 0.9227650272660463 + }, + "b": { + "x": 54, + "n": 72, + "p": 0.75, + "lo": 0.6390649259565606, + "hi": 0.8356094877452105 + }, + "fisher_p": 0.030812294128087756, + "diff": -0.125, + "diff_lo": -0.24578115946930434, + "diff_hi": -0.013760796695364008 + }, + "loops_half_credit": { + "definition": "Score 1.0 for pass, 0.5 for a non-passing looped cell, 0.0 otherwise; compare MEAN scores. Scores are not 0/1 so Wilson and Fisher do not apply; CI is a z-interval on the difference of means.", + "stat": "z-interval on difference of means -- NOT Wilson/Fisher, NOT Welch-t", + "a": { + "mean": 0.875, + "n": 120 + }, + "b": { + "mean": 0.75, + "n": 72 + }, + "diff": -0.125, + "diff_lo": -0.24194208046041848, + "diff_hi": -0.008057919539581501, + "fisher_p": null + }, + "best_of_n_per_family": { + "definition": "One Bernoulli trial per SHARED family: 1 if any cell in that group/family passed. Measures 'can the model ever do this task', not reliability. n = number of families, so it is structurally low-powered.", + "stat": "binomial over families (Wilson/Newcombe/Fisher)", + "a": { + "x": 12, + "n": 12, + "p": 1.0, + "lo": 0.7575059933447592, + "hi": 1.0 + }, + "b": { + "x": 10, + "n": 12, + "p": 0.8333333333333334, + "lo": 0.5519691377470266, + "hi": 0.9530348578161463 + }, + "fisher_p": 0.4782608695652174, + "diff": -0.16666666666666663, + "diff_lo": -0.4480308622529734, + "diff_hi": 0.10376224862757744, + "per_family": { + "p1_bugfix": { + "a": 1, + "b": 1 + }, + "p1_refactor": { + "a": 1, + "b": 1 + }, + "p1_testwrite": { + "a": 1, + "b": 1 + }, + "p2_ci": { + "a": 1, + "b": 1 + }, + "p2_extract": { + "a": 1, + "b": 1 + }, + "p2_hallucination": { + "a": 1, + "b": 1 + }, + "p2_triage": { + "a": 1, + "b": 1 + }, + "p3_business": { + "a": 1, + "b": 0 + }, + "p3_doc": { + "a": 1, + "b": 1 + }, + "p3_market": { + "a": 1, + "b": 1 + }, + "p3_pm": { + "a": 1, + "b": 1 + }, + "p3_writing": { + "a": 1, + "b": 0 + } + } + }, + "first_replicate_only": { + "definition": "Only rows with replicate==1 (one run per family per arm). Removes any weighting from unequal repeat depth; retains one row per arm, so n is families x arms, not families.", + "stat": "binomial (Wilson/Newcombe/Fisher)", + "a": { + "x": 12, + "n": 12, + "p": 1.0, + "lo": 0.7575059933447592, + "hi": 1.0 + }, + "b": { + "x": 36, + "n": 48, + "p": 0.75, + "lo": 0.6121562719049204, + "hi": 0.8507936632620209 + }, + "fisher_p": 0.1006017713744576, + "diff": -0.25, + "diff_lo": -0.3878437280950796, + "diff_hi": 0.012607512873279436 + }, + "depth_matched": { + "definition": "Per shared family, the first k=min(n_a,n_b) cells from each side ordered by (arm, replicate, cell). Both sides then have identical family x depth composition.", + "stat": "binomial (Wilson/Newcombe/Fisher)", + "a": { + "x": 63, + "n": 72, + "p": 0.875, + "lo": 0.7791894627583916, + "hi": 0.9328221577942649 + }, + "b": { + "x": 54, + "n": 72, + "p": 0.75, + "lo": 0.6390649259565606, + "hi": 0.8356094877452105 + }, + "fisher_p": 0.08638632181194707, + "diff": -0.125, + "diff_lo": -0.25009993039569706, + "diff_hi": 0.0034859659204964366 + } + }, + "power": { + "pass_all": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": false + }, + "pass_graded": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": false + }, + "loop_freq30": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": true + }, + "loop_run30": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": true + }, + "ungraded": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": true + }, + "sens_loops_as_failure": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": false + }, + "sens_loops_excluded": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": false + }, + "sens_graded_only": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": false + }, + "sens_loops_half_credit": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": false + }, + "sens_best_of_n_per_family": { + "underpowered": true, + "reasons": [ + "n<30 in an arm (n_a=12, n_b=12)", + "Newcombe CI width 55.2 pp" + ], + "ci_straddles_zero": true + }, + "sens_first_replicate_only": { + "underpowered": true, + "reasons": [ + "n<30 in an arm (n_a=12, n_b=48)", + "Newcombe CI width 40.0 pp" + ], + "ci_straddles_zero": true + }, + "sens_depth_matched": { + "underpowered": false, + "reasons": [], + "ci_straddles_zero": true + } + } + } + }, + "effort_ladder": { + "rows": [ + { + "regime": "ALL (pooled)", + "effort": "low", + "n": 26, + "families": 12, + "n_graded": 26, + "loop_freq30": 0, + "pass_all": { + "x": 21, + "n": 26, + "p": 0.8076923076923077, + "lo": 0.6212374288068301, + "hi": 0.9149293885105758 + }, + "pass_graded": { + "x": 21, + "n": 26, + "p": 0.8076923076923077, + "lo": 0.6212374288068301, + "hi": 0.9149293885105758 + }, + "median_tokens": 16333.0, + "n_tokens": 26, + "median_elapsed": 261.70000000000005 + }, + { + "regime": "ALL (pooled)", + "effort": "medium", + "n": 36, + "families": 12, + "n_graded": 34, + "loop_freq30": 2, + "pass_all": { + "x": 21, + "n": 36, + "p": 0.5833333333333334, + "lo": 0.4220025157487166, + "hi": 0.7285943794855394 + }, + "pass_graded": { + "x": 21, + "n": 34, + "p": 0.6176470588235294, + "lo": 0.450410173444307, + "hi": 0.7609981667549109 + }, + "median_tokens": 13251.5, + "n_tokens": 34, + "median_elapsed": 223.75 + }, + { + "regime": "ALL (pooled)", + "effort": "xhigh", + "n": 61, + "families": 12, + "n_graded": 60, + "loop_freq30": 0, + "pass_all": { + "x": 45, + "n": 61, + "p": 0.7377049180327869, + "lo": 0.6156325821638563, + "hi": 0.8316121363763852 + }, + "pass_graded": { + "x": 45, + "n": 60, + "p": 0.75, + "lo": 0.6276792992295218, + "hi": 0.8422347746994333 + }, + "median_tokens": 36735.5, + "n_tokens": 60, + "median_elapsed": 592.6 + }, + { + "regime": "T0.3/p0.8/pp0", + "effort": "low", + "n": 14, + "families": 12, + "n_graded": 14, + "loop_freq30": 0, + "pass_all": { + "x": 11, + "n": 14, + "p": 0.7857142857142857, + "lo": 0.524107694133997, + "hi": 0.9242861328730683 + }, + "pass_graded": { + "x": 11, + "n": 14, + "p": 0.7857142857142857, + "lo": 0.524107694133997, + "hi": 0.9242861328730683 + }, + "median_tokens": 17366.5, + "n_tokens": 14, + "median_elapsed": 265.5 + }, + { + "regime": "T0.3/p0.8/pp0", + "effort": "medium", + "n": 24, + "families": 12, + "n_graded": 22, + "loop_freq30": 2, + "pass_all": { + "x": 13, + "n": 24, + "p": 0.5416666666666666, + "lo": 0.3507486535891948, + "hi": 0.72108666268789 + }, + "pass_graded": { + "x": 13, + "n": 22, + "p": 0.5909090909090909, + "lo": 0.3873481847631513, + "hi": 0.7674418380352814 + }, + "median_tokens": 10898.0, + "n_tokens": 22, + "median_elapsed": 223.75 + }, + { + "regime": "T0.3/p0.8/pp0", + "effort": "xhigh", + "n": 13, + "families": 12, + "n_graded": 12, + "loop_freq30": 0, + "pass_all": { + "x": 9, + "n": 13, + "p": 0.6923076923076923, + "lo": 0.42369343180549035, + "hi": 0.8731929634428948 + }, + "pass_graded": { + "x": 9, + "n": 12, + "p": 0.75, + "lo": 0.46769466506643426, + "hi": 0.9110583316059453 + }, + "median_tokens": 32094.5, + "n_tokens": 12, + "median_elapsed": 501.95 + }, + { + "regime": "T1/p0.95/pp0", + "effort": "low", + "n": 12, + "families": 12, + "n_graded": 12, + "loop_freq30": 0, + "pass_all": { + "x": 10, + "n": 12, + "p": 0.8333333333333334, + "lo": 0.5519691377470266, + "hi": 0.9530348578161463 + }, + "pass_graded": { + "x": 10, + "n": 12, + "p": 0.8333333333333334, + "lo": 0.5519691377470266, + "hi": 0.9530348578161463 + }, + "median_tokens": 14514.5, + "n_tokens": 12, + "median_elapsed": 231.95 + }, + { + "regime": "T1/p0.95/pp0", + "effort": "medium", + "n": 12, + "families": 12, + "n_graded": 12, + "loop_freq30": 0, + "pass_all": { + "x": 8, + "n": 12, + "p": 0.6666666666666666, + "lo": 0.39062208887279953, + "hi": 0.8618799089087867 + }, + "pass_graded": { + "x": 8, + "n": 12, + "p": 0.6666666666666666, + "lo": 0.39062208887279953, + "hi": 0.8618799089087867 + }, + "median_tokens": 14358.0, + "n_tokens": 12, + "median_elapsed": 225.60000000000002 + }, + { + "regime": "T1/p0.95/pp0", + "effort": "xhigh", + "n": 48, + "families": 12, + "n_graded": 48, + "loop_freq30": 0, + "pass_all": { + "x": 36, + "n": 48, + "p": 0.75, + "lo": 0.6121562719049204, + "hi": 0.8507936632620209 + }, + "pass_graded": { + "x": 36, + "n": 48, + "p": 0.75, + "lo": 0.6121562719049204, + "hi": 0.8507936632620209 + }, + "median_tokens": 38506.5, + "n_tokens": 48, + "median_elapsed": 623.3 + } + ], + "pairwise": [ + { + "regime": "T0.3/p0.8/pp0", + "lo": "low", + "hi": "medium", + "a": { + "x": 11, + "n": 14, + "p": 0.7857142857142857, + "lo": 0.524107694133997, + "hi": 0.9242861328730683 + }, + "b": { + "x": 13, + "n": 24, + "p": 0.5416666666666666, + "lo": 0.3507486535891948, + "hi": 0.72108666268789 + }, + "fisher_p": 0.17500967236948392, + "diff": -0.24404761904761907, + "diff_lo": -0.4799540508006484, + "diff_hi": 0.0731739813053704 + }, + { + "regime": "T0.3/p0.8/pp0", + "lo": "medium", + "hi": "xhigh", + "a": { + "x": 13, + "n": 24, + "p": 0.5416666666666666, + "lo": 0.3507486535891948, + "hi": 0.72108666268789 + }, + "b": { + "x": 9, + "n": 13, + "p": 0.6923076923076923, + "lo": 0.42369343180549035, + "hi": 0.8731929634428948 + }, + "fisher_p": 0.490846686508533, + "diff": 0.15064102564102566, + "diff_lo": -0.1723839825509108, + "diff_hi": 0.41364134699274074 + }, + { + "regime": "T0.3/p0.8/pp0", + "lo": "low", + "hi": "xhigh", + "a": { + "x": 11, + "n": 14, + "p": 0.7857142857142857, + "lo": 0.524107694133997, + "hi": 0.9242861328730683 + }, + "b": { + "x": 9, + "n": 13, + "p": 0.6923076923076923, + "lo": 0.42369343180549035, + "hi": 0.8731929634428948 + }, + "fisher_p": 0.6776167471819645, + "diff": -0.09340659340659341, + "diff_lo": -0.39565777664686097, + "diff_hi": 0.2246460595801592 + }, + { + "regime": "T1/p0.95/pp0", + "lo": "low", + "hi": "medium", + "a": { + "x": 10, + "n": 12, + "p": 0.8333333333333334, + "lo": 0.5519691377470266, + "hi": 0.9530348578161463 + }, + "b": { + "x": 8, + "n": 12, + "p": 0.6666666666666666, + "lo": 0.39062208887279953, + "hi": 0.8618799089087867 + }, + "fisher_p": 0.6404053612291598, + "diff": -0.16666666666666674, + "diff_lo": -0.4675471477432959, + "diff_hi": 0.17578627974292416 + }, + { + "regime": "T1/p0.95/pp0", + "lo": "medium", + "hi": "xhigh", + "a": { + "x": 8, + "n": 12, + "p": 0.6666666666666666, + "lo": 0.39062208887279953, + "hi": 0.8618799089087867 + }, + "b": { + "x": 36, + "n": 48, + "p": 0.75, + "lo": 0.6121562719049204, + "hi": 0.8507936632620209 + }, + "fisher_p": 0.716297870338546, + "diff": 0.08333333333333337, + "diff_lo": -0.15564177692259035, + "diff_hi": 0.3772040046892412 + }, + { + "regime": "T1/p0.95/pp0", + "lo": "low", + "hi": "xhigh", + "a": { + "x": 10, + "n": 12, + "p": 0.8333333333333334, + "lo": 0.5519691377470266, + "hi": 0.9530348578161463 + }, + "b": { + "x": 36, + "n": 48, + "p": 0.75, + "lo": 0.6121562719049204, + "hi": 0.8507936632620209 + }, + "fisher_p": 0.7134225109163015, + "diff": -0.08333333333333337, + "diff_lo": -0.26589660548482524, + "diff_hi": 0.21553983895167977 + }, + { + "regime": "ALL (pooled)", + "lo": "low", + "hi": "medium", + "a": { + "x": 21, + "n": 26, + "p": 0.8076923076923077, + "lo": 0.6212374288068301, + "hi": 0.9149293885105758 + }, + "b": { + "x": 21, + "n": 36, + "p": 0.5833333333333334, + "lo": 0.4220025157487166, + "hi": 0.7285943794855394 + }, + "fisher_p": 0.09799287939067336, + "diff": -0.22435897435897434, + "diff_lo": -0.418078937720897, + "diff_hi": 0.012001329815460732 + }, + { + "regime": "ALL (pooled)", + "lo": "medium", + "hi": "xhigh", + "a": { + "x": 21, + "n": 36, + "p": 0.5833333333333334, + "lo": 0.4220025157487166, + "hi": 0.7285943794855394 + }, + "b": { + "x": 45, + "n": 61, + "p": 0.7377049180327869, + "lo": 0.6156325821638563, + "hi": 0.8316121363763852 + }, + "fisher_p": 0.12287647101820744, + "diff": 0.15437158469945356, + "diff_lo": -0.03537146975506483, + "diff_hi": 0.341042949293901 + }, + { + "regime": "ALL (pooled)", + "lo": "low", + "hi": "xhigh", + "a": { + "x": 21, + "n": 26, + "p": 0.8076923076923077, + "lo": 0.6212374288068301, + "hi": 0.9149293885105758 + }, + "b": { + "x": 45, + "n": 61, + "p": 0.7377049180327869, + "lo": 0.6156325821638563, + "hi": 0.8316121363763852 + }, + "fisher_p": 0.5903154927739722, + "diff": -0.06998738965952078, + "diff_lo": -0.23247260956250387, + "diff_hi": 0.1387803940553771 + } + ], + "note": "Effort is populated only for 3.8 thinking cells; all 3.6 cells and all no-think cells have an empty effort field." + }, + "quant_control": { + "note": "Q8_0 arm at freeze #2 is 19 cells over all 12 families: 2 replicates on the 7 phase-1/phase-2 families, 1 on the 5 phase-3 families.", + "quality_provisional": true, + "quality_note": "8 of the 19 Q8_0 cells are graded in the frozen CSV (freeze #2), so the arm now carries PROVISIONAL quality information: as-graded pass-given-delivery is 5/8. The Wilson interval on that rate spans tens of percentage points, so no quality conclusion is drawn -- the graded outcomes are published as data, not as a quantization verdict. Delivery metrics (terminal, loop rates) cover all 19 cells and are the arm's primary evidence.", + "n_graded": 8, + "n_pass_graded": 5, + "n_fail_graded": 3, + "n_terminal": 19, + "n_with_tokens": 13, + "pass_graded": { + "x": 5, + "n": 8, + "p": 0.625, + "lo": 0.30574239460262737, + "hi": 0.8631557141764026 + }, + "pass_all": { + "x": 5, + "n": 19, + "p": 0.2631578947368421, + "lo": 0.11806412346782508, + "hi": 0.487915508978151 + }, + "loop_freq30": { + "x": 6, + "n": 19, + "p": 0.3157894736842105, + "lo": 0.1536437908779228, + "hi": 0.5398959232467253 + }, + "loop_run30": { + "x": 5, + "n": 19, + "p": 0.2631578947368421, + "lo": 0.11806412346782508, + "hi": 0.487915508978151 + }, + "post_freeze_drift": { + "source": "/home/michael/pr-staging/overlay/manifest.json", + "n_cells": 0, + "n_pass": 0, + "n_fail": 0, + "cells": [], + "note": "Verdicts written to disk AFTER the freeze; excluded from every number in this document and disclosed as drift." + }, + "q8_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_market", + "p3_pm", + "p3_writing" + ], + "reportable_metrics": [ + "terminal", + "looped_freq30", + "looped_run30", + "completion_tokens", + "elapsed_s", + "pass_graded (provisional, n=8)" + ], + "unscored_qualitative": { + "cell": "p3_doc_qwen38q8-nothink-matched_v2", + "status": "quarantined in-flight at freeze #2; NOT a row in the frozen CSV; excluded from every rate and every denominator in this document", + "observation": "139 iterations rewriting brief.md, context grown to ~228k tokens when quarantined", + "note": "UNSCORED qualitative evidence only. It documents that the Q8_0 arm can also exhibit a rewrite-loop failure shape (distinct from the identical-call loop the loop metrics count); it carries no weight in any rate above and is not part of the loop_freq30/loop_run30 counts." + }, + "vs_38_q4_same_sampler": { + "comparator": "A38 (3.8 Q4 no-think T0.3/p0.8/pp0) restricted to the 12 Q8 families", + "loop_freq30": { + "a": { + "x": 29, + "n": 95, + "p": 0.30526315789473685, + "lo": 0.22172919278467978, + "hi": 0.40393396069911497 + }, + "b": { + "x": 6, + "n": 19, + "p": 0.3157894736842105, + "lo": 0.1536437908779228, + "hi": 0.5398959232467253 + }, + "fisher_p": 1.0, + "diff": 0.010526315789473661, + "diff_lo": -0.17928187628552014, + "diff_hi": 0.2496949259006758 + }, + "loop_run30": { + "a": { + "x": 23, + "n": 95, + "p": 0.24210526315789474, + "lo": 0.16708840091876592, + "hi": 0.3371682077489622 + }, + "b": { + "x": 5, + "n": 19, + "p": 0.2631578947368421, + "lo": 0.11806412346782508, + "hi": 0.487915508978151 + }, + "fisher_p": 1.0, + "diff": 0.02105263157894735, + "diff_lo": -0.15240965796950232, + "diff_hi": 0.25799885919114096 + }, + "terminal": { + "a": { + "x": 93, + "n": 95, + "p": 0.9789473684210527, + "lo": 0.9264588050529105, + "hi": 0.9942074931355945 + }, + "b": { + "x": 19, + "n": 19, + "p": 1.0, + "lo": 0.8318207759473837, + "hi": 1.0 + }, + "fisher_p": 1.0, + "diff": 0.021052631578947323, + "diff_lo": -0.14781750435172425, + "diff_hi": 0.07354119494708955 + }, + "pass_graded": { + "note": "PROVISIONAL: graded-only pass rate, Q8 n=8. Underpowered by the power screen; direction only.", + "a": { + "x": 51, + "n": 65, + "p": 0.7846153846153846, + "lo": 0.6703343975431146, + "hi": 0.8671324230884898 + }, + "b": { + "x": 5, + "n": 8, + "p": 0.625, + "lo": 0.30574239460262737, + "hi": 0.8631557141764026 + }, + "fisher_p": 0.37820080949011786, + "diff": -0.1596153846153846, + "diff_lo": -0.4893645017369266, + "diff_hi": 0.10454041827579685 + }, + "tokens": { + "shared_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_market", + "p3_pm", + "p3_writing" + ], + "per_family": [ + { + "family": "p1_bugfix", + "n_a": 1, + "n_b": 0, + "cells_a": 12, + "cells_b": 2, + "coverage_a": 0.08333333333333333, + "coverage_b": 0.0, + "comparable": false, + "median_a": 102609.0, + "median_b": null, + "delta": null, + "ratio": null + }, + { + "family": "p1_refactor", + "n_a": 5, + "n_b": 1, + "cells_a": 7, + "cells_b": 2, + "coverage_a": 0.7142857142857143, + "coverage_b": 0.5, + "comparable": false, + "median_a": 11468.0, + "median_b": 11435.0, + "delta": -33.0, + "ratio": 0.9971224276246948 + }, + { + "family": "p1_testwrite", + "n_a": 2, + "n_b": 1, + "cells_a": 13, + "cells_b": 2, + "coverage_a": 0.15384615384615385, + "coverage_b": 0.5, + "comparable": false, + "median_a": 30595.5, + "median_b": 16665.0, + "delta": -13930.5, + "ratio": 0.5446879443055351 + }, + { + "family": "p2_ci", + "n_a": 7, + "n_b": 2, + "cells_a": 7, + "cells_b": 2, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 3748.0, + "median_b": 4148.0, + "delta": 400.0, + "ratio": 1.1067235859124867 + }, + { + "family": "p2_extract", + "n_a": 7, + "n_b": 2, + "cells_a": 7, + "cells_b": 2, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 2666.0, + "median_b": 2428.5, + "delta": -237.5, + "ratio": 0.9109152288072018 + }, + { + "family": "p2_hallucination", + "n_a": 7, + "n_b": 2, + "cells_a": 7, + "cells_b": 2, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 4377.0, + "median_b": 4381.0, + "delta": 4.0, + "ratio": 1.0009138679460818 + }, + { + "family": "p2_triage", + "n_a": 7, + "n_b": 2, + "cells_a": 7, + "cells_b": 2, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 6289.0, + "median_b": 7102.5, + "delta": 813.5, + "ratio": 1.129352838289076 + }, + { + "family": "p3_business", + "n_a": 6, + "n_b": 1, + "cells_a": 7, + "cells_b": 1, + "coverage_a": 0.8571428571428571, + "coverage_b": 1.0, + "comparable": false, + "median_a": 10344.5, + "median_b": 8334.0, + "delta": -2010.5, + "ratio": 0.8056455121078834 + }, + { + "family": "p3_doc", + "n_a": 6, + "n_b": 1, + "cells_a": 7, + "cells_b": 1, + "coverage_a": 0.8571428571428571, + "coverage_b": 1.0, + "comparable": false, + "median_a": 7108.0, + "median_b": 15304.0, + "delta": 8196.0, + "ratio": 2.153066966797974 + }, + { + "family": "p3_market", + "n_a": 3, + "n_b": 0, + "cells_a": 7, + "cells_b": 1, + "coverage_a": 0.42857142857142855, + "coverage_b": 0.0, + "comparable": false, + "median_a": 103162.0, + "median_b": null, + "delta": null, + "ratio": null + }, + { + "family": "p3_pm", + "n_a": 7, + "n_b": 1, + "cells_a": 7, + "cells_b": 1, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 3308.0, + "median_b": 2935.0, + "delta": -373.0, + "ratio": 0.8872430471584039 + }, + { + "family": "p3_writing", + "n_a": 7, + "n_b": 0, + "cells_a": 7, + "cells_b": 1, + "coverage_a": 1.0, + "coverage_b": 0.0, + "comparable": false, + "median_a": 4927.0, + "median_b": null, + "delta": null, + "ratio": null + } + ], + "n_comparable_families": 0, + "comparable_families": [], + "excluded_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_market", + "p3_pm", + "p3_writing" + ], + "median_of_family_deltas": null, + "median_of_family_ratios": null, + "median_of_family_ratios_ALL": 0.9971224276246948, + "pooled_median_a": 5264.0, + "pooled_median_b": 4446.0, + "pooled_is_censored": true, + "coverage_a": 0.6842105263157895, + "coverage_b": 0.6842105263157895, + "cells_a": 95, + "cells_b": 19, + "with_data_a": 65, + "with_data_b": 13, + "n_families": 12 + }, + "elapsed": { + "shared_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_market", + "p3_pm", + "p3_writing" + ], + "per_family": [ + { + "family": "p1_bugfix", + "n_a": 1, + "n_b": 0, + "cells_a": 12, + "cells_b": 2, + "coverage_a": 0.08333333333333333, + "coverage_b": 0.0, + "comparable": false, + "median_a": 4860.3, + "median_b": null, + "delta": null, + "ratio": null + }, + { + "family": "p1_refactor", + "n_a": 5, + "n_b": 1, + "cells_a": 7, + "cells_b": 2, + "coverage_a": 0.7142857142857143, + "coverage_b": 0.5, + "comparable": false, + "median_a": 195.0, + "median_b": 259.2, + "delta": 64.19999999999999, + "ratio": 1.3292307692307692 + }, + { + "family": "p1_testwrite", + "n_a": 2, + "n_b": 1, + "cells_a": 13, + "cells_b": 2, + "coverage_a": 0.15384615384615385, + "coverage_b": 0.5, + "comparable": false, + "median_a": 684.05, + "median_b": 597.3, + "delta": -86.75, + "ratio": 0.87318178495724 + }, + { + "family": "p2_ci", + "n_a": 7, + "n_b": 2, + "cells_a": 7, + "cells_b": 2, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 71.7, + "median_b": 97.45, + "delta": 25.75, + "ratio": 1.3591352859135286 + }, + { + "family": "p2_extract", + "n_a": 7, + "n_b": 2, + "cells_a": 7, + "cells_b": 2, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 42.8, + "median_b": 53.75, + "delta": 10.950000000000003, + "ratio": 1.2558411214953271 + }, + { + "family": "p2_hallucination", + "n_a": 7, + "n_b": 2, + "cells_a": 7, + "cells_b": 2, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 72.4, + "median_b": 99.25, + "delta": 26.849999999999994, + "ratio": 1.3708563535911602 + }, + { + "family": "p2_triage", + "n_a": 7, + "n_b": 2, + "cells_a": 7, + "cells_b": 2, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 97.0, + "median_b": 151.8, + "delta": 54.80000000000001, + "ratio": 1.5649484536082476 + }, + { + "family": "p3_business", + "n_a": 6, + "n_b": 1, + "cells_a": 7, + "cells_b": 1, + "coverage_a": 0.8571428571428571, + "coverage_b": 1.0, + "comparable": false, + "median_a": 157.95, + "median_b": 176.6, + "delta": 18.650000000000006, + "ratio": 1.1180753402975625 + }, + { + "family": "p3_doc", + "n_a": 6, + "n_b": 1, + "cells_a": 7, + "cells_b": 1, + "coverage_a": 0.8571428571428571, + "coverage_b": 1.0, + "comparable": false, + "median_a": 108.5, + "median_b": 326.8, + "delta": 218.3, + "ratio": 3.0119815668202765 + }, + { + "family": "p3_market", + "n_a": 3, + "n_b": 0, + "cells_a": 7, + "cells_b": 1, + "coverage_a": 0.42857142857142855, + "coverage_b": 0.0, + "comparable": false, + "median_a": 2914.9, + "median_b": null, + "delta": null, + "ratio": null + }, + { + "family": "p3_pm", + "n_a": 7, + "n_b": 1, + "cells_a": 7, + "cells_b": 1, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 50.7, + "median_b": 65.2, + "delta": 14.5, + "ratio": 1.2859960552268244 + }, + { + "family": "p3_writing", + "n_a": 7, + "n_b": 0, + "cells_a": 7, + "cells_b": 1, + "coverage_a": 1.0, + "coverage_b": 0.0, + "comparable": false, + "median_a": 77.4, + "median_b": null, + "delta": null, + "ratio": null + } + ], + "n_comparable_families": 0, + "comparable_families": [], + "excluded_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_market", + "p3_pm", + "p3_writing" + ], + "median_of_family_deltas": null, + "median_of_family_ratios": null, + "median_of_family_ratios_ALL": 1.3292307692307692, + "pooled_median_a": 83.9, + "pooled_median_b": 101.5, + "pooled_is_censored": true, + "coverage_a": 0.6842105263157895, + "coverage_b": 0.6842105263157895, + "cells_a": 95, + "cells_b": 19, + "with_data_a": 65, + "with_data_b": 13, + "n_families": 12 + } + }, + "vs_36_q4_same_sampler": { + "comparator": "A36 (3.6 Q4 no-think T0.3/p0.8/pp0) restricted to the 12 Q8 families", + "loop_freq30": { + "a": { + "x": 3, + "n": 121, + "p": 0.024793388429752067, + "lo": 0.008467314305269862, + "hi": 0.07036434076877973 + }, + "b": { + "x": 6, + "n": 19, + "p": 0.3157894736842105, + "lo": 0.1536437908779228, + "hi": 0.5398959232467253 + }, + "fisher_p": 0.00018699213339247746, + "diff": 0.2909960852544584, + "diff_lo": 0.12256825456783108, + "diff_hi": 0.5156964222186786 + }, + "loop_run30": { + "a": { + "x": 3, + "n": 121, + "p": 0.024793388429752067, + "lo": 0.008467314305269862, + "hi": 0.07036434076877973 + }, + "b": { + "x": 5, + "n": 19, + "p": 0.2631578947368421, + "lo": 0.11806412346782508, + "hi": 0.487915508978151 + }, + "fisher_p": 0.001189584385959215, + "diff": 0.23836450630709002, + "diff_lo": 0.08628257392144548, + "diff_hi": 0.4637142918755771 + }, + "terminal": { + "a": { + "x": 118, + "n": 121, + "p": 0.9752066115702479, + "lo": 0.9296356592312203, + "hi": 0.9915326856947302 + }, + "b": { + "x": 19, + "n": 19, + "p": 1.0, + "lo": 0.8318207759473837, + "hi": 1.0 + }, + "fisher_p": 1.0, + "diff": 0.024793388429752095, + "diff_lo": -0.14417640798149475, + "diff_hi": 0.07036434076877973 + }, + "tokens": { + "shared_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_market", + "p3_pm", + "p3_writing" + ], + "per_family": [ + { + "family": "p1_bugfix", + "n_a": 18, + "n_b": 0, + "cells_a": 19, + "cells_b": 2, + "coverage_a": 0.9473684210526315, + "coverage_b": 0.0, + "comparable": false, + "median_a": 31183.0, + "median_b": null, + "delta": null, + "ratio": null + }, + { + "family": "p1_refactor", + "n_a": 9, + "n_b": 1, + "cells_a": 9, + "cells_b": 2, + "coverage_a": 1.0, + "coverage_b": 0.5, + "comparable": false, + "median_a": 11033.0, + "median_b": 11435.0, + "delta": 402.0, + "ratio": 1.0364361461071332 + }, + { + "family": "p1_testwrite", + "n_a": 10, + "n_b": 1, + "cells_a": 12, + "cells_b": 2, + "coverage_a": 0.8333333333333334, + "coverage_b": 0.5, + "comparable": false, + "median_a": 36840.0, + "median_b": 16665.0, + "delta": -20175.0, + "ratio": 0.4523615635179153 + }, + { + "family": "p2_ci", + "n_a": 9, + "n_b": 2, + "cells_a": 9, + "cells_b": 2, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 4600.0, + "median_b": 4148.0, + "delta": -452.0, + "ratio": 0.9017391304347826 + }, + { + "family": "p2_extract", + "n_a": 9, + "n_b": 2, + "cells_a": 9, + "cells_b": 2, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 2871.0, + "median_b": 2428.5, + "delta": -442.5, + "ratio": 0.8458725182863114 + }, + { + "family": "p2_hallucination", + "n_a": 9, + "n_b": 2, + "cells_a": 9, + "cells_b": 2, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 5101.0, + "median_b": 4381.0, + "delta": -720.0, + "ratio": 0.8588512056459517 + }, + { + "family": "p2_triage", + "n_a": 9, + "n_b": 2, + "cells_a": 9, + "cells_b": 2, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 7707.0, + "median_b": 7102.5, + "delta": -604.5, + "ratio": 0.9215648112105878 + }, + { + "family": "p3_business", + "n_a": 9, + "n_b": 1, + "cells_a": 9, + "cells_b": 1, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 9503.0, + "median_b": 8334.0, + "delta": -1169.0, + "ratio": 0.8769862148795118 + }, + { + "family": "p3_doc", + "n_a": 9, + "n_b": 1, + "cells_a": 9, + "cells_b": 1, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 7801.0, + "median_b": 15304.0, + "delta": 7503.0, + "ratio": 1.9617997692603513 + }, + { + "family": "p3_market", + "n_a": 5, + "n_b": 0, + "cells_a": 9, + "cells_b": 1, + "coverage_a": 0.5555555555555556, + "coverage_b": 0.0, + "comparable": false, + "median_a": 40432.0, + "median_b": null, + "delta": null, + "ratio": null + }, + { + "family": "p3_pm", + "n_a": 9, + "n_b": 1, + "cells_a": 9, + "cells_b": 1, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 4634.0, + "median_b": 2935.0, + "delta": -1699.0, + "ratio": 0.6333621061717738 + }, + { + "family": "p3_writing", + "n_a": 9, + "n_b": 0, + "cells_a": 9, + "cells_b": 1, + "coverage_a": 1.0, + "coverage_b": 0.0, + "comparable": false, + "median_a": 6427.0, + "median_b": null, + "delta": null, + "ratio": null + } + ], + "n_comparable_families": 0, + "comparable_families": [], + "excluded_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_market", + "p3_pm", + "p3_writing" + ], + "median_of_family_deltas": null, + "median_of_family_ratios": null, + "median_of_family_ratios_ALL": 0.8769862148795118, + "pooled_median_a": 7816.0, + "pooled_median_b": 4446.0, + "pooled_is_censored": true, + "coverage_a": 0.9421487603305785, + "coverage_b": 0.6842105263157895, + "cells_a": 121, + "cells_b": 19, + "with_data_a": 114, + "with_data_b": 13, + "n_families": 12 + }, + "elapsed": { + "shared_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_market", + "p3_pm", + "p3_writing" + ], + "per_family": [ + { + "family": "p1_bugfix", + "n_a": 18, + "n_b": 0, + "cells_a": 19, + "cells_b": 2, + "coverage_a": 0.9473684210526315, + "coverage_b": 0.0, + "comparable": false, + "median_a": 894.25, + "median_b": null, + "delta": null, + "ratio": null + }, + { + "family": "p1_refactor", + "n_a": 9, + "n_b": 1, + "cells_a": 9, + "cells_b": 2, + "coverage_a": 1.0, + "coverage_b": 0.5, + "comparable": false, + "median_a": 212.7, + "median_b": 259.2, + "delta": 46.5, + "ratio": 1.218617771509168 + }, + { + "family": "p1_testwrite", + "n_a": 10, + "n_b": 1, + "cells_a": 12, + "cells_b": 2, + "coverage_a": 0.8333333333333334, + "coverage_b": 0.5, + "comparable": false, + "median_a": 638.15, + "median_b": 597.3, + "delta": -40.85000000000002, + "ratio": 0.9359868369505602 + }, + { + "family": "p2_ci", + "n_a": 9, + "n_b": 2, + "cells_a": 9, + "cells_b": 2, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 82.6, + "median_b": 97.45, + "delta": 14.850000000000009, + "ratio": 1.1797820823244554 + }, + { + "family": "p2_extract", + "n_a": 9, + "n_b": 2, + "cells_a": 9, + "cells_b": 2, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 45.5, + "median_b": 53.75, + "delta": 8.25, + "ratio": 1.1813186813186813 + }, + { + "family": "p2_hallucination", + "n_a": 9, + "n_b": 2, + "cells_a": 9, + "cells_b": 2, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 83.1, + "median_b": 99.25, + "delta": 16.150000000000006, + "ratio": 1.1943441636582433 + }, + { + "family": "p2_triage", + "n_a": 9, + "n_b": 2, + "cells_a": 9, + "cells_b": 2, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 118.3, + "median_b": 151.8, + "delta": 33.500000000000014, + "ratio": 1.283178360101437 + }, + { + "family": "p3_business", + "n_a": 9, + "n_b": 1, + "cells_a": 9, + "cells_b": 1, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 146.3, + "median_b": 176.6, + "delta": 30.299999999999983, + "ratio": 1.2071086807928912 + }, + { + "family": "p3_doc", + "n_a": 9, + "n_b": 1, + "cells_a": 9, + "cells_b": 1, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 120.1, + "median_b": 326.8, + "delta": 206.70000000000002, + "ratio": 2.721065778517902 + }, + { + "family": "p3_market", + "n_a": 5, + "n_b": 0, + "cells_a": 9, + "cells_b": 1, + "coverage_a": 0.5555555555555556, + "coverage_b": 0.0, + "comparable": false, + "median_a": 943.4, + "median_b": null, + "delta": null, + "ratio": null + }, + { + "family": "p3_pm", + "n_a": 9, + "n_b": 1, + "cells_a": 9, + "cells_b": 1, + "coverage_a": 1.0, + "coverage_b": 1.0, + "comparable": false, + "median_a": 71.5, + "median_b": 65.2, + "delta": -6.299999999999997, + "ratio": 0.9118881118881119 + }, + { + "family": "p3_writing", + "n_a": 9, + "n_b": 0, + "cells_a": 9, + "cells_b": 1, + "coverage_a": 1.0, + "coverage_b": 0.0, + "comparable": false, + "median_a": 104.0, + "median_b": null, + "delta": null, + "ratio": null + } + ], + "n_comparable_families": 0, + "comparable_families": [], + "excluded_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p2_triage", + "p3_business", + "p3_doc", + "p3_market", + "p3_pm", + "p3_writing" + ], + "median_of_family_deltas": null, + "median_of_family_ratios": null, + "median_of_family_ratios_ALL": 1.1943441636582433, + "pooled_median_a": 120.6, + "pooled_median_b": 101.5, + "pooled_is_censored": true, + "coverage_a": 0.9421487603305785, + "coverage_b": 0.6842105263157895, + "cells_a": 121, + "cells_b": 19, + "with_data_a": 114, + "with_data_b": 13, + "n_families": 12 + } + }, + "q8_cells": [ + { + "cell": "p1_bugfix_qwen38q8-nothink-matched_v1", + "family": "p1_bugfix", + "replicate": 1, + "verdict": "", + "graded": 0, + "passed": 0, + "terminal": 1, + "loop_f": 1, + "loop_r": 1, + "tokens": null, + "elapsed": null + }, + { + "cell": "p1_bugfix_qwen38q8-nothink-matched_v2", + "family": "p1_bugfix", + "replicate": 2, + "verdict": "", + "graded": 0, + "passed": 0, + "terminal": 1, + "loop_f": 0, + "loop_r": 0, + "tokens": null, + "elapsed": null + }, + { + "cell": "p1_refactor_qwen38q8-nothink-matched_v1", + "family": "p1_refactor", + "replicate": 1, + "verdict": "PASS", + "graded": 1, + "passed": 1, + "terminal": 1, + "loop_f": 0, + "loop_r": 0, + "tokens": 11435.0, + "elapsed": 259.2 + }, + { + "cell": "p1_refactor_qwen38q8-nothink-matched_v2", + "family": "p1_refactor", + "replicate": 2, + "verdict": "", + "graded": 0, + "passed": 0, + "terminal": 1, + "loop_f": 1, + "loop_r": 1, + "tokens": null, + "elapsed": null + }, + { + "cell": "p1_testwrite_qwen38q8-nothink-matched_v1", + "family": "p1_testwrite", + "replicate": 1, + "verdict": "", + "graded": 0, + "passed": 0, + "terminal": 1, + "loop_f": 1, + "loop_r": 1, + "tokens": null, + "elapsed": null + }, + { + "cell": "p1_testwrite_qwen38q8-nothink-matched_v2", + "family": "p1_testwrite", + "replicate": 2, + "verdict": "", + "graded": 0, + "passed": 0, + "terminal": 1, + "loop_f": 1, + "loop_r": 0, + "tokens": 16665.0, + "elapsed": 597.3 + }, + { + "cell": "p2_ci_qwen38q8-nothink-matched_v1", + "family": "p2_ci", + "replicate": 1, + "verdict": "PASS", + "graded": 1, + "passed": 1, + "terminal": 1, + "loop_f": 0, + "loop_r": 0, + "tokens": 4136.0, + "elapsed": 96.2 + }, + { + "cell": "p2_ci_qwen38q8-nothink-matched_v2", + "family": "p2_ci", + "replicate": 2, + "verdict": "", + "graded": 0, + "passed": 0, + "terminal": 1, + "loop_f": 0, + "loop_r": 0, + "tokens": 4160.0, + "elapsed": 98.7 + }, + { + "cell": "p2_extract_qwen38q8-nothink-matched_v1", + "family": "p2_extract", + "replicate": 1, + "verdict": "PASS", + "graded": 1, + "passed": 1, + "terminal": 1, + "loop_f": 0, + "loop_r": 0, + "tokens": 2422.0, + "elapsed": 53.0 + }, + { + "cell": "p2_extract_qwen38q8-nothink-matched_v2", + "family": "p2_extract", + "replicate": 2, + "verdict": "", + "graded": 0, + "passed": 0, + "terminal": 1, + "loop_f": 0, + "loop_r": 0, + "tokens": 2435.0, + "elapsed": 54.5 + }, + { + "cell": "p2_hallucination_qwen38q8-nothink-matched_v1", + "family": "p2_hallucination", + "replicate": 1, + "verdict": "PASS", + "graded": 1, + "passed": 1, + "terminal": 1, + "loop_f": 0, + "loop_r": 0, + "tokens": 4316.0, + "elapsed": 97.0 + }, + { + "cell": "p2_hallucination_qwen38q8-nothink-matched_v2", + "family": "p2_hallucination", + "replicate": 2, + "verdict": "", + "graded": 0, + "passed": 0, + "terminal": 1, + "loop_f": 0, + "loop_r": 0, + "tokens": 4446.0, + "elapsed": 101.5 + }, + { + "cell": "p2_triage_qwen38q8-nothink-matched_v1", + "family": "p2_triage", + "replicate": 1, + "verdict": "PASS", + "graded": 1, + "passed": 1, + "terminal": 1, + "loop_f": 0, + "loop_r": 0, + "tokens": 6771.0, + "elapsed": 144.5 + }, + { + "cell": "p2_triage_qwen38q8-nothink-matched_v2", + "family": "p2_triage", + "replicate": 2, + "verdict": "", + "graded": 0, + "passed": 0, + "terminal": 1, + "loop_f": 0, + "loop_r": 0, + "tokens": 7434.0, + "elapsed": 159.1 + }, + { + "cell": "p3_business_qwen38q8-nothink-matched_v1", + "family": "p3_business", + "replicate": 1, + "verdict": "FAIL", + "graded": 1, + "passed": 0, + "terminal": 1, + "loop_f": 0, + "loop_r": 0, + "tokens": 8334.0, + "elapsed": 176.6 + }, + { + "cell": "p3_doc_qwen38q8-nothink-matched_v1", + "family": "p3_doc", + "replicate": 1, + "verdict": "FAIL", + "graded": 1, + "passed": 0, + "terminal": 1, + "loop_f": 0, + "loop_r": 0, + "tokens": 15304.0, + "elapsed": 326.8 + }, + { + "cell": "p3_market_qwen38q8-nothink-matched_v1", + "family": "p3_market", + "replicate": 1, + "verdict": "", + "graded": 0, + "passed": 0, + "terminal": 1, + "loop_f": 1, + "loop_r": 1, + "tokens": null, + "elapsed": null + }, + { + "cell": "p3_pm_qwen38q8-nothink-matched_v1", + "family": "p3_pm", + "replicate": 1, + "verdict": "FAIL", + "graded": 1, + "passed": 0, + "terminal": 1, + "loop_f": 0, + "loop_r": 0, + "tokens": 2935.0, + "elapsed": 65.2 + }, + { + "cell": "p3_writing_qwen38q8-nothink-matched_v1", + "family": "p3_writing", + "replicate": 1, + "verdict": "", + "graded": 0, + "passed": 0, + "terminal": 1, + "loop_f": 1, + "loop_r": 1, + "tokens": null, + "elapsed": null + } + ] + }, + "defect_exposure": { + "A36": { + "n": 121, + "d1_length_gated": { + "n": 36, + "passes": 18, + "graded": 36 + }, + "d2_p3_pm": { + "n": 9, + "passes": 1, + "graded": 9 + }, + "d3_p2_triage": { + "n": 9, + "passes": 0, + "graded": 9 + }, + "union": { + "n": 45, + "passes": 18 + } + }, + "A38": { + "n": 95, + "d1_length_gated": { + "n": 28, + "passes": 20, + "graded": 26 + }, + "d2_p3_pm": { + "n": 7, + "passes": 7, + "graded": 7 + }, + "d3_p2_triage": { + "n": 7, + "passes": 7, + "graded": 7 + }, + "union": { + "n": 35, + "passes": 27 + } + }, + "V36": { + "n": 120, + "d1_length_gated": { + "n": 40, + "passes": 21, + "graded": 40 + }, + "d2_p3_pm": { + "n": 10, + "passes": 1, + "graded": 10 + }, + "d3_p2_triage": { + "n": 10, + "passes": 7, + "graded": 10 + }, + "union": { + "n": 50, + "passes": 28 + } + }, + "V38": { + "n": 96, + "d1_length_gated": { + "n": 32, + "passes": 11, + "graded": 26 + }, + "d2_p3_pm": { + "n": 8, + "passes": 8, + "graded": 8 + }, + "d3_p2_triage": { + "n": 8, + "passes": 5, + "graded": 8 + }, + "union": { + "n": 40, + "passes": 16 + } + }, + "B36": { + "n": 108, + "d1_length_gated": { + "n": 36, + "passes": 10, + "graded": 36 + }, + "d2_p3_pm": { + "n": 9, + "passes": 0, + "graded": 9 + }, + "d3_p2_triage": { + "n": 9, + "passes": 0, + "graded": 9 + }, + "union": { + "n": 45, + "passes": 10 + } + }, + "B38": { + "n": 51, + "d1_length_gated": { + "n": 16, + "passes": 5, + "graded": 16 + }, + "d2_p3_pm": { + "n": 4, + "passes": 1, + "graded": 4 + }, + "d3_p2_triage": { + "n": 4, + "passes": 4, + "graded": 4 + }, + "union": { + "n": 20, + "passes": 9 + } + }, + "C36": { + "n": 120, + "d1_length_gated": { + "n": 40, + "passes": 29, + "graded": 40 + }, + "d2_p3_pm": { + "n": 10, + "passes": 2, + "graded": 10 + }, + "d3_p2_triage": { + "n": 10, + "passes": 10, + "graded": 10 + }, + "union": { + "n": 50, + "passes": 39 + } + }, + "C38": { + "n": 72, + "d1_length_gated": { + "n": 24, + "passes": 6, + "graded": 24 + }, + "d2_p3_pm": { + "n": 6, + "passes": 5, + "graded": 6 + }, + "d3_p2_triage": { + "n": 6, + "passes": 6, + "graded": 6 + }, + "union": { + "n": 30, + "passes": 12 + } + }, + "Q8": { + "n": 19, + "d1_length_gated": { + "n": 4, + "passes": 0, + "graded": 3 + }, + "d2_p3_pm": { + "n": 1, + "passes": 0, + "graded": 1 + }, + "d3_p2_triage": { + "n": 2, + "passes": 1, + "graded": 1 + }, + "union": { + "n": 6, + "passes": 1 + } + } + }, + "defect_free_subset": { + "P1": { + "clean_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p3_market" + ], + "a": { + "x": 63, + "n": 76, + "p": 0.8289473684210527, + "lo": 0.729021253009869, + "hi": 0.8972198094472776 + }, + "b": { + "x": 24, + "n": 60, + "p": 0.4, + "lo": 0.2856949053488255, + "hi": 0.5263394650795924 + }, + "fisher_p": 2.8984698310373e-07, + "diff": -0.42894736842105263, + "diff_lo": -0.5620893259960159, + "diff_hi": -0.2678669879342698 + }, + "P2": { + "clean_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p3_market" + ], + "a": { + "x": 67, + "n": 70, + "p": 0.9571428571428572, + "lo": 0.8814033125147246, + "hi": 0.9853184589958887 + }, + "b": { + "x": 32, + "n": 56, + "p": 0.5714285714285714, + "lo": 0.4413762073081303, + "hi": 0.6923103731606176 + }, + "fisher_p": 1.1589592463919984e-07, + "diff": -0.3857142857142858, + "diff_lo": -0.5187837486015175, + "diff_hi": -0.24306475500891467 + }, + "P3": { + "clean_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p3_market" + ], + "a": { + "x": 57, + "n": 63, + "p": 0.9047619047619048, + "lo": 0.8073829893716311, + "hi": 0.955616515723108 + }, + "b": { + "x": 24, + "n": 31, + "p": 0.7741935483870968, + "lo": 0.6018758161665794, + "hi": 0.886048644387375 + }, + "fisher_p": 0.11328971633793, + "diff": -0.13056835637480801, + "diff_lo": -0.310233579202258, + "diff_hi": 0.017736113091953276 + }, + "P4": { + "clean_families": [ + "p1_bugfix", + "p1_refactor", + "p1_testwrite", + "p2_ci", + "p2_extract", + "p2_hallucination", + "p3_market" + ], + "a": { + "x": 66, + "n": 70, + "p": 0.9428571428571428, + "lo": 0.8620804952516177, + "hi": 0.9775562208992887 + }, + "b": { + "x": 42, + "n": 42, + "p": 1.0, + "lo": 0.916201209134296, + "hi": 1.0 + }, + "fisher_p": 0.2950375312760634, + "diff": 0.05714285714285716, + "diff_lo": -0.033555893496437325, + "diff_hi": 0.13791950474838233 + } + }, + "family_pass_asgraded": { + "p1_bugfix": { + "3.6": { + "n": 48, + "graded": 47, + "passes": 47 + }, + "3.8": { + "n": 34, + "graded": 16, + "passes": 14 + } + }, + "p1_refactor": { + "3.6": { + "n": 38, + "graded": 38, + "passes": 24 + }, + "3.8": { + "n": 27, + "graded": 24, + "passes": 17 + } + }, + "p1_testwrite": { + "3.6": { + "n": 41, + "graded": 39, + "passes": 36 + }, + "3.8": { + "n": 34, + "graded": 20, + "passes": 10 + } + }, + "p2_ci": { + "3.6": { + "n": 38, + "graded": 38, + "passes": 38 + }, + "3.8": { + "n": 27, + "graded": 26, + "passes": 26 + } + }, + "p2_extract": { + "3.6": { + "n": 38, + "graded": 38, + "passes": 38 + }, + "3.8": { + "n": 27, + "graded": 26, + "passes": 26 + } + }, + "p2_hallucination": { + "3.6": { + "n": 38, + "graded": 38, + "passes": 38 + }, + "3.8": { + "n": 27, + "graded": 26, + "passes": 26 + } + }, + "p2_triage": { + "3.6": { + "n": 38, + "graded": 38, + "passes": 17 + }, + "3.8": { + "n": 27, + "graded": 26, + "passes": 23 + } + }, + "p3_business": { + "3.6": { + "n": 38, + "graded": 38, + "passes": 35 + }, + "3.8": { + "n": 26, + "graded": 23, + "passes": 4 + } + }, + "p3_doc": { + "3.6": { + "n": 38, + "graded": 38, + "passes": 28 + }, + "3.8": { + "n": 26, + "graded": 25, + "passes": 9 + } + }, + "p3_market": { + "3.6": { + "n": 38, + "graded": 33, + "passes": 32 + }, + "3.8": { + "n": 26, + "graded": 13, + "passes": 7 + } + }, + "p3_pm": { + "3.6": { + "n": 38, + "graded": 38, + "passes": 4 + }, + "3.8": { + "n": 26, + "graded": 26, + "passes": 21 + } + }, + "p3_writing": { + "3.6": { + "n": 38, + "graded": 38, + "passes": 11 + }, + "3.8": { + "n": 26, + "graded": 21, + "passes": 8 + } + } + }, + "defect_diagnostics": { + "provenance": "computed from raw grade.json; identity joined from frozen CSV by cell id; cells with graded==0 in the frozen CSV are excluded even if a grade.json now exists on disk (post-freeze drift)", + "note": "Diagnostics only. These do NOT alter any pass rate reported elsewhere; the results tables are strictly as-graded.", + "D3_p2_triage": { + "n_cells": 64, + "all_three_spam_na": 64, + "fails": 24, + "fail_blocked_by_urgency_only": 24, + "would_flip_to_pass": { + "3.6": 21, + "3.8": 3 + }, + "verdicts": { + "PASS": 40, + "FAIL": 24 + }, + "by_model": { + "3.6:PASS": 17, + "3.6:FAIL": 21, + "3.8:FAIL": 3, + "3.8:PASS": 23 + }, + "correction_favours": "3.6" + }, + "D1_word_gate": { + "p3_business": { + "3.6": { + "n": 38, + "PASS": 35, + "FAIL": 3, + "over_limit": 3, + "fail_with_over_limit": 3 + }, + "3.8": { + "n": 23, + "FAIL": 18, + "over_limit": 18, + "fail_with_over_limit": 18, + "MISSING_OUTPUT": 1, + "PASS": 4 + } + }, + "p3_doc": { + "3.6": { + "n": 38, + "PASS": 28, + "FAIL": 10, + "over_limit": 10 + }, + "3.8": { + "n": 25, + "FAIL": 16, + "over_limit": 16, + "PASS": 9 + } + }, + "p3_doc_overshoot_wordcounts": { + "3.6": [ + 707, + 707, + 707, + 707, + 707, + 707, + 707, + 707, + 709, + 713 + ], + "3.8": [ + 701, + 702, + 708, + 718, + 719, + 720, + 722, + 722, + 722, + 723, + 725, + 726, + 727, + 730, + 733, + 772 + ] + }, + "p3_doc_limit": 700, + "p3_writing": { + "3.6": { + "n": 38, + "FAIL": 27, + "PASS": 11, + "fail_length_only": 11 + }, + "3.8": { + "n": 21, + "FAIL": 13, + "fail_length_only": 13, + "PASS": 8 + } + }, + "correction_favours": "3.8" + }, + "D2_p3_pm": { + "3.6": { + "n": 38, + "FAIL": 34, + "fail_with_risk_recall_below_min": 34, + "PASS": 4 + }, + "3.8": { + "n": 26, + "PASS": 21, + "FAIL": 5, + "fail_with_risk_recall_below_min": 5 + }, + "risk_recall_distribution": { + "3.6": { + "2": 34, + "3": 4 + }, + "3.8": { + "3": 16, + "4": 5, + "2": 5 + } + }, + "min_risks_threshold": 3, + "correction_favours": "3.6" + } + } +} \ No newline at end of file diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/apply_grade_corrections.py b/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/apply_grade_corrections.py new file mode 100755 index 00000000..a5ac3f79 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/apply_grade_corrections.py @@ -0,0 +1,1261 @@ +#!/usr/bin/env python3 +"""Non-destructive grader-correction overlay for the Qwen3.6 vs Qwen3.8 MMBT PR. + +WHAT THIS IS +============ +Three grader defects were identified in the MMBT harness. Repo convention is that +`grade.json`, the task briefs under `tooling/tasks/`, and the ground-truth files under +`tooling/graders/ground_truth/` are IMMUTABLE run evidence: they are never edited, so +that grades stay longitudinally comparable with every earlier campaign. Corrections are +therefore published as a separate overlay. This script produces that overlay. + +For every cell it corrects, it emits a `grade.corrected.json` carrying the original +verdict, the correction outcome, which defect(s) fired, and the measured evidence that +justifies the change. It never mutates a single byte of run evidence. + +The outcome takes two deliberately distinct shapes. D2/D3 corrections ARE verdict +corrections and carry `corrected_verdict`. D1 corrections are GATE INVALIDATIONS and +carry `gate_invalidated: true` plus `verdict_without_length_gate` -- and, unless a D2/D3 +verdict correction also fired on the same cell, NO `corrected_verdict` at all, so a +downstream aggregator that reads the field (not the prose) cannot mistake a length-gate +invalidation for a verified PASS. + +THE THREE DEFECTS +================= +D1 -- WORD-GATE TOKENIZER MISMATCH + The four phase-3 length-gated graders measure length with + `len(re.findall(r"\\b\\w+\\b", text))`. The models budget with shell `wc -w`. The two + counters disagree because `wc -w` counts whitespace-delimited runs (so markdown + punctuation such as `|`, `---`, `**` and `#` become "words") while the regex counts + alphanumeric runs (so a hyphenated token like `mid-July` counts twice). + NEITHER COUNTER IS GROUND TRUTH. The honest framing is that two reasonable counters + disagree by more than the margin at a hard threshold, and the model was optimising + against the one it could actually observe. This overlay reports both numbers side by + side and, where a FAIL is attributable to the length gate ALONE and every deliverable + is within its ceiling under `wc -w`, records `gate_invalidated: true` together with + `verdict_without_length_gate` (computed from the non-length gates). Per the + grader-defects doc these cells are to be read as "not a valid FAIL", NOT as "a + verified PASS", which is why a D1-only cell carries no `corrected_verdict`. + +D2 -- p3_pm KEYWORD LITERALISM + `phase3_project_mgmt_grade.py` RISK_KEYWORDS R3 requires one of the literal strings + "legal unresponsive" / "legal silent" / "legal hasn't" / "blocking on legal". + Qwen3.6 characteristically writes the uncontracted "legal has not responded", which is + semantically identical and literally unmatched. + The repo ALREADY CONTAINS the fix, unit-tested, in + `tooling/correct_gemma4_project_mgmt_grades.py` -- but its cell enumeration was + hardcoded to `p3_pm_gemma4-31b-q4_v{n}`. We REUSE that module rather than + reimplementing it (see `load_upstream_pm_corrector` and `correct_project_mgmt`), and + generalise it by driving cell selection from the frozen dataset instead. + MECHANISM, PRECISELY: upstream R3 is an ADJACENCY REGEX -- `legal` within 200 + characters of a completion term (has not / not yet responded / sign-off / approval / + unresponsive / silent / delay / contract) -- which matches strictly more than + contractions. So the overlay records, for every corrected cell, the exact span of + report text the regex matched (`matched_span`), making the applied mechanism + self-documenting rather than relying on the "contraction literalism" shorthand. + +D3 -- p2_triage BRIEF / GROUND-TRUTH CONTRADICTION + `tooling/tasks/task_triage.md` defines an urgency value for noise/spam. The ground + truth `tooling/graders/ground_truth/phase2_triage.json` labels the spam tickets with a + different value, and the grader does exact string match. Every graded triage cell of + both models followed the brief and was penalised for it. The allowed vocabulary and + the noise category are PARSED OUT OF THE BRIEF (see `parse_triage_brief`) rather than + hardcoded, so the correction is traceable to the document that authorised it. + +ALL THREE CORRECTIONS ARE STRICTLY LENIENCY. The script verifies empirically -- over +every graded cell, not just the corrected ones -- that no cell flips PASS -> FAIL. + +SAFETY +====== +`guarded_write` refuses, with a hard error, to write to `grade.json`, to anything under +`tooling/tasks/`, or to anything under a `ground_truth/` directory. The default output +root is a MIRROR tree under the PR staging directory: + + //logs//grade.corrected.json + +which is byte-for-byte the sibling layout the overlay takes inside the repo, so it can be +dropped in at merge time without this script ever touching a checkout. `--in-place` +writes true siblings into the checkouts instead; it is off by default and still cannot +touch protected paths. + +IDEMPOTENCE +=========== +Every per-cell payload is deterministic: sorted keys, no wall-clock, no absolute paths +that vary by run. Re-running produces byte-identical per-cell files. The manifest +carries `overlay_digest`, a sha256 over all per-cell payloads, which is the value to +compare across runs; `generated_at` in the manifest is the only volatile field and is +excluded from the digest. + +USAGE +===== + apply_grade_corrections.py --dry-run # print summary table, write nothing + apply_grade_corrections.py # write the overlay mirror + apply_grade_corrections.py --json-summary out.json +""" +from __future__ import annotations + +import argparse +import csv +import hashlib +import importlib.util +import json +import re +import subprocess +import sys +import tarfile +from contextlib import contextmanager +from datetime import datetime, timezone +from pathlib import Path + +SCHEMA_VERSION = "mmbt-grade-correction-overlay/2" + +DEFAULT_CHECKOUT_ROOT = Path("/home/michael") +DEFAULT_DATASET = Path("/home/michael/mmbt-frozen-dataset-v2.csv") +DEFAULT_OUTPUT_ROOT = Path("/home/michael/pr-staging/overlay") + +# Freeze #2 stamp (see /home/michael/FREEZE2_STAMP.txt). Freeze #1 +# (mmbt-frozen-dataset.csv, 746 cells, 2026-08-16T11:49:14Z) is superseded. +FREEZE_STAMP = "2026-08-16T14:23:09Z" + +# The frozen dataset is the single source of truth for cell identity, model and sampler. +# Identity is NEVER recomputed from directory names. +FROZEN_DATASET_NOTE = ( + "cell identity, model and sampler are taken verbatim from the frozen dataset; " + "never recomputed from directory names" +) + + +# --------------------------------------------------------------------------- +# Write guard +# --------------------------------------------------------------------------- + +class ProtectedPathError(RuntimeError): + """Raised when something tries to write to immutable run evidence.""" + + +PROTECTED_BASENAMES = {"grade.json", "receipt.json", "transcript.jsonl", "summary.json"} +PROTECTED_DIR_NAMES = {"ground_truth", "tasks"} + + +def assert_writable(path: Path) -> None: + """Hard-refuse any write to immutable run evidence. + + Refuses grade.json (and its sibling run evidence), anything under a `tasks/` + directory (the task briefs), and anything under a `ground_truth/` directory. + """ + resolved = Path(path).resolve() + if resolved.name in PROTECTED_BASENAMES: + raise ProtectedPathError( + f"REFUSING to write {resolved}: {resolved.name} is immutable run evidence. " + "Corrections are published as a separate overlay, never by editing grades." + ) + for part in resolved.parts: + if part in PROTECTED_DIR_NAMES: + raise ProtectedPathError( + f"REFUSING to write {resolved}: it lives under a protected '{part}/' " + "directory (task briefs and ground truth are immutable)." + ) + + +def guarded_write(path: Path, text: str) -> None: + assert_writable(path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +# --------------------------------------------------------------------------- +# Small helpers +# --------------------------------------------------------------------------- + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def canonical_json(payload: dict) -> str: + return json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=False) + "\n" + + +def read_tar_member(archive: Path, relname: str) -> bytes | None: + """Return the exact bytes of a top-level workspace deliverable, or None. + + Requires exactly one match so an ambiguous archive is never silently mis-measured. + """ + with tarfile.open(archive, "r:gz") as bundle: + matches = [ + m for m in bundle.getmembers() + if m.isfile() and m.name.removeprefix("./") == relname + ] + if len(matches) != 1: + return None + handle = bundle.extractfile(matches[0]) + if handle is None: + return None + return handle.read() + + +# --------------------------------------------------------------------------- +# D1: word counting +# --------------------------------------------------------------------------- + +# The grader's counter, verbatim from the four phase-3 length-gated graders: +# phase3_business_memo_grade.py:73, phase3_doc_synthesis_grade.py:48, +# phase3_project_mgmt_grade.py:64, phase3_writing_editing_grade.py:18 +GRADER_WORD_RE = re.compile(r"\b\w+\b") + +# GNU wc -w counts maximal runs of non-whitespace characters. Whitespace is +# space, tab, newline, vertical tab, form feed, carriage return. +_WC_WHITESPACE = b" \t\n\v\f\r" + +# LOCALE MATTERS, AND IT IS NOT A DETAIL. +# GNU wc's word counter skips characters that are not printable in the active locale. +# Under LC_ALL=C every byte >= 0x80 is non-printable, so a standalone em dash (U+2014, +# three high bytes between two spaces) forms NO word at all -- these deliverables are +# full of em dashes, and the C locale undercounts one of them by 17 words out of 700. +# The harness container is the authority on which count the model actually saw: +# docker image inspect bench-sandbox:latest --format "{{json .Config.Env}}" +# -> [..., "LANG=C.UTF-8", ...] +# and `wc -w` run inside that image reproduces the C.UTF-8 number, not the C number. +# So we pin C.UTF-8. Pinning C here would silently inflate the correction count. +_HARNESS_LOCALE = "C.UTF-8" +_HARNESS_LOCALE_PROVENANCE = ( + "bench-sandbox:latest Config.Env LANG=C.UTF-8; verified by running wc -w inside " + "that image against an extracted deliverable" +) + + +def grader_word_count(text: str) -> int: + return len(GRADER_WORD_RE.findall(text)) + + +def wc_w_pure_python(data: bytes) -> int: + """Byte-level reimplementation of the GNU `wc -w` word-counting loop.""" + count = 0 + in_word = False + for byte in data: + if byte in _WC_WHITESPACE: + in_word = False + elif not in_word: + in_word = True + count += 1 + return count + + +def wc_w_subprocess(data: bytes) -> int | None: + """Run the real `wc -w` binary on the exact deliverable bytes, in the harness locale.""" + try: + proc = subprocess.run( + ["wc", "-w"], input=data, capture_output=True, check=True, + env={"LC_ALL": _HARNESS_LOCALE, "LANG": _HARNESS_LOCALE, + "PATH": "/usr/bin:/bin"}, + ) + except (OSError, subprocess.CalledProcessError): + return None + try: + return int(proc.stdout.split()[0]) + except (IndexError, ValueError): + return None + + +def measure_deliverable(data: bytes) -> dict: + """Measure one deliverable with both counters. No guessed percentages anywhere.""" + text = data.decode("utf-8", errors="replace") + shell = wc_w_subprocess(data) + pure = wc_w_pure_python(data) + return { + "bytes": len(data), + "sha256": sha256_bytes(data), + "grader_regex_word_count": grader_word_count(text), + "wc_w_count": shell if shell is not None else pure, + "wc_w_source": ( + f"shell wc -w under LC_ALL={_HARNESS_LOCALE}" if shell is not None + else "pure-python wc -w equivalent" + ), + "wc_w_locale": _HARNESS_LOCALE, + "wc_w_locale_provenance": _HARNESS_LOCALE_PROVENANCE, + "wc_w_shell": shell, + "wc_w_pure_python": pure, + "wc_w_counters_agree": (shell is None or shell == pure), + } + + +def transcript_wc_w_calls(cell_dir: Path) -> int | None: + """Count literal `wc -w` invocations in the transcript. + + This is the evidence that `wc -w` is the counter the model was actually budgeting + against -- it is not used to decide any correction, only to document why the + disagreement matters. + """ + path = cell_dir / "transcript.jsonl" + if not path.is_file(): + return None + total = 0 + with path.open("rb") as handle: + carry = b"" + for chunk in iter(lambda: handle.read(1 << 20), b""): + buf = carry + chunk + total += buf.count(b"wc -w") + carry = buf[-8:] + return total + + +# --------------------------------------------------------------------------- +# D1: per-grader length-gate models +# +# Each entry describes, for one length-gated grader: the deliverables it word-counts, +# and whether every NON-length gate passed. A FAIL is "length-attributable" only when +# every other gate passed and the sole failing condition was the word ceiling. +# --------------------------------------------------------------------------- + +def _pm_recall(value: str | None) -> int: + # project_mgmt stores recalls as "n/6" strings. + if not isinstance(value, str) or "/" not in value: + return -1 + try: + return int(value.split("/")[0]) + except ValueError: + return -1 + + +def length_profile_business_memo(grade: dict) -> dict | None: + scores = grade.get("scores") or {} + thresholds = grade.get("thresholds") or {} + ceiling = thresholds.get("max_word_count") + if ceiling is None or "memo_word_count" not in scores: + return None + # Gate, verbatim: bias_recall >= min_bias_recall AND stance_pushback AND wc <= ceiling + others_pass = ( + (scores.get("bias_recall") or 0) >= thresholds.get("min_bias_recall", 0.5) + and bool(scores.get("stance_pushback")) + ) + return { + "deliverables": [{"relname": "memo.md", "ceiling": ceiling, + "stored_grader_count": scores.get("memo_word_count")}], + "non_length_gates_pass": others_pass, + "gate_source": "phase3_business_memo_grade.py:115", + } + + +def length_profile_doc_synthesis(grade: dict) -> dict | None: + scores = grade.get("scores") or {} + thresholds = grade.get("thresholds") or {} + ceiling = thresholds.get("max_word_count") + if ceiling is None or "word_count" not in scores: + return None + others_pass = ( + (scores.get("facts_captured") or 0) >= thresholds.get("min_facts_captured", 6) + ) + return { + "deliverables": [{"relname": "brief.md", "ceiling": ceiling, + "stored_grader_count": scores.get("word_count")}], + "non_length_gates_pass": others_pass, + "gate_source": "phase3_doc_synthesis_grade.py:70", + } + + +def length_profile_project_mgmt(grade: dict) -> dict | None: + scores = grade.get("scores") or {} + thresholds = grade.get("thresholds") or {} + ceiling = thresholds.get("max_word_count") + if ceiling is None or "word_count" not in scores: + return None + others_pass = ( + _pm_recall(scores.get("workstream_recall")) >= thresholds.get("min_workstreams", 4) + and _pm_recall(scores.get("risk_recall")) >= thresholds.get("min_risks", 3) + and _pm_recall(scores.get("decision_recall")) >= thresholds.get("min_decisions", 3) + and _pm_recall(scores.get("milestone_recall")) >= thresholds.get("min_milestones", 3) + and len(scores.get("sections_present") or []) >= 3 + ) + return { + "deliverables": [{"relname": "status_report.md", "ceiling": ceiling, + "stored_grader_count": scores.get("word_count")}], + "non_length_gates_pass": others_pass, + "gate_source": "phase3_project_mgmt_grade.py:95-102", + } + + +def length_profile_writing_editing(grade: dict) -> dict | None: + per_audience = grade.get("per_audience") or {} + if not per_audience: + return None + deliverables = [] + others_pass = True + for audience, result in sorted(per_audience.items()): + if result.get("verdict") == "MISSING" or "word_count" not in result: + return None # a missing deliverable is not a length problem + if not (result.get("required_content_pass") and result.get("prohibited_content_pass")): + others_pass = False + deliverables.append({ + "relname": result.get("filename"), + "audience": audience, + "ceiling": result.get("max_words"), + "stored_grader_count": result.get("word_count"), + }) + return { + "deliverables": deliverables, + "non_length_gates_pass": others_pass, + "gate_source": "phase3_writing_editing_grade.py:110 (all audiences must PASS)", + } + + +LENGTH_GATED_GRADERS = { + "business_memo": length_profile_business_memo, + "doc_synthesis": length_profile_doc_synthesis, + "project_mgmt": length_profile_project_mgmt, + "writing_editing": length_profile_writing_editing, +} + + +def correct_word_gate(cell_dir: Path, grade: dict) -> dict | None: + """Apply D1. Returns None when the cell is not a length-gate candidate.""" + task = grade.get("task") + profile_fn = LENGTH_GATED_GRADERS.get(task) + if profile_fn is None: + return None + profile = profile_fn(grade) + if profile is None: + return None + + archive = cell_dir / "workspace_final.tar.gz" + if not archive.is_file(): + return None + + measurements = [] + all_within_wc_w = True + any_over_grader = False + extraction_verified = True + for spec in profile["deliverables"]: + relname = spec.get("relname") + ceiling = spec.get("ceiling") + if not relname or ceiling is None: + extraction_verified = False + break + data = read_tar_member(archive, relname) + if data is None: + extraction_verified = False + break + m = measure_deliverable(data) + m["relname"] = relname + if "audience" in spec: + m["audience"] = spec["audience"] + m["ceiling"] = ceiling + m["stored_grader_count"] = spec.get("stored_grader_count") + # INTEGRITY CHECK: re-running the grader's own counter on the bytes we extracted + # must reproduce the number the grader recorded. If it does not, we are not + # looking at the artifact the grader looked at, and we refuse to correct. + m["grader_recount_matches_stored"] = ( + m["grader_regex_word_count"] == spec.get("stored_grader_count") + ) + if not m["grader_recount_matches_stored"]: + extraction_verified = False + m["over_ceiling_by_grader_regex"] = m["grader_regex_word_count"] > ceiling + m["within_ceiling_by_wc_w"] = m["wc_w_count"] <= ceiling + m["counter_delta"] = m["grader_regex_word_count"] - m["wc_w_count"] + if m["over_ceiling_by_grader_regex"]: + any_over_grader = True + if not m["within_ceiling_by_wc_w"]: + all_within_wc_w = False + measurements.append(m) + + if not measurements: + return None + + length_attributable_fail = bool( + grade.get("verdict") == "FAIL" + and profile["non_length_gates_pass"] + and any_over_grader + ) + applies = bool( + length_attributable_fail and extraction_verified and all_within_wc_w + ) + return { + "defect": "D1", + "applies": applies, + "title": "word-gate tokenizer mismatch", + "gate_source": profile["gate_source"], + "non_length_gates_pass": profile["non_length_gates_pass"], + "length_attributable_fail": length_attributable_fail, + "extraction_verified": extraction_verified, + "any_deliverable_over_ceiling_by_grader_regex": any_over_grader, + "all_deliverables_within_ceiling_by_wc_w": all_within_wc_w, + "measurements": measurements, + "model_wc_w_calls_in_transcript": transcript_wc_w_calls(cell_dir), + "framing": ( + "neither counter is ground truth: wc -w counts markdown punctuation as words, " + "the grader regex splits hyphenated tokens. Two reasonable counters disagree " + "by more than the margin at a hard threshold." + ), + } + + +# --------------------------------------------------------------------------- +# D2: reuse of the repo's existing, unit-tested project-management corrector +# --------------------------------------------------------------------------- + +_UPSTREAM_CACHE: dict[str, object] = {} + + +def load_upstream_pm_corrector(repo_root: Path): + """Import the repo's own `tooling/correct_gemma4_project_mgmt_grades.py`. + + We deliberately do NOT reimplement its rules. We import the module and call its + `apply_correction` so the overlay uses the same regexes and the same verdict + recomputation that the repo already unit-tests in + `tooling/test_correct_gemma4_project_mgmt_grades.py`. + + The ONLY thing we change is cell selection: upstream `build()` hardcodes the names + `p3_pm_gemma4-31b-q4_v{n}`, which is why it never ran against the Qwen cells. We + bypass `build()` entirely and drive selection from the frozen dataset instead. + """ + path = repo_root / "tooling" / "correct_gemma4_project_mgmt_grades.py" + if not path.is_file(): + raise FileNotFoundError(f"upstream corrector missing: {path}") + digest = sha256_file(path) + if digest in _UPSTREAM_CACHE: + return _UPSTREAM_CACHE[digest], digest, path + spec = importlib.util.spec_from_file_location(f"upstream_pm_{digest[:12]}", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + _UPSTREAM_CACHE[digest] = module + return module, digest, path + + +@contextmanager +def upstream_rules_scoped(module, rule_ids: set[str] | None): + """Narrow upstream's RULES table to the rules this PR actually verified. + + Upstream ships four semantic-equivalence rules (R2, R3, D3_mobile, D4_option_b). + This PR verified only R3 firsthand -- the "legal has not responded" vs "legal hasn't" + natural experiment. Applying rules we did not verify would be a broader correction + than the PR claims, so by default we scope upstream to R3. `--pm-rules all` restores + the full upstream table for reviewers who want to see the wider effect. + """ + if rule_ids is None: + yield sorted(module.RULES) + return + original = module.RULES + module.RULES = {k: v for k, v in original.items() if k in rule_ids} + try: + yield sorted(module.RULES) + finally: + module.RULES = original + + +def correct_project_mgmt(cell_dir: Path, repo_root: Path, grade: dict, + rule_ids: set[str] | None) -> dict | None: + """Apply D2 by delegating to the repo's own corrector.""" + if grade.get("task") != "project_mgmt": + return None + archive = cell_dir / "workspace_final.tar.gz" + if not archive.is_file(): + return None + + module, module_sha256, module_path = load_upstream_pm_corrector(repo_root) + try: + report_text, report_sha256 = module.read_archived_report(archive) + except (OSError, tarfile.TarError, UnicodeDecodeError, ValueError): + return None + + with upstream_rules_scoped(module, rule_ids) as active_rules: + corrected_grade, changes = module.apply_correction(grade, report_text) + + # Evidence: for each rule that fired, show the phrase the model actually wrote next to + # the literal keyword list it failed to match, and record the EXACT span the upstream + # adjacency regex matched -- the rule matches more than contractions, so the matched + # span (not the narrative) is the record of the mechanism actually applied. + literal_misses = {} + if grade.get("task") == "project_mgmt": + for change in changes: + item = change["item"] + category = change["category"] + span = _pattern_match_span(report_text, change["pattern"]) + change["matched_span"] = span["matched_span"] if span else None + literal_misses[item] = { + "literal_keywords_required_by_grader": + _grader_keyword_list(repo_root, category, item), + "matched_span_of_upstream_pattern": span, + "matched_phrase_in_report": _first_match_excerpt( + report_text, change["pattern"] + ), + "semantic_rule": change["reason"], + "upstream_pattern": change["pattern"], + } + + return { + "defect": "D2", + "applies": bool(changes), + "title": "p3_pm keyword literalism", + "mechanism_note": ( + "the rule actually applied is upstream R3, an adjacency regex (the word " + "legal within 200 characters of a completion term such as has not / " + "sign-off / approval / unresponsive / silent / delay / contract), which " + "matches strictly more than contractions; matched_span in each change " + "records the exact text that satisfied it" + ), + "upstream_module": { + "path_in_repo": "tooling/correct_gemma4_project_mgmt_grades.py", + "sha256": module_sha256, + "reused": "read_archived_report + apply_correction (not reimplemented)", + "generalised_by": ( + "cell selection driven from the frozen dataset instead of upstream " + "build()'s hardcoded p3_pm_gemma4-31b-q4_v{n} names" + ), + "active_rules": active_rules, + }, + "status_report_source": "workspace_final.tar.gz:./status_report.md", + "status_report_sha256": report_sha256, + "changes": changes, + "evidence": literal_misses, + "upstream_corrected_verdict": corrected_grade.get("verdict"), + "corrected_scores": corrected_grade.get("scores"), + "corrected_details": corrected_grade.get("details"), + } + + +_GRADER_KEYWORD_CACHE: dict[tuple[str, str, str], list] = {} + + +def _grader_keyword_list(repo_root: Path, category: str, item: str) -> list: + """Read the literal keyword list the grader required, straight from the grader.""" + key = (str(repo_root), category, item) + if key in _GRADER_KEYWORD_CACHE: + return _GRADER_KEYWORD_CACHE[key] + grader = repo_root / "tooling" / "graders" / "phase3_project_mgmt_grade.py" + table = {"risks": "RISK_KEYWORDS", "decisions": "DECISION_KEYWORDS", + "workstreams": "WORKSTREAM_KEYWORDS", "milestones": "MILESTONE_KEYWORDS"}.get(category) + result: list = [] + if grader.is_file() and table: + src = grader.read_text(encoding="utf-8") + block = re.search(rf"{table}\s*=\s*\{{(.*?)\n\}}", src, re.S) + if block: + row = re.search(rf'"{re.escape(item)}"\s*:\s*\[(.*?)\]', block.group(1), re.S) + if row: + result = re.findall(r'"([^"]*)"', row.group(1)) + _GRADER_KEYWORD_CACHE[key] = result + return result + + +def _first_match_excerpt(text: str, pattern: str, window: int = 120) -> str | None: + normalized = re.sub(r"\s+", " ", text.lower()) + hit = re.search(pattern, normalized) + if not hit: + return None + start = max(0, hit.start() - 10) + return normalized[start:hit.end() + window].strip()[:260] + + +def _pattern_match_span(text: str, pattern: str) -> dict | None: + """Return the exact span the upstream rule matched, under upstream's normalisation. + + `apply_correction` searches a lowercased, whitespace-collapsed copy of the report, so + the span is reported in that same normalisation, with its offsets, and is the + verbatim text that satisfied the adjacency regex -- no paraphrase, no excerpt window. + """ + normalized = re.sub(r"\s+", " ", text.lower()) + hit = re.search(pattern, normalized) + if not hit: + return None + return { + "matched_span": hit.group(0), + "start_in_normalized_text": hit.start(), + "end_in_normalized_text": hit.end(), + "normalization": ( + "lowercased, whitespace collapsed to single spaces " + "(identical to upstream apply_correction)" + ), + } + + +# --------------------------------------------------------------------------- +# D3: triage brief vs ground truth +# --------------------------------------------------------------------------- + +def parse_triage_brief(brief_path: Path) -> dict: + """Parse the allowed urgency vocabulary and the noise category OUT OF THE BRIEF. + + Nothing here is hardcoded to a particular token: we find the "Urgency (closed + vocabulary)" bullet list, and within it the value whose definition mentions + noise/spam. Likewise for the category vocabulary. The correction is therefore + traceable to the exact brief line that authorised the model's answer. + """ + lines = brief_path.read_text(encoding="utf-8").splitlines() + + def bullets_under(header_re: str) -> list[tuple[int, str, str]]: + out = [] + inside = False + for idx, line in enumerate(lines, start=1): + if re.match(header_re, line.strip(), re.I): + inside = True + continue + if inside and line.startswith("## "): + break + if inside: + m = re.match(r"^-\s+`([^`]+)`\s*[—-]+\s*(.*)$", line.strip()) + if m: + out.append((idx, m.group(1), m.group(2))) + return out + + urgency = bullets_under(r"^##\s*Urgency") + categories = bullets_under(r"^##\s*Categories") + if not urgency: + raise ValueError(f"could not parse urgency vocabulary from {brief_path}") + + noise_re = re.compile(r"noise|spam", re.I) + noise_urgency = [(n, term, desc) for n, term, desc in urgency if noise_re.search(desc)] + if len(noise_urgency) != 1: + raise ValueError( + f"expected exactly one noise/spam urgency value in {brief_path}, " + f"found {[t for _, t, _ in noise_urgency]}" + ) + line_no, term, desc = noise_urgency[0] + noise_categories = sorted({t for _, t, d in categories if noise_re.search(t) or noise_re.search(d)}) + return { + "brief_path_in_repo": "tooling/tasks/task_triage.md", + "urgency_vocabulary": [t for _, t, _ in urgency], + "category_vocabulary": [t for _, t, _ in categories], + "noise_urgency_value": term, + "noise_urgency_brief_line_number": line_no, + "noise_urgency_brief_line": f"- `{term}` — {desc}", + "noise_categories": noise_categories, + } + + +def load_triage_ground_truth(repo_root: Path) -> dict: + path = repo_root / "tooling" / "graders" / "ground_truth" / "phase2_triage.json" + return json.loads(path.read_text(encoding="utf-8")) + + +def correct_triage(repo_root: Path, grade: dict) -> dict | None: + """Apply D3: credit brief-compliant noise-urgency answers the ground truth contradicts.""" + if grade.get("task") != "triage": + return None + scores = grade.get("scores") or {} + errors = grade.get("errors") or {} + if "urgency_accuracy" not in scores: + return None + + brief = parse_triage_brief(repo_root / "tooling" / "tasks" / "task_triage.md") + gt = load_triage_ground_truth(repo_root) + gt_tickets = gt.get("tickets", {}) + n = len(gt_tickets) + + urgency_errors = errors.get("urgency_errors") or [] + category_errors = errors.get("category_errors") or [] + missing = errors.get("missing_tickets") or [] + + # Recompute the grader's own arithmetic so the correction is not built on a rounded + # number, then cross-check against what the grader stored. + correct_urgency = n - len(urgency_errors) - len(missing) + correct_category = n - len(category_errors) - len(missing) + recomputed_urg = round(correct_urgency / n, 3) if n else None + recomputed_cat = round(correct_category / n, 3) if n else None + arithmetic_ok = ( + recomputed_urg == scores.get("urgency_accuracy") + and recomputed_cat == scores.get("category_accuracy") + ) + + noise_value = brief["noise_urgency_value"] + credited = [] + for err in urgency_errors: + tid = err.get("id") + gt_entry = gt_tickets.get(tid) or {} + if ( + err.get("predicted") == noise_value + and err.get("actual") != noise_value + and gt_entry.get("category") in brief["noise_categories"] + ): + credited.append({ + "ticket_id": tid, + "model_answer": err.get("predicted"), + "ground_truth_label": err.get("actual"), + "ground_truth_category": gt_entry.get("category"), + "authorised_by_brief_line": brief["noise_urgency_brief_line_number"], + }) + + if not credited or not arithmetic_ok: + return { + "defect": "D3", + "applies": False, + "title": "p2_triage brief / ground-truth contradiction", + "grader_arithmetic_reproduced": arithmetic_ok, + "credited_tickets": credited, + "brief": brief, + } + + corrected_correct = correct_urgency + len(credited) + corrected_urg = round(corrected_correct / n, 3) + thresholds = grade.get("thresholds") or {} + dup_recall = (scores.get("duplicate_recall") or 0) + corrected_verdict = "PASS" if ( + (recomputed_cat or 0) >= thresholds.get("category_accuracy", 0.80) + and corrected_urg >= thresholds.get("urgency_accuracy", 0.70) + and dup_recall >= thresholds.get("duplicate_recall", 0.50) + and len(missing) == 0 + ) else "FAIL" + + return { + "defect": "D3", + "applies": True, + "title": "p2_triage brief / ground-truth contradiction", + "brief": brief, + "ground_truth_path_in_repo": "tooling/graders/ground_truth/phase2_triage.json", + "contradiction": ( + f"brief line {brief['noise_urgency_brief_line_number']} defines urgency " + f"'{brief['noise_urgency_value']}' for noise/spam; the ground truth labels " + f"those same tickets differently, and the grader does exact string match" + ), + "grader_arithmetic_reproduced": arithmetic_ok, + "tickets_total": n, + "credited_tickets": credited, + "credited_count": len(credited), + "urgency_accuracy_original": scores.get("urgency_accuracy"), + "urgency_accuracy_corrected": corrected_urg, + "urgency_threshold": thresholds.get("urgency_accuracy", 0.70), + "category_accuracy": recomputed_cat, + "duplicate_recall": dup_recall, + "corrected_verdict": corrected_verdict, + } + + +# --------------------------------------------------------------------------- +# Composition +# --------------------------------------------------------------------------- + +def compose_verdict(grade: dict, d1: dict | None, d2: dict | None, d3: dict | None) -> dict: + """Compose the correction outcome from whichever defects fired. + + Two outcome kinds are deliberately distinct in the schema: + + * D2/D3 are VERDICT CORRECTIONS: the grader misread a compliant answer, the corrected + gate arithmetic yields a real verdict, and the cell carries `corrected_verdict`. + * D1 is a GATE INVALIDATION: two reasonable word counters disagree at the threshold, + so the length-gate FAIL is not valid evidence of failure. That is NOT the same + claim as a verified PASS, so a D1 cell carries `gate_invalidated: true` plus + `verdict_without_length_gate` (what the non-length gates yield) and -- unless a + D2/D3 verdict correction also fired on the same cell -- no `corrected_verdict`. + + Every correction is monotone leniency: a correction can only satisfy a gate that was + previously unsatisfied. It can never unsatisfy one. + """ + outcome = { + "applied": [], + "corrected_verdict": None, + "gate_invalidated": False, + "verdict_without_length_gate": None, + } + d1_fired = bool(d1 and d1.get("applies")) + if d1_fired: + outcome["gate_invalidated"] = True + + if d3 and d3.get("applies"): + outcome["applied"].append("D3") + outcome["corrected_verdict"] = d3["corrected_verdict"] + if d1_fired: # defensive: triage is not length-gated, so this cannot co-fire + outcome["applied"].append("D1") + outcome["verdict_without_length_gate"] = d3["corrected_verdict"] + return outcome + + if grade.get("task") == "project_mgmt": + d2_fired = bool(d2 and d2.get("applies")) + if not (d2_fired or d1_fired): + return outcome + if d2_fired: + outcome["applied"].append("D2") + if d1_fired: + outcome["applied"].append("D1") + # Recompute the project_mgmt gate with upstream-corrected recalls. + source = (d2.get("corrected_scores") if d2_fired else None) or (grade.get("scores") or {}) + details = (d2.get("corrected_details") if d2_fired else None) or (grade.get("details") or {}) + thresholds = grade.get("thresholds") or {} + counts = { + cat: sum(1 for v in (details.get(cat) or {}).values() if v.get("matched")) + for cat in ("workstreams", "risks", "decisions", "milestones") + } + recalls_and_sections_ok = ( + counts["workstreams"] >= thresholds.get("min_workstreams", 4) + and counts["risks"] >= thresholds.get("min_risks", 3) + and counts["decisions"] >= thresholds.get("min_decisions", 3) + and counts["milestones"] >= thresholds.get("min_milestones", 3) + and len(source.get("sections_present") or (grade.get("scores") or {}).get("sections_present") or []) >= 3 + ) + stored_length_ok = ( + (grade.get("scores") or {}).get("word_count", 10 ** 9) + <= thresholds.get("max_word_count", 700) + ) + if d2_fired: + # The real length gate stays in force for the verdict correction; D1 never + # silently upgrades a verdict -- its invalidation is reported separately. + outcome["corrected_verdict"] = ( + "PASS" if (recalls_and_sections_ok and stored_length_ok) else "FAIL" + ) + if d1_fired: + outcome["verdict_without_length_gate"] = ( + "PASS" if recalls_and_sections_ok else "FAIL" + ) + return outcome + + if d1_fired: + outcome["applied"].append("D1") + # D1 `applies` already established every non-length gate passed. + outcome["verdict_without_length_gate"] = "PASS" + return outcome + + return outcome + + +# --------------------------------------------------------------------------- +# Driver +# --------------------------------------------------------------------------- + +def load_frozen_rows(dataset: Path) -> list[dict]: + with dataset.open(newline="", encoding="utf-8") as handle: + return list(csv.DictReader(handle)) + + +def process(args) -> dict: + checkout_root = args.checkout_root.resolve() + rows = load_frozen_rows(args.dataset) + rule_ids = None if args.pm_rules == "all" else {r.strip() for r in args.pm_rules.split(",") if r.strip()} + + results = [] + skipped = {"quarantined": 0, "no_grade_json": 0, "unreadable_grade_json": 0} + invariant_checked = 0 + invariant_violations = [] + post_freeze_divergence = [] + + for row in rows: + cell, repo = row["cell"], row["repo"] + if cell.startswith("_"): + skipped["quarantined"] += 1 + continue + repo_root = checkout_root / repo + cell_dir = repo_root / "logs" / cell + grade_path = cell_dir / "grade.json" + if not grade_path.is_file(): + skipped["no_grade_json"] += 1 + continue + try: + grade = json.loads(grade_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + skipped["unreadable_grade_json"] += 1 + continue + + frozen_graded = row.get("graded") == "1" + disk_verdict = grade.get("verdict") + if disk_verdict != row.get("verdict"): + # The corpus is still being written. Anything that changed since the freeze + # is quarantined from headline counts, never silently absorbed. + post_freeze_divergence.append({ + "repo": repo, "cell": cell, + "frozen_verdict": row.get("verdict"), "on_disk_verdict": disk_verdict, + "frozen_graded_flag": row.get("graded"), + }) + + d1 = correct_word_gate(cell_dir, grade) + try: + d2 = correct_project_mgmt(cell_dir, repo_root, grade, rule_ids) + except FileNotFoundError: + d2 = None + d3 = correct_triage(repo_root, grade) + + outcome = compose_verdict(grade, d1, d2, d3) + applied = outcome["applied"] + corrected_verdict = outcome["corrected_verdict"] + + # INVARIANT: all three corrections are strictly leniency. Checked over every + # graded cell, including cells no defect touched, and over BOTH output verdict + # fields (a gate waiver can only relax, never tighten, so it is held to the + # same bar). + if disk_verdict in ("PASS", "FAIL"): + invariant_checked += 1 + for field, value in ( + ("corrected_verdict", corrected_verdict), + ("verdict_without_length_gate", outcome["verdict_without_length_gate"]), + ): + if disk_verdict == "PASS" and value == "FAIL": + invariant_violations.append({"repo": repo, "cell": cell, + "original": disk_verdict, + "field": field, + "corrected": value, + "applied": applied}) + + if not applied: + continue + + payload = { + "schema_version": SCHEMA_VERSION, + "cell": cell, + "repo": repo, + "family": row.get("family"), + "task": grade.get("task"), + "identity": { + "model": row.get("model"), + "quant": row.get("quant"), + "mode": row.get("mode"), + "regime": row.get("regime"), + "arm": row.get("arm"), + "replicate": row.get("replicate"), + "source": FROZEN_DATASET_NOTE, + }, + "original_verdict": disk_verdict, + "defects_applied": applied, + "frozen_dataset": { + "graded_at_freeze": frozen_graded, + "verdict_at_freeze": row.get("verdict"), + "matches_on_disk": disk_verdict == row.get("verdict"), + }, + "evidence": {k: v for k, v in + (("D1", d1), ("D2", d2), ("D3", d3)) if v is not None}, + "grade_json_sha256": sha256_file(grade_path), + "policy": ( + "grade.json, task briefs and ground truth are immutable run evidence; " + "this overlay modifies no run artifact and no raw verdict" + ), + } + # `corrected_verdict` exists ONLY where a D2/D3 verdict correction fired. + if corrected_verdict is not None: + payload["corrected_verdict"] = corrected_verdict + payload["verdict_changed"] = corrected_verdict != disk_verdict + # D1 is recorded as a gate invalidation, never as a verdict. + if outcome["gate_invalidated"]: + payload["gate_invalidated"] = True + payload["verdict_without_length_gate"] = outcome["verdict_without_length_gate"] + payload["gate_invalidated_meaning"] = ( + "the original FAIL is attributable solely to the length gate, where two " + "reasonable word counters disagree across the threshold; read this cell " + "as NOT A VALID FAIL, not as a verified PASS " + "(grader-defects doc, D1 correction limits)" + ) + results.append({ + "row": row, "payload": payload, "cell_dir": cell_dir, + "applied": applied, "original": disk_verdict, "corrected": corrected_verdict, + "gate_invalidated": outcome["gate_invalidated"], + "verdict_without_length_gate": outcome["verdict_without_length_gate"], + "frozen_graded": frozen_graded, + }) + + return { + "results": results, + "skipped": skipped, + "invariant_checked": invariant_checked, + "invariant_violations": invariant_violations, + "post_freeze_divergence": post_freeze_divergence, + "rows_total": len(rows), + "rule_ids": sorted(rule_ids) if rule_ids else "all upstream rules", + } + + +def summarise(state: dict) -> dict: + """Per-defect counts. D1 counts GATE INVALIDATIONS, not verdict changes -- the two + are different claims and are kept under different keys so an aggregator cannot sum + them into one number by accident.""" + frozen = [r for r in state["results"] if r["frozen_graded"]] + by_defect: dict[str, dict[str, int]] = {} + for r in frozen: + for d in r["applied"]: + model = r["row"]["model"] + if d == "D1": + slot = by_defect.setdefault(d, {"3.6": 0, "3.8": 0, "total": 0, + "gate_invalidated_3.6": 0, + "gate_invalidated_3.8": 0, + "gate_invalidated_total": 0}) + slot[model] = slot.get(model, 0) + 1 + slot["total"] += 1 + if r["gate_invalidated"]: + slot[f"gate_invalidated_{model}"] = slot.get(f"gate_invalidated_{model}", 0) + 1 + slot["gate_invalidated_total"] += 1 + else: + slot = by_defect.setdefault(d, {"3.6": 0, "3.8": 0, "total": 0, + "verdict_changed_3.6": 0, + "verdict_changed_3.8": 0, + "verdict_changed_total": 0}) + slot[model] = slot.get(model, 0) + 1 + slot["total"] += 1 + if r["corrected"] is not None and r["corrected"] != r["original"]: + slot[f"verdict_changed_{model}"] = slot.get(f"verdict_changed_{model}", 0) + 1 + slot["verdict_changed_total"] += 1 + return { + "cells_touched_frozen": len(frozen), + "cells_touched_including_post_freeze": len(state["results"]), + "by_defect": by_defect, + } + + +def d2_mechanism_report(state: dict) -> dict: + """Manifest-level record of WHICH text satisfied the upstream adjacency regex, per + corrected cell. The rule matches more than contractions, so the matched spans -- not + the narrative shorthand -- document the mechanism actually applied.""" + spans = [] + patterns = set() + for r in state["results"]: + if "D2" not in r["applied"]: + continue + d2 = (r["payload"].get("evidence") or {}).get("D2") or {} + for change in d2.get("changes") or []: + if change.get("pattern"): + patterns.add(change["pattern"]) + spans.append({ + "repo": r["row"]["repo"], + "cell": r["row"]["cell"], + "item": change.get("item"), + "matched_span": change.get("matched_span"), + }) + spans.sort(key=lambda e: (e["repo"], e["cell"], str(e["item"]))) + return { + "note": ( + "the mechanism actually applied is upstream rule R3, an adjacency regex " + "(the word legal within 200 characters of a completion term), which matches " + "strictly more than contractions; matched_span is the exact normalised " + "report text that satisfied the regex in each corrected cell" + ), + "patterns": sorted(patterns), + "matched_spans": spans, + } + + +def print_table(state: dict, summary: dict) -> None: + print("=" * 78) + print("MMBT GRADE-CORRECTION OVERLAY -- DRY RUN (nothing written)") + print("=" * 78) + print(f"frozen dataset rows : {state['rows_total']}") + print(f"skipped, quarantined : {state['skipped']['quarantined']}") + print(f"skipped, no grade.json on disk : {state['skipped']['no_grade_json']}") + print(f"skipped, unreadable grade.json : {state['skipped']['unreadable_grade_json']}") + print(f"D2 upstream rules active : {state['rule_ids']}") + print() + print(f"cells corrected (frozen-graded) : {summary['cells_touched_frozen']}") + print(f"cells corrected (incl. post-freeze) : {summary['cells_touched_including_post_freeze']}") + print() + header = f"{'defect':<8}{'3.6':>8}{'3.8':>8}{'total':>8} {'effect (3.6/3.8/tot)':>40}" + print(header) + print("-" * len(header)) + for defect in sorted(summary["by_defect"]): + s = summary["by_defect"][defect] + if "gate_invalidated_total" in s: + effect = (f"{s['gate_invalidated_3.6']}/{s['gate_invalidated_3.8']}" + f"/{s['gate_invalidated_total']} gates invalidated") + else: + effect = (f"{s['verdict_changed_3.6']}/{s['verdict_changed_3.8']}" + f"/{s['verdict_changed_total']} verdicts changed") + print(f"{defect:<8}{s['3.6']:>8}{s['3.8']:>8}{s['total']:>8} {effect:>40}") + print() + print("--- per-family breakdown (frozen-graded cells only) ---") + print(" vchg = verdict corrections (D2/D3); ginv = length-gate invalidations (D1)") + fam: dict[tuple, dict] = {} + for r in state["results"]: + if not r["frozen_graded"]: + continue + key = (r["row"]["family"], ",".join(r["applied"])) + slot = fam.setdefault(key, {"3.6": 0, "3.8": 0, + "vchg36": 0, "vchg38": 0, "ginv36": 0, "ginv38": 0}) + suffix = "36" if r["row"]["model"] == "3.6" else "38" + slot[r["row"]["model"]] += 1 + if r["corrected"] is not None and r["corrected"] != r["original"]: + slot["vchg" + suffix] += 1 + if r["gate_invalidated"]: + slot["ginv" + suffix] += 1 + head = (f"{'family':<14}{'defects':<10}{'n3.6':>6}{'n3.8':>6}" + f"{'vchg3.6':>9}{'vchg3.8':>9}{'ginv3.6':>9}{'ginv3.8':>9}") + print(head) + print("-" * len(head)) + for key in sorted(fam): + s = fam[key] + print(f"{key[0]:<14}{key[1]:<10}{s['3.6']:>6}{s['3.8']:>6}" + f"{s['vchg36']:>9}{s['vchg38']:>9}{s['ginv36']:>9}{s['ginv38']:>9}") + print() + print("--- leniency invariant ---") + print(f"graded cells checked for PASS -> FAIL : {state['invariant_checked']}") + print(f"violations (PASS -> FAIL) : {len(state['invariant_violations'])}") + for v in state["invariant_violations"]: + print(" VIOLATION:", v) + print() + print("--- post-freeze corpus drift (excluded from headline counts) ---") + print(f"cells whose on-disk verdict differs from the frozen dataset : " + f"{len(state['post_freeze_divergence'])}") + for d in state["post_freeze_divergence"]: + print(f" {d['repo']}/{d['cell']}: frozen={d['frozen_verdict']!r} " + f"disk={d['on_disk_verdict']!r} graded_flag={d['frozen_graded_flag']}") + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--dataset", type=Path, default=DEFAULT_DATASET) + ap.add_argument("--checkout-root", type=Path, default=DEFAULT_CHECKOUT_ROOT) + ap.add_argument("--output-root", type=Path, default=DEFAULT_OUTPUT_ROOT) + ap.add_argument("--dry-run", action="store_true", + help="print the summary table and write nothing at all") + ap.add_argument("--in-place", action="store_true", + help="write true grade.corrected.json siblings into the checkouts " + "instead of the staging mirror (still cannot touch protected paths)") + ap.add_argument("--pm-rules", default="R3", + help="upstream project-management rules to apply: 'R3' (default, the " + "only rule this PR verified) or 'all'") + ap.add_argument("--json-summary", type=Path, default=None) + args = ap.parse_args() + + state = process(args) + summary = summarise(state) + + if args.dry_run: + print_table(state, summary) + else: + payloads = [] + for r in state["results"]: + text = canonical_json(r["payload"]) + payloads.append(text) + if args.in_place: + out = r["cell_dir"] / "grade.corrected.json" + else: + out = (args.output_root / r["row"]["repo"] / "logs" + / r["row"]["cell"] / "grade.corrected.json") + guarded_write(out, text) + digest = hashlib.sha256("".join(sorted(payloads)).encode("utf-8")).hexdigest() + manifest = { + "schema_version": SCHEMA_VERSION, + "generated_at": datetime.now(timezone.utc).isoformat(), + "overlay_digest": digest, + "note": "overlay_digest excludes generated_at and is stable across runs", + "frozen_dataset": str(args.dataset), + "frozen_dataset_sha256": sha256_file(args.dataset), + "frozen_dataset_stamp": FREEZE_STAMP, + "summary": summary, + "d2_mechanism": d2_mechanism_report(state), + "invariant_checked": state["invariant_checked"], + "invariant_violations": state["invariant_violations"], + "post_freeze_divergence": state["post_freeze_divergence"], + "skipped": state["skipped"], + } + guarded_write(args.output_root / "manifest.json", canonical_json(manifest)) + print(json.dumps({"written": len(payloads), "overlay_digest": digest}, indent=2)) + + if args.json_summary: + guarded_write(args.json_summary, canonical_json({ + "summary": summary, + "invariant_checked": state["invariant_checked"], + "invariant_violations": state["invariant_violations"], + "post_freeze_divergence": state["post_freeze_divergence"], + "skipped": state["skipped"], + "detail": [ + {"repo": r["row"]["repo"], "cell": r["row"]["cell"], + "family": r["row"]["family"], "model": r["row"]["model"], + "applied": r["applied"], "original": r["original"], + "corrected": r["corrected"], + "gate_invalidated": r["gate_invalidated"], + "verdict_without_length_gate": r["verdict_without_length_gate"], + "frozen_graded": r["frozen_graded"]} + for r in state["results"] + ], + })) + + return 1 if state["invariant_violations"] else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/build_results.sh b/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/build_results.sh new file mode 100755 index 00000000..e35a3524 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/build_results.sh @@ -0,0 +1,27 @@ +#!/bin/sh +# Rebuild the MMBT 3.6-vs-3.8 results tables from the frozen dataset. +# +# Order matters: mmbt_results.py writes results.json, defect_diag.py APPENDS the +# defect_diagnostics block to it, and make_md.py renders both into markdown. +# Running them out of order (or running mmbt_results.py again afterwards) drops +# the diagnostics block. +# +# No third-party dependencies; python3 stdlib only. +set -e +cd "$(dirname "$0")" + +echo "== validating estimators ==" +python3 test_stats.py + +echo "== computing tables from the frozen dataset ==" +python3 mmbt_results.py + +echo "== appending grader-defect diagnostics ==" +python3 defect_diag.py > /dev/null + +echo "== rendering markdown ==" +python3 make_md.py + +echo +echo "Outputs:" +ls -l /home/michael/pr-staging/results.json /home/michael/pr-staging/results-tables.md diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/defect_diag.py b/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/defect_diag.py new file mode 100755 index 00000000..5e8822a5 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/defect_diag.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Grader-defect diagnostics for the MMBT 3.6-vs-3.8 PR. + +PROVENANCE NOTE: unlike every other number in results.json, this block is +computed from the raw grade.json files rather than from the frozen CSV. That +is deliberate and allowed -- the frozen CSV carries only the final pass/fail +bit, and diagnosing WHY a cell failed requires the grader's own score fields. +No headline pass rate is computed here. Cell->model identity is taken from the +frozen CSV, never inferred from the directory name. +""" +import csv, glob, json, os, collections + +REPOS = ["mmbt-q36-card", "mmbt-qwen36-compare", "mmbt-q38-card", + "mmbt-qwen38-eaaa8ca", "mmbt-q38-q8"] +CSV_PATH = "/home/michael/mmbt-frozen-dataset-v2.csv" +RESULTS = "/home/michael/pr-staging/results.json" + +# cell -> identity, straight from the frozen dataset +IDENT = {} +for r in csv.DictReader(open(CSV_PATH)): + IDENT[r["cell"]] = r + + +def grades(family): + out = [] + for repo in REPOS: + for gp in glob.glob("/home/michael/%s/logs/%s_*/grade.json" % (repo, family)): + cell = os.path.basename(os.path.dirname(gp)) + if cell.startswith("_"): + continue + ident = IDENT.get(cell) + if ident is None: + continue # not in the frozen set -> not counted + if ident.get("graded") != "1": + # Ungraded AS OF THE FREEZE: any grade.json present here was + # written after the freeze (post-freeze drift, e.g. the Q8_0 + # arm). The frozen CSV is the source of truth, so these cells + # stay out of the diagnostics exactly as they stay out of the + # headline numbers. Disclosed in quant_control.post_freeze_drift. + continue + try: + out.append((cell, ident["model"], json.load(open(gp)))) + except Exception: + pass + return out + + +D = {"provenance": "computed from raw grade.json; identity joined from frozen CSV by cell id; " + "cells with graded==0 in the frozen CSV are excluded even if a grade.json " + "now exists on disk (post-freeze drift)", + "note": "Diagnostics only. These do NOT alter any pass rate reported elsewhere; the " + "results tables are strictly as-graded."} + +# ---- D3: p2_triage spam-urgency penalty ----------------------------------- +SPAM = {"004", "009", "021"} +tri = grades("p2_triage") +d3 = dict(n_cells=len(tri), all_three_spam_na=0, fails=0, + fail_blocked_by_urgency_only=0, would_flip_to_pass=collections.Counter(), + verdicts=collections.Counter(), by_model=collections.Counter()) +for cell, model, g in tri: + s = g.get("scores") or {} + if not s: + continue + ue = g.get("errors", {}).get("urgency_errors", []) + spam_na = sum(1 for e in ue + if e["id"] in SPAM and e.get("predicted") == "n/a" and e.get("actual") == "low") + if spam_na == 3: + d3["all_three_spam_na"] += 1 + d3["verdicts"][g["verdict"]] += 1 + d3["by_model"][model + ":" + g["verdict"]] += 1 + if g["verdict"] == "FAIL": + d3["fails"] += 1 + blocked_urg = s["urgency_accuracy"] < 0.70 + blocked_other = s["category_accuracy"] < 0.80 or s["duplicate_recall"] < 0.50 + if blocked_urg and not blocked_other: + d3["fail_blocked_by_urgency_only"] += 1 + # restore the 3 spam tickets (30 tickets total in the ground truth) + adj = s["urgency_accuracy"] + spam_na / 30.0 + if adj >= 0.70 and not blocked_other: + d3["would_flip_to_pass"][model] += 1 +d3["would_flip_to_pass"] = dict(d3["would_flip_to_pass"]) +d3["verdicts"] = dict(d3["verdicts"]) +d3["by_model"] = dict(d3["by_model"]) +d3["correction_favours"] = "3.6" +D["D3_p2_triage"] = d3 + +# ---- D1: phase-3 word gate ------------------------------------------------ +d1 = {} +# p3_business: single memo, explicit within-limit flag +bus = collections.defaultdict(lambda: collections.Counter()) +for cell, model, g in grades("p3_business"): + s = g.get("scores") or {} + bus[model]["n"] += 1 + bus[model][g["verdict"]] += 1 + if s.get("memo_within_word_limit") is False: + bus[model]["over_limit"] += 1 + if g["verdict"] == "FAIL": + bus[model]["fail_with_over_limit"] += 1 + if g["verdict"] == "FAIL" and s.get("stance_pushback") is False: + bus[model]["fail_with_stance_miss"] += 1 +d1["p3_business"] = {m: dict(v) for m, v in bus.items()} + +doc = collections.defaultdict(lambda: collections.Counter()) +overs = collections.defaultdict(list) +for cell, model, g in grades("p3_doc"): + s = g.get("scores") or {} + doc[model]["n"] += 1 + doc[model][g["verdict"]] += 1 + if s.get("within_word_limit") is False: + doc[model]["over_limit"] += 1 + overs[model].append(s.get("word_count")) +d1["p3_doc"] = {m: dict(v) for m, v in doc.items()} +d1["p3_doc_overshoot_wordcounts"] = {m: sorted(v) for m, v in overs.items()} +d1["p3_doc_limit"] = 700 + +wri = collections.defaultdict(lambda: collections.Counter()) +for cell, model, g in grades("p3_writing"): + wri[model]["n"] += 1 + wri[model][g["verdict"]] += 1 + pa = g.get("per_audience") or {} + only_len = None + for _, sub in pa.items(): + if sub.get("verdict") == "FAIL": + if sub.get("within_word_limit") is False and \ + sub.get("required_content_pass") and sub.get("prohibited_content_pass"): + only_len = True if only_len is None else only_len + else: + only_len = False + if g.get("verdict") == "FAIL" and only_len: + wri[model]["fail_length_only"] += 1 +d1["p3_writing"] = {m: dict(v) for m, v in wri.items()} +d1["correction_favours"] = "3.8" +D["D1_word_gate"] = d1 + +# ---- D2: p3_pm risk-keyword literalism ------------------------------------ +pm = collections.defaultdict(lambda: collections.Counter()) +risk_short = collections.defaultdict(list) +for cell, model, g in grades("p3_pm"): + s = g.get("scores") or {} + pm[model]["n"] += 1 + pm[model][g["verdict"]] += 1 + rr = s.get("risk_recall") + if isinstance(rr, str) and "/" in rr: + got, tot = rr.split("/") + got, tot = int(got), int(tot) + risk_short[model].append(got) + if g["verdict"] == "FAIL" and got < 3: + pm[model]["fail_with_risk_recall_below_min"] += 1 + if s.get("word_count") is not None and s["word_count"] > 700: + pm[model]["over_word_limit"] += 1 +d2 = {m: dict(v) for m, v in pm.items()} +d2["risk_recall_distribution"] = {m: dict(collections.Counter(v)) for m, v in risk_short.items()} +d2["min_risks_threshold"] = 3 +d2["correction_favours"] = "3.6" +D["D2_p3_pm"] = d2 + +R = json.load(open(RESULTS)) +R["defect_diagnostics"] = D +json.dump(R, open(RESULTS, "w"), indent=2) +print(json.dumps(D, indent=2)) diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/freeze_dataset.py b/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/freeze_dataset.py new file mode 100755 index 00000000..5e9ce6bb --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/freeze_dataset.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Freeze the Qwen3.6-vs-Qwen3.8 corpus into ONE canonical per-cell dataset. + +Every number in the PR must derive from this file and nothing else. Three of four all-cells +figures quoted during the investigation drifted 4-9pp because they were computed against a live +corpus at different moments; a frozen dataset makes that class of error impossible. + +The repo gitignores /logs/ ("raw per-run logs and agent workspaces are not published") and the +workspace tarballs alone are 1.5 GB, so the publishable artifact is this derived table. + +MODEL AND SAMPLER IDENTITY COME FROM receipt.json, NEVER FROM THE DIRECTORY NAME. +An arm named "...-offspec" was found to actually carry the vendor-card sampler, so filename +inference is unsafe. "nothink" is resolved from enable_thinking, not by substring matching +(the string "nothink" contains "think"). + +Columns: + cell, repo, family, arm, replicate, model, quant, mode, effort, regime, + temperature, top_p, top_k, presence_penalty, seed, + verdict, graded, terminal, looped_freq30, looped_run30, max_freq, max_run, + n_tools, distinct_ratio, completion_tokens, elapsed_s, label_primary, + container_death_signature, transcript_mtime +""" +import csv +import collections +import glob +import json +import os +import re +import sys + +OUT = sys.argv[1] if len(sys.argv) > 1 else "/home/michael/mmbt-frozen-dataset.csv" +REPOS = [ + "/home/michael/mmbt-q38-card", + "/home/michael/mmbt-qwen38-eaaa8ca", + "/home/michael/mmbt-q36-card", + "/home/michael/mmbt-qwen36-compare", + "/home/michael/mmbt-q38-q8", +] +PASSING = ("PASS", "STRUCTURAL_PASS") +NAME = re.compile(r"^(p[123]_[a-z]+)_(.+)_v(\d+)$") + +rows = [] +skipped = collections.Counter() + +for repo in REPOS: + for d in sorted(glob.glob(os.path.join(repo, "logs", "*"))): + base = os.path.basename(d) + if not os.path.isdir(d) or base.startswith("_"): + skipped["quarantined_or_notdir"] += 1 + continue + m = NAME.match(base) + if not m: + skipped["unparseable_name"] += 1 + continue + tpath = os.path.join(d, "transcript.jsonl") + rpath = os.path.join(d, "receipt.json") + if not os.path.exists(tpath): + skipped["no_transcript"] += 1 + continue + + # --- identity from the receipt, never the filename --- + model = quant = "" + temp = top_p = top_k = pp = seed = "" + thinking = None + effort = "" + if os.path.exists(rpath): + try: + rec = json.load(open(rpath)) + except Exception: + rec = {} + served = ((rec.get("vllm") or {}).get("served_model_name") + or (rec.get("serving") or {}).get("served_model_name") or "") + model = served + dflt = rec.get("inference_request_defaults") or {} + temp = dflt.get("temperature", "") + top_p = dflt.get("top_p", "") + top_k = dflt.get("top_k", "") + pp = dflt.get("presence_penalty", "") + seed = dflt.get("seed", "") + thinking = dflt.get("enable_thinking", None) + effort = dflt.get("reasoning_effort") or "" + if not model: + skipped["no_model_in_receipt"] += 1 + + low = model.lower() + family_model = "3.8" if "3.8" in low or "3-8" in low else ("3.6" if "3.6" in low or "3-6" in low else "?") + if "q8_0" in low or "q8-0" in low: + quant = "Q8_0" + elif "q4_k_xl" in low or "q4-k-xl" in low: + quant = "UD-Q4_K_XL" + else: + quant = "?" + + mode = "no-think" if thinking is False else ("think" if thinking is True else "?") + + # Sampler point recorded VERBATIM rather than bucketed. An earlier binary + # matched/vendor split was misleading: every Qwen3.8 THINKING arm runs at + # T1.0/p0.95/pp0.0, which is Qwen3.6s vendor point, NOT Qwen3.8s + # (T0.7/p0.8/pp1.5). No Qwen3.8 thinking run at Qwen3.8s own vendor sampler + # exists in this corpus. Bucketing hid that; the verbatim triple exposes it. + def _f(x): + try: + return float(x) + except Exception: + return None + tf, pf, ppf = _f(temp), _f(top_p), _f(pp) + if tf is None: + regime = "?" + else: + regime = "T%g/p%g/pp%g" % (tf, pf if pf is not None else -1, + ppf if ppf is not None else -1) + + # --- outcomes --- + verdict = "" + gpath = os.path.join(d, "grade.json") + if os.path.exists(gpath): + try: + verdict = json.load(open(gpath)).get("verdict") or "" + except Exception: + pass + label_primary = "" + lpath = os.path.join(d, "label.json") + if os.path.exists(lpath): + try: + label_primary = json.load(open(lpath)).get("primary") or "" + except Exception: + label_primary = "unparseable" + tok = wall = "" + spath = os.path.join(d, "summary.json") + has_summary = os.path.exists(spath) + if has_summary: + try: + sj = json.load(open(spath)) + tok = sj.get("total_completion_tokens", "") + wall = sj.get("elapsed_s", "") + except Exception: + pass + + try: + recs = [json.loads(l) for l in open(tpath) if l.strip()] + except Exception: + skipped["unreadable_transcript"] += 1 + continue + tools = [r for r in recs if r.get("type") == "tool"] + n_tools = len(tools) + max_freq = max_run = 0 + distinct_ratio = "" + dead_frac = 0.0 + if n_tools: + blobs = [json.dumps(r.get("args"), sort_keys=True) for r in tools] + max_freq = max(collections.Counter(blobs).values()) + cur = mx = 1 + for i in range(1, len(blobs)): + cur = cur + 1 if blobs[i] == blobs[i - 1] else 1 + mx = max(mx, cur) + max_run = mx if len(blobs) > 1 else 1 + distinct_ratio = round(len(set(blobs)) / n_tools, 4) + dead = sum(1 for r in tools + if (r.get("wall_s") or 1) <= 0.05 and 130 <= (r.get("result_len") or 0) <= 200) + dead_frac = dead / n_tools + + rows.append(dict( + cell=base, repo=os.path.basename(repo), family=m.group(1), arm=m.group(2), + replicate=int(m.group(3)), model=family_model, quant=quant, mode=mode, + effort=effort, regime=regime, temperature=temp, top_p=top_p, top_k=top_k, + presence_penalty=pp, seed=seed, + verdict=verdict, graded=int(bool(verdict)), + terminal=int(has_summary or bool(label_primary)), + passed=int(verdict in PASSING), + looped_freq30=int(n_tools >= 5 and max_freq >= 30), + looped_run30=int(n_tools >= 5 and max_run >= 30), + max_freq=max_freq, max_run=max_run, n_tools=n_tools, + distinct_ratio=distinct_ratio, + completion_tokens=tok, elapsed_s=wall, label_primary=label_primary, + container_death_signature=int(n_tools >= 5 and dead_frac > 0.40), + transcript_mtime=int(os.path.getmtime(tpath)), + )) + +cols = ["cell", "repo", "family", "arm", "replicate", "model", "quant", "mode", "effort", + "regime", "temperature", "top_p", "top_k", "presence_penalty", "seed", + "verdict", "graded", "passed", "terminal", "looped_freq30", "looped_run30", + "max_freq", "max_run", "n_tools", "distinct_ratio", "completion_tokens", + "elapsed_s", "label_primary", "container_death_signature", "transcript_mtime"] + +rows.sort(key=lambda r: (r["model"], r["mode"], r["regime"], r["family"], r["replicate"])) +with open(OUT, "w", newline="") as f: + w = csv.DictWriter(f, fieldnames=cols) + w.writeheader() + for r in rows: + w.writerow(r) + +print("wrote %s (%d cells)" % (OUT, len(rows))) +print("skipped:", dict(skipped)) +print() +c = collections.Counter((r["model"], r["quant"], r["mode"], r["regime"]) for r in rows) +print("%-6s%-14s%-10s%-16s%7s" % ("model", "quant", "mode", "regime", "cells")) +for k in sorted(c): + print("%-6s%-14s%-10s%-16s%7d" % (k[0], k[1], k[2], k[3], c[k])) +print() +print("container-death-signature cells outside quarantine: %d" + % sum(r["container_death_signature"] for r in rows)) +print("cells with unknown model identity: %d" % sum(1 for r in rows if r["model"] == "?")) +print("cells with unknown mode: %d" % sum(1 for r in rows if r["mode"] == "?")) diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/make_md.py b/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/make_md.py new file mode 100755 index 00000000..cf2fd033 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/make_md.py @@ -0,0 +1,895 @@ +#!/usr/bin/env python3 +"""Render /home/michael/pr-staging/results-tables.md from results.json.""" +import json +from collections import OrderedDict + +R = json.load(open("/home/michael/pr-staging/results.json")) +OUT = "/home/michael/pr-staging/results-tables.md" +L = [] +def w(s=""): L.append(s) + +def pct(p, nd=1): + return "n/a" if p is None else ("%." + str(nd) + "f%%") % (100 * p) + +def ci(lo, hi, nd=1): + if lo is None or hi is None: return "n/a" + return ("[%." + str(nd) + "f, %." + str(nd) + "f]") % (100 * lo, 100 * hi) + +def pp(d, nd=1): + return "n/a" if d is None else ("%+." + str(nd) + "f pp") % (100 * d) + +def pv(p): + if p is None: return "n/a" + if p < 1e-4: return "%.1e" % p + if p < 0.001: return "%.5f" % p + return "%.4g" % p + +def sig(p): + if p is None: return "" + return " **\\***" if p < 0.05 else "" + +def num(v, nd=0): + if v is None: return "n/a" + return ("{:,." + str(nd) + "f}").format(v) + +G = R["groups"] +C = R["comparisons"] + +# ============================================================ header +w("# Qwen3.6-27B vs Qwen3.8-27B — Definitive Results Tables") +w() +w("All numbers below are computed from the frozen dataset") +w("`%s` (%d cells, frozen %s)." % (R["provenance"]["source_csv"], R["provenance"]["n_cells"], + R["provenance"]["frozen_at"])) +w("No live re-scan was used for any figure. Model, quantization and sampler identity are taken") +w("verbatim from the frozen CSV, which sourced them from each run's `receipt.json` — never from") +w("directory names.") +w() +w("**Reproduce:** `python3 mmbt_results.py` writes `results.json`; `python3 make_md.py` renders") +w("this file; `python3 test_stats.py` validates the estimators. No third-party dependencies.") +w() +w("### Reading these tables") +w() +w("- **Sign convention.** Every delta in this document is arithmetically **3.8 minus 3.6**,") +w(" applied identically to every table. Note that this means the *direction of badness flips") +w(" with the metric*: for good-direction metrics (terminal rate, pass rate) a **negative** delta") +w(" means 3.8 is worse, while for bad-direction metrics (ungraded rate, loop rate) a") +w(" **positive** delta means 3.8 is worse. Each table states which metric it reports.") +w("- **Every rate carries its denominator** as `x/n`.") +w("- **CIs** are Wilson 95% score intervals for single proportions and Newcombe 95% hybrid-score") +w(" intervals for differences. `p` is a two-sided Fisher exact test. All three are implemented") +w(" from scratch in pure python; formulas are in the appendix and in the source comments.") +w("- **\\*** marks `p < 0.05`. No multiplicity correction is applied — with %d comparison" % len(C)) +w(" families and 7 scorings each, treat isolated marginal results with suspicion.") +w("- `passed == 1` means verdict in {PASS, STRUCTURAL_PASS}. **Every ungraded cell has") +w(" `passed == 0`**, so an all-cells pass rate silently merges \"graded and failed\" with") +w(" \"never produced a gradeable artifact\". Both components are reported separately.") +w() + +# ============================================================ 0. inventory +w("---") +w() +w("## Headline") +w() +p1l = C["P1"]["delivery"]["loop_freq30"]; p2l = C["P2"]["delivery"]["loop_freq30"] +p1a = C["P1"]["quality"]["pass_all"]; p1g = C["P1"]["quality"]["pass_graded"] +w("**1. The one large, unambiguous effect is a delivery regression in no-think mode, not a") +w("quality regression.** 3.8's loop rate is %s (%s) at the matched sampler and %s (%s) at each" + % (pp(p1l["diff"]), pv(p1l["fisher_p"]), pp(p2l["diff"]), pv(p2l["fisher_p"]))) +w("model's own vendor sampler. This is measured upstream of grading, so no scoring choice, and") +w("none of the three grader defects, can touch it.") +w() +w("**2. That regression disappears in thinking mode.** Loop rate falls to %d/%d (T0.3) and %d/%d" + % (G["B38"]["loop_freq30"]["x"], G["B38"]["n"], G["C38"]["loop_freq30"]["x"], G["C38"]["n"])) +w("(T1/p0.95/pp0) for 3.8, neither distinguishable from 3.6.") +w() +w("**3. Conditional on delivering, 3.8 is not clearly worse at the matched no-think sampler.**") +_flip = " — the sign flips" if (p1a["diff"] or 0) < 0 < (p1g["diff"] or 0) else \ + (" — the gap collapses" if abs(p1g["diff"] or 0) < abs(p1a["diff"] or 0) / 2 else "") +w("All-cells pass rate is %s (%s) but graded-only is %s (%s)%s. \"3.8 is worse at the task\"" + % (pp(p1a["diff"]), pv(p1a["fisher_p"]), pp(p1g["diff"]), pv(p1g["fisher_p"]), _flip)) +w("is **not** a supportable summary of this corpus; \"3.8 fails more often by not finishing\" is.") +w() +w("**4. Thinking mode costs ~2.2–3.0× more tokens on 3.8 with no pass-rate gain**, and 3.8's") +w("effort ladder buys tokens rather than quality (no effort step is significant).") +w() +_q = R["quant_control"] +w("**5. Two things reported here are provisional at best and are labelled as such:** the Q8_0") +w("quantization arm (%d of %d cells graded at freeze #2 — enough for provisional loop and" + % (_q["n_graded"], _q["pass_all"]["n"])) +w("graded-outcome rates with wide intervals, not enough for any quality verdict, §6) and every") +w("`best_of_n` / `first_replicate` row (n = 12, §9).") +w() +w("Full robustness accounting is in §7; what the data cannot support is in §9.") +w() +w("---") +w() +w("## 0. Group inventory and missing-data structure") +w() +w("| Group | Definition | Cells | Families | Graded | Graded+failed | Ungraded (looped) | Ungraded (no loop) | Graded despite loop |") +w("|---|---|---:|---:|---:|---:|---:|---:|---:|") +for gid, g in G.items(): + m = g["missing"] + w("| `%s` | %s | %d | %d | %d | %d | %d | %d | %d |" % ( + gid, g["label"], g["n"], g["families"], g["n_graded"], + m["graded_fail"], m["ungraded_looped"], m["ungraded_not_looped"], m["graded_but_looped"])) +w() +w("Notes on group composition, which matter for how much weight each comparison can carry:") +w() +for gid, g in G.items(): + w("- **`%s`** — %s. *%s* Arms pooled: %s." % ( + gid, g["label"], g["note"], ", ".join("`%s`" % a for a in g["arms"]))) +w() +w("All %d runs used **seed 42**. Replicates are therefore repeated runs at a *fixed* seed, not" + % R["provenance"]["n_cells"]) +w("a seed sweep; within-arm variation reflects server/batching nondeterminism, not sampled seeds.") +w("This limits how far replicate counts can be read as independent draws.") +w() + +# ============================================================ 1. delivery +w("---") +w() +w("## 1. Delivery reliability") +w() +_ln = R["provenance"]["loop_nesting_counts"] +w("Loop detection is reported under both available metrics. `looped_run30` is strictly nested") +w("inside `looped_freq30` across the whole corpus (%d cells flagged by both, %d by frequency" + % (_ln["both"], _ln["freq_only"])) +w("only, %d by run only), so `freq30` is the more inclusive detector and `run30` the stricter one." + % _ln["run_only"]) +w() +w("### 1a. Per-group delivery rates (Wilson 95% CI)") +w() +w("| Group | n | Terminal | Ungraded | Loop (freq30) | Loop (run30) |") +w("|---|---:|---|---|---|---|") +for gid, g in G.items(): + w("| `%s` | %d | %s %s | %s %s | %s %s | %s %s |" % ( + gid, g["n"], + "%d/%d %s" % (g["terminal"]["x"], g["terminal"]["n"], pct(g["terminal"]["p"])), + ci(g["terminal"]["lo"], g["terminal"]["hi"]), + "%d/%d %s" % (g["ungraded"]["x"], g["ungraded"]["n"], pct(g["ungraded"]["p"])), + ci(g["ungraded"]["lo"], g["ungraded"]["hi"]), + "%d/%d %s" % (g["loop_freq30"]["x"], g["loop_freq30"]["n"], pct(g["loop_freq30"]["p"])), + ci(g["loop_freq30"]["lo"], g["loop_freq30"]["hi"]), + "%d/%d %s" % (g["loop_run30"]["x"], g["loop_run30"]["n"], pct(g["loop_run30"]["p"])), + ci(g["loop_run30"]["lo"], g["loop_run30"]["hi"]))) +w() +w("### 1b. Paired 3.6-vs-3.8 delivery contrasts") +w() +for pid, c in C.items(): + w("**%s — %s**" % (pid, c["description"])) + w() + w("| Metric | 3.6 `%s` | 3.8 `%s` | Δ (3.8−3.6) | Newcombe 95%% CI | Fisher p |" % tuple(c["pair"])) + w("|---|---|---|---:|---|---:|") + for k, lab in [("terminal", "Terminal"), ("ungraded", "Ungraded"), + ("loop_freq30", "Loop (freq30)"), ("loop_run30", "Loop (run30)")]: + v = c["delivery"][k] + w("| %s | %d/%d %s %s | %d/%d %s %s | %s | %s | %s%s |" % ( + lab, v["a"]["x"], v["a"]["n"], pct(v["a"]["p"]), ci(v["a"]["lo"], v["a"]["hi"]), + v["b"]["x"], v["b"]["n"], pct(v["b"]["p"]), ci(v["b"]["lo"], v["b"]["hi"]), + pp(v["diff"]), ci(v["diff_lo"], v["diff_hi"]), pv(v["fisher_p"]), sig(v["fisher_p"]))) + w() + +# ============================================================ 2. quality +w("---") +w() +w("## 2. Quality") +w() +w("Two pass rates are reported for every pair because they answer different questions:") +w() +w("- **All-cells** (loops count as failure) — *how often does a run of this model produce a") +w(" passing deliverable?* This is the end-to-end number.") +w("- **Graded-only** — *given that a run produced something gradeable, how good was it?* This") +w(" discards every delivery failure and is maximally charitable to a model that fails by") +w(" not delivering.") +w() +w("Where these two disagree, the disagreement **is** the finding.") +w() +for pid, c in C.items(): + w("**%s — %s vs %s**" % (pid, c["label_a"], c["label_b"])) + w() + w("| Scoring | 3.6 | 3.8 | Δ (3.8−3.6) | Newcombe 95% CI | Fisher p |") + w("|---|---|---|---:|---|---:|") + for k, lab in [("pass_all", "All cells (loops = fail)"), ("pass_graded", "Graded only")]: + v = c["quality"][k] + w("| %s | %d/%d %s %s | %d/%d %s %s | %s | %s | %s%s |" % ( + lab, v["a"]["x"], v["a"]["n"], pct(v["a"]["p"]), ci(v["a"]["lo"], v["a"]["hi"]), + v["b"]["x"], v["b"]["n"], pct(v["b"]["p"]), ci(v["b"]["lo"], v["b"]["hi"]), + pp(v["diff"]), ci(v["diff_lo"], v["diff_hi"]), pv(v["fisher_p"]), sig(v["fisher_p"]))) + w() + +# ============================================================ 3. per-family +w("---") +w() +w("## 3. Per-family breakdown") +w() +w("All-cells pass rate (loops count as failure), with graded and loop counts so any cell can be") +w("re-derived. `g` = graded cells, `L` = cells flagged by `looped_freq30`.") +w() +for pid, c in C.items(): + w("### %s — %s vs %s" % (pid, c["label_a"], c["label_b"])) + w() + w("| Family | 3.6 pass/n (g, L) | 3.6 rate | 3.8 pass/n (g, L) | 3.8 rate | Δ (3.8−3.6) | Fisher p |") + w("|---|---|---:|---|---:|---:|---:|") + for f in c["family_breakdown"]: + w("| `%s` | %d/%d (g=%d, L=%d) | %s | %d/%d (g=%d, L=%d) | %s | %s | %s%s |" % ( + f["family"], f["pass_a"], f["n_a"], f["graded_a"], f["loop_a"], pct(f["rate_a"]), + f["pass_b"], f["n_b"], f["graded_b"], f["loop_b"], pct(f["rate_b"]), + pp(f["delta"]), pv(f["fisher_p"]), sig(f["fisher_p"]))) + w() + _fns = [f[k] for f in c["family_breakdown"] for k in ("n_a", "n_b") if f[k]] + w("Per-family n is %d–%d per side, so **no single family row is individually conclusive**;" + % (min(_fns), max(_fns))) + w("they are provided so the pooled numbers can be audited, not to support family-level claims.") + w() + +# ============================================================ 4. cost +w("---") +w() +w("## 4. Cost, matched by family") +w() +w("**Why family matching is mandatory here.** Task families differ in cost by more than an order") +w("of magnitude (`p2_extract` ~2.5k tokens vs `p1_bugfix` ~35k). Two groups with different family") +w("mixes produce medians that can be ordered either way at will. Earlier in this project exactly") +w("that produced two contradictory cost claims from the same corpus. Every figure below is") +w("computed per family first.") +w() +w("**A second, subtler trap.** `completion_tokens` is missing for exactly the cells that were") +w("never graded, and looping is the dominant cause of that. Missingness is therefore *not*") +w("random — it is concentrated in the runs that would have been most expensive. A family is") +w("counted as **cost-comparable** only when both sides retain ≥3 cells with cost data *and*") +w("≥50% coverage of that side's cells in the family. Non-comparable families are shown but") +w("excluded from the summary ratio.") +w() +for pid, c in C.items(): + t = c["cost"]["tokens"]; e = c["cost"]["elapsed"] + w("### %s — %s vs %s" % (pid, c["label_a"], c["label_b"])) + w() + w("Token-data coverage: 3.6 %s (%d/%d cells) · 3.8 %s (%d/%d cells). " + "Cost-comparable families: **%d of %d**.%s" % ( + pct(t["coverage_a"], 0), t["with_data_a"], t["cells_a"], + pct(t["coverage_b"], 0), t["with_data_b"], t["cells_b"], + t["n_comparable_families"], t["n_families"], + (" Excluded: %s." % ", ".join("`%s`" % x for x in t["excluded_families"])) + if t["excluded_families"] else "")) + w() + w("| Family | 3.6 med tok (n) | 3.8 med tok (n) | tok ratio | 3.6 med s | 3.8 med s | s ratio | comparable |") + w("|---|---:|---:|---:|---:|---:|---:|:--:|") + emap = {x["family"]: x for x in e["per_family"]} + for a in t["per_family"]: + b = emap[a["family"]] + w("| `%s` | %s (%d) | %s (%d) | %s | %s (%d) | %s (%d) | %s | %s |" % ( + a["family"], num(a["median_a"]), a["n_a"], num(a["median_b"]), a["n_b"], + ("%.2f" % a["ratio"]) if a["ratio"] else "n/a", + num(b["median_a"], 1), b["n_a"], num(b["median_b"], 1), b["n_b"], + ("%.2f" % b["ratio"]) if b["ratio"] else "n/a", + "yes" if a["comparable"] else "**no**")) + w() + w("**Median of per-family ratios (comparable families only): tokens ×%s, wall-clock ×%s.**" % ( + ("%.2f" % t["median_of_family_ratios"]) if t["median_of_family_ratios"] else "n/a", + ("%.2f" % e["median_of_family_ratios"]) if e["median_of_family_ratios"] else "n/a")) + w() + # Is the excluded set systematically more expensive than the retained set? + # If so, the summary ratio is computed over the cheap tail and must say so. + if t["excluded_families"]: + inc = [x["median_a"] for x in t["per_family"] if x["comparable"] and x["median_a"]] + exc = [x["median_a"] for x in t["per_family"] if not x["comparable"] and x["median_a"]] + if inc and exc: + mi = sorted(inc)[len(inc) // 2] + me = sorted(exc)[len(exc) // 2] + if me > mi: + w("> ⚠️ **The excluded families are the expensive ones.** Median 3.6 cost is %s tokens" + % num(me)) + w("> across the excluded families versus %s across the retained ones (×%.1f). Because" + % (num(mi), me / mi)) + w("> cost data goes missing exactly when a run loops, and 3.8 loops most on the") + w("> long-horizon families, the ×%s summary ratio above is computed over the *cheap*" + % ("%.2f" % t["median_of_family_ratios"])) + w("> tail of the task set. **It must not be read as \"3.8 is cheaper overall\"** — the") + w("> comparison is silent on precisely the families where 3.8's cost would be worst.") + w() + ta = c["cost"]["tokens_per_success_a"]; tb = c["cost"]["tokens_per_success_b"] + w("| Tokens per successful deliverable | 3.6 | 3.8 |") + w("|---|---:|---:|") + w("| Total completion tokens recorded | %s | %s |" % (num(ta["total_tokens"]), num(tb["total_tokens"]))) + w("| Cells with token data | %d/%d | %d/%d |" % ( + ta["n_cells_with_tokens"], ta["n_cells"], tb["n_cells_with_tokens"], tb["n_cells"])) + w("| Passing cells | %d | %d |" % (ta["passes"], tb["passes"])) + w("| **Tokens per success** | **%s**%s | **%s**%s |" % ( + num(ta["tokens_per_success"]), " (lower bound)" if ta["is_lower_bound"] else "", + num(tb["tokens_per_success"]), " (lower bound)" if tb["is_lower_bound"] else "")) + w("| Tokens burned on cells that never got graded | %s | %s |" % ( + num(ta["tokens_on_ungraded"]), num(tb["tokens_on_ungraded"]))) + w() + notes = [x for x in (ta["lower_bound_note"], tb["lower_bound_note"]) if x] + if notes: + for side, n_ in zip(("3.6", "3.8"), (ta["lower_bound_note"], tb["lower_bound_note"])): + if n_: + w("- %s: %s" % (side, n_)) + w() + +# ============================================================ 5. effort +w("---") +w() +w("## 5. Effort ladder within 3.8 thinking") +w() +w("`effort` is populated **only** for 3.8 thinking cells; every 3.6 cell and every no-think cell") +w("has an empty effort field, so there is no 3.6 effort ladder to compare against. The ladder is") +w("broken out by sampler as well as pooled, because the two 3.8 thinking groups have very") +w("different effort mixes (T0.3: 14/24/13 low/med/xhigh; T1: 12/12/48) and pooling them") +w("confounds effort with sampler.") +w() +w("| Sampler | Effort | Cells | Families | Graded | Loops | Pass (all cells) | Wilson 95% CI | Pass (graded) | Median tokens | Median s |") +w("|---|---|---:|---:|---:|---:|---|---|---|---:|---:|") +for r in R["effort_ladder"]["rows"]: + pa, pg = r["pass_all"], r["pass_graded"] + w("| %s | %s | %d | %d | %d | %d | %d/%d %s | %s | %s | %s | %s |" % ( + r["regime"], r["effort"], r["n"], r["families"], r["n_graded"], r["loop_freq30"], + pa["x"], pa["n"], pct(pa["p"]), ci(pa["lo"], pa["hi"]), + ("%d/%d %s" % (pg["x"], pg["n"], pct(pg["p"]))) if pg["p"] is not None else "n/a", + num(r["median_tokens"]), num(r["median_elapsed"], 1))) +w() +w("### Pairwise effort steps (Δ = higher effort − lower effort, all-cells pass rate)") +w() +w("| Sampler | Step | Lower | Higher | Δ | Newcombe 95% CI | Fisher p |") +w("|---|---|---|---|---:|---|---:|") +for p in R["effort_ladder"]["pairwise"]: + w("| %s | %s → %s | %d/%d | %d/%d | %s | %s | %s%s |" % ( + p["regime"], p["lo"], p["hi"], p["a"]["x"], p["a"]["n"], p["b"]["x"], p["b"]["n"], + pp(p["diff"]), ci(p["diff_lo"], p["diff_hi"]), pv(p["fisher_p"]), sig(p["fisher_p"]))) +w() +# Recompute the prose numbers from the pairwise data rather than hardcoding +# them, so they can never drift from the table above. +_pw = R["effort_ladder"]["pairwise"] +_absd = [abs(100 * p["diff"]) for p in _pw] +_minp = min(p["fisher_p"] for p in _pw if p["fisher_p"] is not None) +w("**Not one effort step is statistically distinguishable** (every Fisher p ≥ %.3f; every" + % _minp) +w("Newcombe CI straddles zero). Cell counts per rung are 12–48, which cannot resolve the") +w("%.1f–%.1f pp differences observed. The apparent non-monotonicity — `medium` scoring lowest at" + % (min(_absd), max(_absd))) +w("both samplers — is **not** supported as a real effect and should not be reported as one.") +w() +_lrows = {(r["regime"], r["effort"]): r["median_tokens"] for r in R["effort_ladder"]["rows"]} +_t3lo, _t3xh = _lrows[("T0.3/p0.8/pp0", "low")], _lrows[("T0.3/p0.8/pp0", "xhigh")] +_t1lo, _t1xh = _lrows[("T1/p0.95/pp0", "low")], _lrows[("T1/p0.95/pp0", "xhigh")] +_ratios = sorted((_t3xh / _t3lo, _t1xh / _t1lo)) +w("What the ladder *does* show cleanly is cost: median tokens rise roughly %.1f–%.1f× from `low`" + % (_ratios[0], _ratios[1])) +w("to `xhigh` (T0.3: %s → %s; T1: %s → %s), with no accompanying pass-rate gain." + % (num(_t3lo), num(_t3xh), num(_t1lo), num(_t1xh))) +w("On this corpus, raising 3.8's thinking effort buys tokens, not quality.") +w() + +# ============================================================ 6. Q8 +w("---") +w() +w("## 6. Q8_0 quantization control") +w() +Q = R["quant_control"] +w("%s" % Q["note"]) +w() +w("> **What changed at freeze #2.** At freeze #1 this arm was 11 cells with zero graded and") +w("> carried no quality information at all. At freeze #2 it is %d cells with %d graded" + % (Q["pass_all"]["n"], Q["n_graded"])) +w("> (%d PASS, %d FAIL), so it now supports **provisional rates with wide intervals** —" + % (Q["n_pass_graded"], Q["n_fail_graded"])) +w("> loop rate %d/%d %s %s and graded-only pass rate %d/%d %s %s (Wilson 95%%)." % ( + Q["loop_freq30"]["x"], Q["loop_freq30"]["n"], pct(Q["loop_freq30"]["p"]), + ci(Q["loop_freq30"]["lo"], Q["loop_freq30"]["hi"]), + Q["pass_graded"]["x"], Q["pass_graded"]["n"], pct(Q["pass_graded"]["p"]), + ci(Q["pass_graded"]["lo"], Q["pass_graded"]["hi"]))) +w("> The loop interval excludes zero, so \"the no-think loop occurs at Q8_0 at a real,") +w("> non-negligible rate\" is now a provisional *rate* claim rather than an existence-only") +w("> observation. The quality numbers stay **descriptive only**: on n=%d graded cells no" + % Q["n_graded"]) +w("> pass-rate comparison against a Q4 arm can separate quantization from noise, and none is") +w("> asserted. %d of the %d cells ran to a terminal state and %d produced full token accounting." + % (Q["n_terminal"], Q["pass_all"]["n"], Q["n_with_tokens"])) +w() +uq = Q.get("unscored_qualitative") +if uq: + w("> **Unscored qualitative evidence, disclosed and excluded.** `%s`" % uq["cell"]) + w("> was quarantined in-flight at freeze #2 and is **not a row in the frozen CSV**: it was") + w("> observed in a rewrite loop — %s." % uq["observation"]) + w("> This is a different failure shape from the identical-call loop that `looped_freq30` /") + w("> `looped_run30` count, it is UNSCORED, and it is excluded from every rate and every") + w("> denominator in this document. It is recorded because a second loop subclass at Q8_0 is") + w("> qualitatively relevant to the quantization question even though it carries no") + w("> statistical weight.") + w() +qd = Q.get("post_freeze_drift") +if qd and qd["n_cells"]: + p_fams = sorted(c["family"] for c in qd["cells"] if c["on_disk_verdict"] == "PASS") + f_fams = sorted(c["family"] for c in qd["cells"] if c["on_disk_verdict"] == "FAIL") + w("> **Post-freeze grading drift, disclosed.** %d of these %d cells carry a verdict on" + % (qd["n_cells"], Q["pass_all"]["n"])) + w("> disk that differs from the frozen CSV — %d PASS (%s) and %d FAIL (%s)," % ( + qd["n_pass"], ", ".join("`%s`" % f for f in p_fams), + qd["n_fail"], ", ".join("`%s`" % f for f in f_fams))) + w("> per the overlay manifest's `post_freeze_divergence` ledger. Those grades postdate the") + w("> %s freeze and are **excluded from every number in this document**," % R["provenance"]["frozen_at"]) + w("> per the freeze discipline; they belong to the next freeze.") + w() +w("Per-cell detail for all %d Q8_0 cells as of freeze #2:" % Q["pass_all"]["n"]) +w() +w("| Family | Rep | Verdict | Graded | Terminal | Loop freq30 | Loop run30 | Tokens | Elapsed s |") +w("|---|---:|---|---:|---:|---:|---:|---:|---:|") +for r in Q["q8_cells"]: + w("| `%s` | %s | %s | %d | %d | %d | %d | %s | %s |" % ( + r["family"], r.get("replicate", ""), r["verdict"] if r["verdict"] else "*(none)*", + r["graded"], r["terminal"], + r["loop_f"], r["loop_r"], num(r["tokens"]), num(r["elapsed"], 1))) +w() +w("Delivery and (provisionally) graded-only quality against Q4 arms restricted to the same") +w("%d families:" % len(Q["q8_families"])) +w() +for key, lab in [("vs_38_q4_same_sampler", "vs 3.8 Q4 (same sampler, same model — isolates quantization)"), + ("vs_36_q4_same_sampler", "vs 3.6 Q4 (same sampler, different model)")]: + blk = Q[key] + w("**%s**" % lab) + w() + w("| Metric | Q4 comparator | Q8_0 | Δ (Q8−Q4) | Newcombe 95% CI | Fisher p |") + w("|---|---|---|---:|---|---:|") + metric_rows = [("terminal", "Terminal"), ("loop_freq30", "Loop (freq30)"), + ("loop_run30", "Loop (run30)")] + if "pass_graded" in blk: + metric_rows.append(("pass_graded", "Pass, graded only (PROVISIONAL)")) + for k, lb in metric_rows: + v = blk[k] + w("| %s | %d/%d %s | %d/%d %s | %s | %s | %s%s |" % ( + lb, v["a"]["x"], v["a"]["n"], pct(v["a"]["p"]), v["b"]["x"], v["b"]["n"], pct(v["b"]["p"]), + pp(v["diff"]), ci(v["diff_lo"], v["diff_hi"]), pv(v["fisher_p"]), sig(v["fisher_p"]))) + t = blk["tokens"] + w() + # Most Q8 families carry only 1-2 replicates, so the >=3-cells-per-side + # comparability rule of §4 is rarely met. Report the raw per-family pairs + # instead, explicitly as description rather than inference. + pairs = [x for x in t["per_family"] if x["ratio"]] + if pairs: + rs = sorted(x["ratio"] for x in pairs) + med = rs[len(rs) // 2] if len(rs) % 2 else (rs[len(rs)//2-1] + rs[len(rs)//2]) / 2 + w("Token cost, descriptive only (Q8_0 carries 1–2 replicates per family, so the") + w("≥3-cells-per-side comparability rule of §4 is generally not satisfied and no summary") + w("ratio is claimed): %d families have data on both sides; median per-family token " + "ratio Q8/Q4 = **×%.2f** (range ×%.2f–×%.2f)." % (len(pairs), med, min(rs), max(rs))) + w() + if key == "vs_36_q4_same_sampler": + w("> **Any loop difference in this table is a model effect, not a quantization") + w("> effect.** This row compares 3.8-Q8_0 against 3.6-Q4 — the models differ as well as the") + w("> quantization — so it simply reproduces the no-think looping gap already established in") + w("> §1. Only the preceding table (3.8-Q8_0 vs 3.8-Q4, same model and sampler) isolates") + w("> quantization.") + w() +w("With %d cells and 1–2 replicates per family, **every interval here still spans tens of" + % Q["pass_all"]["n"]) +w("percentage points**. What the arm now establishes: the identical-call loop occurs at Q8_0 at") +w("a rate whose Wilson interval excludes zero, so the loop is not an artifact of the UD-Q4_K_XL") +w("quantization. What it still cannot do: support any quality verdict, or attribute (or") +w("exonerate) the delivery regression as a quantization effect — there is still no matched 3.6") +w("Q8_0 arm, and the graded subset is n=%d. The graded outcomes above are published as data" + % Q["n_graded"]) +w("for the next freeze to build on, not as findings.") +w() + +# ============================================================ 7. sensitivity +w("---") +w() +w("## 7. Sensitivity: does the 3.6-vs-3.8 delta survive rescoring?") +w() +w("Seven scorings per comparison. The first six are the requested set; `graded_only` is added") +w("because it is the natural upper bound on charity toward a model whose failures are delivery") +w("failures.") +w() +SENS_ORDER = ["loops_as_failure", "loops_excluded", "loops_half_credit", "graded_only", + "best_of_n_per_family", "first_replicate_only", "depth_matched"] +first = C[list(C)[0]]["sensitivity"] +for k in SENS_ORDER: + if k in first: + w("- **`%s`** — %s" % (k, first[k]["definition"])) +w() +for pid, c in C.items(): + w("### %s — %s vs %s" % (pid, c["label_a"], c["label_b"])) + w() + w("| Scoring | 3.6 | 3.8 | Δ (3.8−3.6) | 95% CI on Δ | Fisher p |") + w("|---|---|---|---:|---|---:|") + for k in SENS_ORDER: + v = c["sensitivity"].get(k) + if not v: continue + a, b = v["a"], v["b"] + astr = ("%d/%d %s" % (a["x"], a["n"], pct(a["p"]))) if a.get("x") is not None \ + else ("mean %.3f (n=%d)" % (a["mean"], a["n"])) + bstr = ("%d/%d %s" % (b["x"], b["n"], pct(b["p"]))) if b.get("x") is not None \ + else ("mean %.3f (n=%d)" % (b["mean"], b["n"])) + w("| `%s` | %s | %s | %s | %s | %s%s |" % ( + k, astr, bstr, pp(v["diff"]), ci(v["diff_lo"], v["diff_hi"]), + pv(v.get("fisher_p")), sig(v.get("fisher_p")))) + w() + ds = [c["sensitivity"][k]["diff"] for k in SENS_ORDER if k in c["sensitivity"]] + ps = [c["sensitivity"][k].get("fisher_p") for k in SENS_ORDER if k in c["sensitivity"]] + nsig = sum(1 for p in ps if p is not None and p < 0.05) + ntest = sum(1 for p in ps if p is not None) + neg = all(d < 0 for d in ds); pos = all(d > 0 for d in ds) + w("Range of Δ across the seven scorings: **%s to %s**. Sign is %s. " + "Significant in **%d of %d** scorings that admit a Fisher test." % ( + pp(min(ds)), pp(max(ds)), + "consistently negative (3.8 worse)" if neg else + ("consistently positive (3.8 better)" if pos else "**NOT consistent — it flips**"), + nsig, ntest)) + w() +w("### What survives every scoring") +w() +w("Computed directly from the table above rather than asserted:") +w() +for pid, c in C.items(): + ds = [c["sensitivity"][k]["diff"] for k in SENS_ORDER if k in c["sensitivity"]] + ps = [c["sensitivity"][k].get("fisher_p") for k in SENS_ORDER if k in c["sensitivity"]] + nsig = sum(1 for p in ps if p is not None and p < 0.05) + ntest = sum(1 for p in ps if p is not None) + neg = all(d < 0 for d in ds); pos = all(d > 0 for d in ds) + lf = c["delivery"]["loop_freq30"] + if neg and nsig == ntest: + verdict = ("**Robust.** 3.8 is worse under every scoring and the difference is " + "significant under all %d testable ones." % ntest) + elif neg: + verdict = ("**Directionally robust, not uniformly significant.** 3.8 is worse under all " + "seven scorings, significant in %d of %d." % (nsig, ntest)) + elif pos: + verdict = ("**Directionally positive, not significant.** 3.8 is nominally better under " + "all seven scorings but significant in %d of %d." % (nsig, ntest)) + else: + verdict = ("**Not robust — the sign flips with the scoring rule.** No quality claim in " + "either direction survives. Significant in %d of %d." % (nsig, ntest)) + w("- **%s** (%s vs %s): %s" % (pid, c["label_a"], c["label_b"], verdict)) +w() +_p1l = C["P1"]["delivery"]["loop_freq30"] +w("**A confound in P2, and why it cannot carry the effect.** P2's vendor-matched design ties") +w("model identity to `presence_penalty`: the 3.8 vendor sampler carries pp=1.5 while 3.6's") +w("carries pp=0, so any P2 delta could in principle be a presence-penalty artifact rather than") +w("a model difference. That objection fails against the delivery finding: P1 holds pp=0 on") +w("**both** sides and still shows the %s loop delta (%s), so the looping regression" + % (pp(_p1l["diff"]), pv(_p1l["fisher_p"]))) +w("appears with the confound removed entirely. P2's *quality* deltas have no pp-free") +w("replication at the same (no-think) mode — P1, the pp=0 no-think pair, flips sign across") +w("scorings — so the confound caveat does apply to P2's pass rates. One more reason the loop") +w("finding, not the pass-rate finding, is this document's headline.") +w() +w("Three statements survive **every scoring rule**, in every comparison where they can be") +w("measured at all:") +w() +w("1. **3.8's no-think looping is real and large.** Loop rate (freq30) is %s in P1 (%s) and %s in" + % (pp(C["P1"]["delivery"]["loop_freq30"]["diff"]), pv(C["P1"]["delivery"]["loop_freq30"]["fisher_p"]), + pp(C["P2"]["delivery"]["loop_freq30"]["diff"]))) +w(" P2 (%s), with CIs excluding zero. It is the single largest and most reliable effect in the" + % pv(C["P2"]["delivery"]["loop_freq30"]["fisher_p"])) +w(" corpus, and it is not sensitive to any scoring choice because it is measured upstream of grading.") +w("2. **The looping is confined to no-think mode.** In thinking mode the loop rate collapses to") +w(" %d/%d and %d/%d for 3.8 (P3, P4) against %d/%d and %d/%d for 3.6 — no significant difference" + % (G["B38"]["loop_freq30"]["x"], G["B38"]["n"], G["C38"]["loop_freq30"]["x"], G["C38"]["n"], + G["B36"]["loop_freq30"]["x"], G["B36"]["n"], G["C36"]["loop_freq30"]["x"], G["C36"]["n"])) +w(" and, at T1/p0.95/pp0, zero loops on either side.") +w("3. **3.8 costs more in thinking mode.** Median per-family token ratio is ×%.2f (P3) and ×%.2f" + % (C["P3"]["cost"]["tokens"]["median_of_family_ratios"], + C["P4"]["cost"]["tokens"]["median_of_family_ratios"])) +w(" (P4), with wall-clock tracking it (×%.2f, ×%.2f), and no pass-rate gain to show for it." + % (C["P3"]["cost"]["elapsed"]["median_of_family_ratios"], + C["P4"]["cost"]["elapsed"]["median_of_family_ratios"])) +w() +w("The claim that does **not** survive is any clean statement that 3.8 produces worse *output*.") +_p1ad = C["P1"]["quality"]["pass_all"]["diff"] +_p1gd = C["P1"]["quality"]["pass_graded"]["diff"] +if _p1ad < 0 < _p1gd: + w("In P1 the sign flips: 3.8 is %.1f pp worse end-to-end but %.1f pp **better** among graded" + % (-100 * _p1ad, 100 * _p1gd)) + w("cells. Both cannot be summarized as \"3.8 is worse at the task\" — the honest reading is") + w("that 3.8 fails more often by not finishing, not by finishing badly.") +elif _p1ad < 0 and _p1gd < 0: + w("In P1, 3.8 is %.1f pp worse end-to-end but only %.1f pp worse among graded cells, with the" + % (-100 * _p1ad, -100 * _p1gd)) + w("graded-only CI straddling zero — the end-to-end gap is dominated by delivery, and the") + w("honest reading remains that 3.8 fails mostly by not finishing, not by finishing badly.") +else: + w("In P1 the end-to-end delta is %s and the graded-only delta is %s — the gap between the" + % (pp(_p1ad), pp(_p1gd))) + w("two is the delivery effect, and the honest reading remains that delivery, not output") + w("quality, is where the models separate.") +w() + +# ============================================================ 8. defects +w("---") +w() +w("## 8. Grader-defect exposure (why these tables are labelled *as-graded*)") +w() +w("Every pass rate in this document is **as-graded**: it uses the `passed` bit exactly as the") +w("original graders produced it. Three grader defects are verified in this corpus. Per repo") +w("convention they are corrected in a **separate non-destructive overlay**, never by editing") +w("`grade.json` or the task briefs, so longitudinal comparability is preserved. The tables here") +w("are the baseline that overlay will be applied to.") +w() +w("Diagnostics below are computed from raw `grade.json` files (the frozen CSV carries only the") +w("final pass/fail bit, so diagnosing *why* a cell failed requires the grader's score fields).") +w("Cell→model identity is joined from the frozen CSV by cell id, and cells that were ungraded") +w("at the freeze are excluded even where a post-freeze `grade.json` now exists on disk (see the") +w("drift note in §6). **No headline number above depends on this section.**") +w() +w("### Exposure: cells sitting in a defect-affected family") +w() +w("| Group | n | D1 length-gated (4 families) | D2 `p3_pm` | D3 `p2_triage` |") +w("|---|---:|---|---|---|") +for gid, v in R["defect_exposure"].items(): + w("| `%s` | %d | n=%d, graded=%d, pass=%d | n=%d, graded=%d, pass=%d | n=%d, graded=%d, pass=%d |" % ( + gid, v["n"], + v["d1_length_gated"]["n"], v["d1_length_gated"]["graded"], v["d1_length_gated"]["passes"], + v["d2_p3_pm"]["n"], v["d2_p3_pm"]["graded"], v["d2_p3_pm"]["passes"], + v["d3_p2_triage"]["n"], v["d3_p2_triage"]["graded"], v["d3_p2_triage"]["passes"])) +w() +D = R["defect_diagnostics"] +w("### D1 — word-gate tokenizer mismatch · correction favours **3.8**") +w() +b = D["D1_word_gate"]["p3_business"]; d = D["D1_word_gate"]["p3_doc"] +w("| Family | 3.6 pass/n | 3.6 over word limit | 3.8 pass/n | 3.8 over word limit |") +w("|---|---:|---:|---:|---:|") +w("| `p3_business` | %d/%d | %d | %d/%d | %d |" % ( + b["3.6"].get("PASS", 0), b["3.6"]["n"], b["3.6"].get("over_limit", 0), + b["3.8"].get("PASS", 0), b["3.8"]["n"], b["3.8"].get("over_limit", 0))) +w("| `p3_doc` | %d/%d | %d | %d/%d | %d |" % ( + d["3.6"].get("PASS", 0), d["3.6"]["n"], d["3.6"].get("over_limit", 0), + d["3.8"].get("PASS", 0), d["3.8"]["n"], d["3.8"].get("over_limit", 0))) +w() +w("In `p3_business` **every** FAIL on both sides is a word-limit FAIL (3.6: %d/%d, 3.8: %d/%d);" + % (b["3.6"].get("fail_with_over_limit", 0), b["3.6"].get("FAIL", 0), + b["3.8"].get("fail_with_over_limit", 0), b["3.8"].get("FAIL", 0))) +w("the substantive `stance_pushback` criterion was met by every graded cell of both models.") +w() +LIM = D["D1_word_gate"]["p3_doc_limit"] +ov = D["D1_word_gate"]["p3_doc_overshoot_wordcounts"] +w("Overshoot sizes in `p3_doc` against its %d-word ceiling, i.e. how far over the grader's own" % LIM) +w("counter each failing deliverable landed:") +w() +w("| Model | Over-limit cells | Word counts | Overshoot range | Median overshoot |") +w("|---|---:|---|---:|---:|") +for m in ["3.6", "3.8"]: + vals = sorted(ov.get(m, [])) + if not vals: continue + over = [v - LIM for v in vals] + med = over[len(over) // 2] if len(over) % 2 else (over[len(over)//2-1] + over[len(over)//2]) / 2 + w("| %s | %d | %s | %d–%d words | %.1f words |" % ( + m, len(vals), ", ".join(str(v) for v in vals), min(over), max(over), med)) +w() +w("The bulk of these are single-digit to low-double-digit overruns on a 700-word budget — 3.6's") +w("largest is %d words over and its median is %.0f — which is well inside the disagreement between" + % (max(v - LIM for v in ov["3.6"]), + sorted(v - LIM for v in ov["3.6"])[len(ov["3.6"]) // 2])) +w("the two counters. That is the substance of D1: the graders count `\\b\\w+\\b` while both models") +w("budgeted with shell `wc -w`, and neither counter is ground truth. **The tail runs longer on") +w("3.8's side** — its worst `p3_doc` overshoot is %d words (%d against a %d ceiling) versus" + % (max(v - LIM for v in ov["3.8"]), max(ov["3.8"]), LIM)) +w("3.6's %d — so the overlay must report per-cell outcomes rather than assume a blanket" + % max(v - LIM for v in ov["3.6"])) +w("reversal; correcting D1 should be expected to rescue most, though not necessarily all, of") +w("3.8's length failures.") +w() +w("### D2 — `p3_pm` risk-keyword literalism · correction favours **3.6**") +w() +pm = D["D2_p3_pm"] +w("| Model | Pass/n | FAILs blocked by risk_recall < 3 | risk_recall distribution |") +w("|---|---:|---:|---|") +for m in ["3.6", "3.8"]: + if m not in pm: continue + dist = pm["risk_recall_distribution"].get(m, {}) + w("| %s | %d/%d | %d | %s |" % ( + m, pm[m].get("PASS", 0), pm[m]["n"], pm[m].get("fail_with_risk_recall_below_min", 0), + ", ".join("%s risks: %d cells" % (k, v) for k, v in sorted(dist.items())))) +w() +w("**%d of %d** 3.6 cells land on exactly `2/6` risks — one short of the `min_risks = 3`" + % (pm["3.6"]["risk_recall_distribution"].get("2", 0) if isinstance(pm["3.6"].get("risk_recall_distribution"), dict) + else D["D2_p3_pm"]["risk_recall_distribution"]["3.6"].get("2", 0), pm["3.6"]["n"])) +w("threshold — and every one of the %d 3.6 FAILs is blocked by that rule alone. This is the" + % pm["3.6"].get("FAIL", 0)) +w("signature of a single unmatched keyword, consistent with the verified R3 literalism ") +w("(3.6 writes \"legal has not responded\"; the rule requires a contracted form).") +w() +w("### D3 — `p2_triage` brief/ground-truth contradiction · correction favours **3.6**") +w() +t3 = D["D3_p2_triage"] +w("| Quantity | Value |") +w("|---|---:|") +w("| Graded triage cells examined | %d |" % t3["n_cells"]) +w("| Cells answering `n/a` on all three spam tickets (004, 009, 021) | **%d/%d** |" % ( + t3["all_three_spam_na"], t3["n_cells"])) +w("| FAIL cells | %d |" % t3["fails"]) +w("| FAIL cells blocked by urgency accuracy *alone* | **%d/%d** |" % ( + t3["fail_blocked_by_urgency_only"], t3["fails"])) +w("| FAIL cells that flip to PASS once the spam penalty is removed | **%d/%d** |" % ( + sum(t3["would_flip_to_pass"].values()), t3["fails"])) +w("| …of which 3.6 | %d |" % t3["would_flip_to_pass"].get("3.6", 0)) +w("| …of which 3.8 | %d |" % t3["would_flip_to_pass"].get("3.8", 0)) +w() +w("> **Correction to the briefed characterization of D3.** The defect brief states that") +w("> `p2_triage` \"has zero discriminating power as graded\". The frozen data does not support") +w("> that as written: **as graded** the family splits %d PASS / %d FAIL and does discriminate" + % (t3["verdicts"].get("PASS", 0), t3["verdicts"].get("FAIL", 0))) +w("> (3.6 %d/%d vs 3.8 %d/%d). The accurate statement is the reverse in time: all %d cells take" + % (t3["by_model"].get("3.6:PASS", 0), + t3["by_model"].get("3.6:PASS", 0) + t3["by_model"].get("3.6:FAIL", 0), + t3["by_model"].get("3.8:PASS", 0), + t3["by_model"].get("3.8:PASS", 0) + t3["by_model"].get("3.8:FAIL", 0), + t3["n_cells"])) +w("> the identical 0.100 urgency penalty, and because the threshold is 0.700 that penalty decides") +w("> the verdict purely by where each cell already sat — cells at 0.767–0.800 survive, cells at") +w("> 0.633–0.667 do not. The observed split is threshold noise, not signal. It is **after** the") +w("> correction that the family becomes uniformly passing and loses all discriminating power.") +w() +w("### Net direction of the pending overlay") +w() +w("The three defects **do not push the same way**: D1 materially helps 3.8, while D2 and D3 help") +w("3.6. Anyone assuming the corrections will uniformly move the headline in one direction is") +w("wrong. The as-graded numbers in this document should not be read as biased against either") +w("model until the overlay is computed.") +w() +w("As one bound on how much of the quality signal is defect-contaminated, restricting to the") +w("seven families with **no** verified grader defect (`%s`):" + % "`, `".join(R["defect_free_subset"]["P1"]["clean_families"])) +w() +w("| Comparison | 3.6 | 3.8 | Δ (3.8−3.6) | Newcombe 95% CI | Fisher p |") +w("|---|---|---|---:|---|---:|") +for pid, v in R["defect_free_subset"].items(): + w("| %s | %d/%d %s | %d/%d %s | %s | %s | %s%s |" % ( + pid, v["a"]["x"], v["a"]["n"], pct(v["a"]["p"]), v["b"]["x"], v["b"]["n"], pct(v["b"]["p"]), + pp(v["diff"]), ci(v["diff_lo"], v["diff_hi"]), pv(v["fisher_p"]), sig(v["fisher_p"]))) +w() +w("This subset is **not** a substitute for the overlay: it changes the family mix (and in P1/P2 it") +w("is heavily confounded by 3.8's looping, which is concentrated in `p1_bugfix` and `p1_testwrite`).") +w("It is reported only to show that the defect families are load-bearing for the as-graded totals.") +w() + +# ============================================================ 9. power +w("---") +w() +w("## 9. Comparisons too underpowered to report") +w() +w("Screening rule, applied uniformly: a comparison is flagged **UNDERPOWERED** if either arm has") +w("n < 30, or the Newcombe 95% CI on the difference is wider than 30 pp — i.e. the interval") +w("cannot separate a small effect from a large one in either direction.") +w() +rows = [] +for pid, c in C.items(): + for k, v in c["power"].items(): + blk = (c["quality"].get(k.replace("sens_", "")) or c["delivery"].get(k) or + c["sensitivity"].get(k.replace("sens_", ""))) + if blk is None or blk.get("diff_lo") is None: continue + wdt = blk["diff_hi"] - blk["diff_lo"] + rows.append((pid, k, blk["diff"], blk["diff_lo"], blk["diff_hi"], wdt, v, + blk.get("fisher_p"))) +w("This flag is about **precision, not significance**, and the two are independent. A result can") +_ex = next((r for r in rows if r[6]["underpowered"] and r[7] is not None and r[7] < 0.05), None) +if _ex: + w("be flagged here and still be statistically significant (%s `%s`: %s, p = %s, but a" + % (_ex[0], _ex[1].replace("sens_", ""), pp(_ex[2]), pv(_ex[7]))) + w("%.1f pp-wide interval). Read such rows as \"the direction is probably real, the magnitude" + % (100 * _ex[5])) +else: + w("be flagged here and still be statistically significant. Read such rows as \"the direction") + w("is probably real, the magnitude") +_p3q = C["P3"]["quality"]["pass_all"] +w("is not pinned down\". Conversely, P3's non-significant rows are flagged because the data cannot") +w("distinguish \"no difference\" from \"a difference of %.0f pp in either direction\" — **P3 is not" + % (100 * max(abs(_p3q["diff_lo"]), abs(_p3q["diff_hi"])))) +w("evidence of equivalence.**") +w() +w("| Comparison | Metric | Δ | 95% CI | Width | Verdict |") +w("|---|---|---:|---|---:|---|") +for pid, k, d, lo, hi, wdt, v, _fp in rows: + if not v["underpowered"]: continue + w("| %s | `%s` | %s | %s | %.1f pp | UNDERPOWERED — %s |" % ( + pid, k.replace("sens_", ""), pp(d), ci(lo, hi), 100 * wdt, "; ".join(v["reasons"]))) +w() +w("Explicitly **not** reportable:") +w() +_bw = [100 * (c["sensitivity"]["best_of_n_per_family"]["diff_hi"] + - c["sensitivity"]["best_of_n_per_family"]["diff_lo"]) for c in C.values()] +_ew = [100 * (p["diff_hi"] - p["diff_lo"]) for p in R["effort_ladder"]["pairwise"]] +_q9 = R["quant_control"] +w("- **The Q8_0 quantization arm for any quality *verdict*.** %d of %d cells graded at freeze #2;" + % (_q9["n_graded"], _q9["pass_all"]["n"])) +w(" the graded-only rate %d/%d is published in §6 as provisional data with a Wilson interval" + % (_q9["pass_graded"]["x"], _q9["pass_graded"]["n"])) +w(" spanning tens of points — direction-finding for the next freeze, not evidence.") +w("- **Every `best_of_n_per_family` result.** n = 12 families per side by construction; CI widths") +w(" are %.1f–%.1f pp. These rows are shown for completeness of the sensitivity grid, not as evidence." + % (min(_bw), max(_bw))) +_frn = [c["sensitivity"]["first_replicate_only"][s]["n"] for c in C.values() for s in ("a", "b")] +w("- **Every `first_replicate_only` result** for the same reason (n = %d–%d)." + % (min(_frn), max(_frn))) +_nword = {6: "six", 7: "seven", 8: "eight", 9: "nine", 10: "ten"}.get(len(_ew), str(len(_ew))) +w("- **Every effort-ladder step.** All %s pairwise steps are non-significant with CI widths" + % _nword) +w(" of %.1f–%.1f pp." % (min(_ew), max(_ew))) +_fam_ns = [f[k] for c in C.values() for f in c["family_breakdown"] for k in ("n_a", "n_b") if f[k]] +w("- **All per-family rows in §3** individually (n = %d–%d per side)." + % (min(_fam_ns), max(_fam_ns))) +w("- **P4's quality delta** is borderline and, more importantly, structurally biased: the shared") +w(" T1/p0.95/pp0 sampler is 3.6's vendor point and not 3.8's, so 3.8 runs off-spec by construction.") +w() + +# ============================================================ 10. appendix +w("---") +w() +w("## 10. Method appendix") +w() +w("### Estimators (pure python, no scipy)") +w() +w("**Wilson score interval** for a proportion — inverts the score test rather than using the") +w("Wald form, so it stays inside [0,1] and behaves at x = 0 and x = n (several arms here are 0/n):") +w() +w("```") +w("center = (p + z^2/(2n)) / (1 + z^2/n)") +w("half = z * sqrt( p(1-p)/n + z^2/(4n^2) ) / (1 + z^2/n)") +w("CI = center -/+ half z = 1.959963984540054") +w("```") +w() +w("**Fisher exact test**, two-sided. Conditioning on both margins, the top-left cell follows the") +w("hypergeometric law; the p-value is the Fisher/Irwin sum-of-small-probabilities form used by") +w("R's `fisher.test`:") +w() +w("```") +w("P(a) = C(r1,a) * C(r2,c1-a) / C(r1+r2,c1)") +w("p = sum of P(x) over all tables x with the same margins where P(x) <= P(observed)") +w("```") +w() +w("Computed in exact rational arithmetic (`fractions.Fraction`), so no floating-point tie-breaking.") +w() +w("**Newcombe method 10** (hybrid score) CI for a difference of proportions — propagates the two") +w("Wilson intervals instead of a pooled Wald SE, which is what keeps it sane at boundary counts:") +w() +w("```") +w("lower = (p1-p2) - sqrt( (p1-l1)^2 + (u2-p2)^2 )") +w("upper = (p1-p2) + sqrt( (u1-p1)^2 + (p2-l2)^2 )") +w("```") +w() +w("**Half-credit scoring** produces per-cell scores in {0, 0.5, 1}, which are not Bernoulli.") +w("Wilson/Newcombe/Fisher are therefore **not** applied to it; that row uses a z-interval on") +w("the difference of means (`z_diff_mean` — a normal approximation with critical value z, not") +w("a Welch-t interval) and is labelled as such.") +w() +w("### Validation") +w() +w("`test_stats.py` checks each estimator against its defining equation or an exact independent") +w("enumeration rather than against remembered published values:") +w() +w("- Wilson limits are verified to be exact roots of the score equation (the score statistic") +w(" evaluates to z at both limits to 1e-9), and to bracket p̂ for every (x, n) with n ≤ 199.") +w("- The hypergeometric pmf is verified to sum to exactly 1 in rational arithmetic, and to match") +w(" brute-force enumeration over all C(11,4) subsets.") +w("- Fisher reproduces the tea-tasting table `[[3,1],[1,3]]` as exactly 17/35, is symmetric under") +w(" row and column swaps, and agrees with the chi-square approximation at large n.") +w("- Newcombe is verified against its construction, verified antisymmetric under group swap, and") +w(" its zero-exclusion agrees with Fisher p < 0.05 on 98.2% of a 441-table grid.") +w() +w("One real defect was found and fixed by this suite: at x = 0 and x = n the Wilson closed form") +w("missed the exact boundary by ~1e-16, leaving the interval not quite containing p̂. Those two") +w("cases are now snapped to exactly 0 and 1.") +w() +w("### Known limitations of this dataset") +w() +_nc = R["provenance"]["n_cells"] +_dr = R["provenance"].get("depth_range", {}) +_cm = R["provenance"]["cost_missing"] +w("1. **Single seed.** All %d runs use seed 42; replicates are repeat runs, not independent" % _nc) +w(" seeds. Binomial CIs treat cells as independent draws, which is optimistic.") +w("2. **Unequal and non-random depth.** Replicate counts vary by family and arm (%s per" + % (("%d–%d" % (_dr["min"], _dr["max"])) if _dr else "1–19")) +w(" family-group). The `depth_matched` scoring exists to bound the effect of this.") +w("3. **Non-random missing cost data.** %d of %d cells lack `completion_tokens`; %s" + % (_cm["n"], _nc, + ("all %d are" % _cm["n"]) if _cm["all_ungraded"] else "nearly all are")) +w(" ungraded and most looped. Cost figures for affected arms are lower bounds.") +w("4. **Pooled effort in 3.8 thinking.** Groups `B38` and `C38` each pool three effort levels;") +w(" their aggregate pass rates are effort-mix-weighted, not single-configuration numbers.") +w("5. **As-graded only.** Three verified grader defects remain uncorrected here by design.") +w("6. **Arm pooling.** Several groups pool two or more arms that share model, quant, mode and") +w(" sampler triple but were run as separate sweeps (listed in §0).") +w() + +open(OUT, "w").write("\n".join(L) + "\n") +print("wrote", OUT, len(L), "lines") diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/mmbt_results.py b/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/mmbt_results.py new file mode 100755 index 00000000..ca53abbf --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/mmbt_results.py @@ -0,0 +1,862 @@ +#!/usr/bin/env python3 +""" +MMBT Qwen3.6-27B vs Qwen3.8-27B -- definitive results tables. + +Single source of truth: /home/michael/mmbt-frozen-dataset-v2.csv (802 cells, +freeze #2, frozen 2026-08-16T14:23:09Z). Nothing here re-scans the live corpus. +Freeze #1 (mmbt-frozen-dataset.csv, 746 cells, 2026-08-16T11:49:14Z) is superseded. + +All statistics are implemented in pure python (no scipy). Formulas are in +comments at each implementation. +""" +from __future__ import annotations + +import csv +import json +import math +from fractions import Fraction +from collections import OrderedDict, defaultdict + +CSV_PATH = "/home/michael/mmbt-frozen-dataset-v2.csv" +FROZEN_AT = "2026-08-16T14:23:09Z" # freeze #2 stamp, /home/michael/FREEZE2_STAMP.txt +OUT_MD = "/home/michael/pr-staging/results-tables.md" +OUT_JSON = "/home/michael/pr-staging/results.json" + +Z = 1.959963984540054 # standard normal 97.5th percentile -> 95% two-sided + + +# -------------------------------------------------------------------------- +# STATISTICS (pure python) +# -------------------------------------------------------------------------- + +def wilson(x, n, z=Z): + """Wilson score interval for a binomial proportion. + + Solves |p_hat - p| / sqrt(p(1-p)/n) = z for p. Closed form: + + center = (p_hat + z^2/(2n)) / (1 + z^2/n) + half = z * sqrt( p_hat(1-p_hat)/n + z^2/(4n^2) ) / (1 + z^2/n) + CI = center -/+ half + + Unlike the Wald interval this never leaves [0,1] and stays sensible at + x = 0 or x = n, which matters here because several cells are 0/N or N/N. + """ + if n == 0: + return (None, None, None) + p = x / n + denom = 1.0 + z * z / n + center = (p + z * z / (2.0 * n)) / denom + half = z * math.sqrt(p * (1.0 - p) / n + z * z / (4.0 * n * n)) / denom + lo, hi = center - half, center + half + # At x=0 the two terms are algebraically equal, and at x=n they sum to + # exactly 1; in floating point they miss by ~1e-16, which would leave the + # interval failing to contain p_hat. Snap those exact boundary cases. + if x == 0: + lo = 0.0 + if x == n: + hi = 1.0 + return (p, max(0.0, lo), min(1.0, hi)) + + +def _hypergeom_pmf(a, r1, r2, c1): + """P(X=a) for the central (non-central psi=1) hypergeometric: + + P(a) = C(r1, a) * C(r2, c1 - a) / C(r1 + r2, c1) + + Exact rational arithmetic -- no floating point until the final compare. + """ + n = r1 + r2 + return Fraction(math.comb(r1, a) * math.comb(r2, c1 - a), math.comb(n, c1)) + + +def fisher_exact_two_sided(a, b, c, d): + """Two-sided Fisher exact test on the 2x2 table + + success failure + g1 a b (row total r1 = a+b) + g2 c d (row total r2 = c+d) + + Conditioning on both margins, the count `a` follows the hypergeometric + distribution above. The two-sided p-value is the standard "sum of small + probabilities" (Fisher/Irwin) definition used by R's fisher.test: + + p = sum over all tables t with the same margins of P(t), + for every t with P(t) <= P(observed) * (1 + eps) + + eps = 1e-9 guards against ties being dropped by rounding. Exact rationals + make that guard nearly cosmetic, but it is kept for safety. + """ + r1, r2, c1 = a + b, c + d, a + c + n = r1 + r2 + if r1 == 0 or r2 == 0 or c1 == 0 or c1 == n: + return 1.0 + lo = max(0, c1 - r2) + hi = min(r1, c1) + p_obs = _hypergeom_pmf(a, r1, r2, c1) + tol = p_obs * Fraction(1000000001, 1000000000) + total = Fraction(0) + for x in range(lo, hi + 1): + px = _hypergeom_pmf(x, r1, r2, c1) + if px <= tol: + total += px + return min(1.0, float(total)) + + +def newcombe_diff(x1, n1, x2, n2, z=Z): + """Newcombe's method 10 ("hybrid score") 95% CI for p1 - p2. + + Let (l1,u1) and (l2,u2) be the Wilson intervals for p1 and p2. Then + + lower = (p1 - p2) - sqrt( (p1 - l1)^2 + (u2 - p2)^2 ) + upper = (p1 - p2) + sqrt( (u1 - p1)^2 + (p2 - l2)^2 ) + + This propagates each group's *score* interval rather than a pooled Wald + SE, so it behaves at boundary counts where Wald difference CIs break. + """ + if n1 == 0 or n2 == 0: + return (None, None, None) + p1, l1, u1 = wilson(x1, n1, z) + p2, l2, u2 = wilson(x2, n2, z) + d = p1 - p2 + lower = d - math.sqrt((p1 - l1) ** 2 + (u2 - p2) ** 2) + upper = d + math.sqrt((u1 - p1) ** 2 + (p2 - l2) ** 2) + return (d, max(-1.0, lower), min(1.0, upper)) + + +def z_diff_mean(xs, ys, z=Z): + """z-interval (normal approximation) on the difference of MEANS of + bounded scores. + + d = mean(xs) - mean(ys) + se = sqrt( var(xs)/n1 + var(ys)/n2 ) (unbiased sample variances) + CI = d -/+ z*se + + NOT a Welch t-interval: the critical value is the fixed normal z, with no + Welch-Satterthwaite degrees of freedom. At the n here (>=51 per side) the + difference is negligible, but the name should not overclaim. + + Used only for the half-credit scoring, where the per-cell score is in + {0, 0.5, 1} and therefore NOT binomial -- Wilson/Newcombe do not apply. + Labelled as such everywhere it is reported. + """ + n1, n2 = len(xs), len(ys) + if n1 < 2 or n2 < 2: + return (None, None, None) + m1 = sum(xs) / n1 + m2 = sum(ys) / n2 + v1 = sum((v - m1) ** 2 for v in xs) / (n1 - 1) + v2 = sum((v - m2) ** 2 for v in ys) / (n2 - 1) + se = math.sqrt(v1 / n1 + v2 / n2) + d = m1 - m2 + return (d, d - z * se, d + z * se) + + +def median(vals): + v = sorted(vals) + n = len(v) + if n == 0: + return None + if n % 2: + return float(v[n // 2]) + return (v[n // 2 - 1] + v[n // 2]) / 2.0 + + +# -------------------------------------------------------------------------- +# DATA +# -------------------------------------------------------------------------- + +def load(): + with open(CSV_PATH, newline="") as fh: + rows = list(csv.DictReader(fh)) + for r in rows: + for k in ("graded", "passed", "terminal", "looped_freq30", "looped_run30", + "replicate", "n_tools", "max_freq", "max_run"): + r[k] = int(r[k]) + for k in ("completion_tokens", "elapsed_s"): + r[k] = float(r[k]) if r[k].strip() != "" else None + return rows + + +ROWS = load() + +# Group keys are (model, quant, mode, regime) -- exactly the identity fields +# the frozen CSV carries from receipt.json. Never derived from paths. +GROUPS = OrderedDict([ + ("A36", dict(label="3.6 no-think T0.3/p0.8/pp0", + note="off-spec for 3.6 (its card is T1/p0.95/pp0); sampler-matched to A38", + key=("3.6", "UD-Q4_K_XL", "no-think", "T0.3/p0.8/pp0"))), + ("A38", dict(label="3.8 no-think T0.3/p0.8/pp0", + note="off-spec for 3.8 (its card is T0.7/p0.8/pp1.5); sampler-matched to A36", + key=("3.8", "UD-Q4_K_XL", "no-think", "T0.3/p0.8/pp0"))), + ("V36", dict(label="3.6 no-think T1/p0.95/pp0", + note="3.6 VENDOR point", + key=("3.6", "UD-Q4_K_XL", "no-think", "T1/p0.95/pp0"))), + ("V38", dict(label="3.8 no-think T0.7/p0.8/pp1.5", + note="3.8 VENDOR point", + key=("3.8", "UD-Q4_K_XL", "no-think", "T0.7/p0.8/pp1.5"))), + ("B36", dict(label="3.6 think T0.3/p0.8/pp0", + note="off-spec for 3.6; sampler-matched to B38", + key=("3.6", "UD-Q4_K_XL", "think", "T0.3/p0.8/pp0"))), + ("B38", dict(label="3.8 think T0.3/p0.8/pp0", + note="off-spec for 3.8; sampler-matched to B36; MIXED effort (low/medium/xhigh)", + key=("3.8", "UD-Q4_K_XL", "think", "T0.3/p0.8/pp0"))), + ("C36", dict(label="3.6 think T1/p0.95/pp0", + note="3.6 VENDOR point", + key=("3.6", "UD-Q4_K_XL", "think", "T1/p0.95/pp0"))), + ("C38", dict(label="3.8 think T1/p0.95/pp0", + note="this is 3.6's vendor point, NOT 3.8's; sampler-matched to C36 but off-spec " + "for 3.8; MIXED effort (low/medium/xhigh)", + key=("3.8", "UD-Q4_K_XL", "think", "T1/p0.95/pp0"))), + ("Q8", dict(label="3.8 Q8_0 no-think T0.3/p0.8/pp0", + note="quantization control; 19 cells over all 12 families " + "(2 replicates on the 7 phase-1/2 families, 1 on the 5 phase-3)", + key=("3.8", "Q8_0", "no-think", "T0.3/p0.8/pp0"))), +]) + +for gid, g in GROUPS.items(): + m, q, md, rg = g["key"] + g["rows"] = [r for r in ROWS + if r["model"] == m and r["quant"] == q and r["mode"] == md and r["regime"] == rg] + +PAIRS = [ + ("P1", "A36", "A38", "No-think, SAMPLER-MATCHED (both T0.3/p0.8/pp0). Off-spec for both " + "models; the cleanest controlled-sampler contrast."), + ("P2", "V36", "V38", "No-think, VENDOR-MATCHED (each model at its own model-card sampler: " + "3.6 T1/p0.95/pp0, 3.8 T0.7/p0.8/pp1.5). Best-foot-forward contrast; " + "sampler differs by construction."), + ("P3", "B36", "B38", "Thinking, SAMPLER-MATCHED (both T0.3/p0.8/pp0). Off-spec for both. " + "3.8 side pools three effort levels."), + ("P4", "C36", "C38", "Thinking at T1/p0.95/pp0. Sampler-matched, but that triple is 3.6's " + "vendor point and is NOT 3.8's -- 3.8 is running off-spec here and this " + "pair is biased in 3.6's favour. 3.8 side pools three effort levels."), +] + +FAMILIES = sorted(set(r["family"] for r in ROWS)) +LENGTH_GATED = ["p3_business", "p3_doc", "p3_pm", "p3_writing"] # verified: hard word gate +TRIAGE = ["p2_triage"] +PM = ["p3_pm"] + + +# -------------------------------------------------------------------------- +# METRIC BLOCKS +# -------------------------------------------------------------------------- + +def rate_block(rows, pred, denom_rows=None): + dr = rows if denom_rows is None else denom_rows + n = len(dr) + x = sum(1 for r in dr if pred(r)) + p, lo, hi = wilson(x, n) + return dict(x=x, n=n, p=p, lo=lo, hi=hi) + + +def cmp_block(rows_a, rows_b, pred, denom_a=None, denom_b=None): + """SIGN CONVENTION, used identically everywhere in this file: + diff = p_b - p_a (i.e. 3.8 minus 3.6) + so a NEGATIVE diff always means the newer model is worse on that metric. + Group `a` is always the 3.6 arm and group `b` always the 3.8 arm. + """ + A = rate_block(rows_a, pred, denom_a) + B = rate_block(rows_b, pred, denom_b) + if A["n"] == 0 or B["n"] == 0: + return dict(a=A, b=B, fisher_p=None, diff=None, diff_lo=None, diff_hi=None) + p = fisher_exact_two_sided(A["x"], A["n"] - A["x"], B["x"], B["n"] - B["x"]) + d, lo, hi = newcombe_diff(B["x"], B["n"], A["x"], A["n"]) + return dict(a=A, b=B, fisher_p=p, diff=d, diff_lo=lo, diff_hi=hi) + + +IS_TERMINAL = lambda r: r["terminal"] == 1 +IS_UNGRADED = lambda r: r["graded"] == 0 +IS_LOOP_F = lambda r: r["looped_freq30"] == 1 +IS_LOOP_R = lambda r: r["looped_run30"] == 1 +IS_PASS = lambda r: r["passed"] == 1 + + +def group_summary(rows): + graded = [r for r in rows if r["graded"] == 1] + return dict( + n=len(rows), + # Missing-data structure. "passed==0" pools two very different things: + # a cell that was graded and failed, and a cell that was never graded at + # all. Anyone treating all-cells pass rate as a quality measure needs + # these counts, so they are carried explicitly. + missing=dict( + graded_fail=sum(1 for r in rows if r["graded"] == 1 and r["passed"] == 0), + ungraded_looped=sum(1 for r in rows if r["graded"] == 0 and r["looped_freq30"] == 1), + ungraded_not_looped=sum(1 for r in rows + if r["graded"] == 0 and r["looped_freq30"] == 0), + graded_but_looped=sum(1 for r in rows + if r["graded"] == 1 and r["looped_freq30"] == 1)), + terminal=rate_block(rows, IS_TERMINAL), + ungraded=rate_block(rows, IS_UNGRADED), + loop_freq30=rate_block(rows, IS_LOOP_F), + loop_run30=rate_block(rows, IS_LOOP_R), + pass_graded=rate_block(graded, IS_PASS), + pass_all=rate_block(rows, IS_PASS), + n_graded=len(graded), + families=len(set(r["family"] for r in rows)), + arms=sorted(set(r["arm"] for r in rows)), + cost_missing=sum(1 for r in rows if r["completion_tokens"] is None), + ) + + +# -------------------------------------------------------------------------- +# COST, MATCHED BY FAMILY +# -------------------------------------------------------------------------- + +def cost_by_family(rows_a, rows_b, field): + """Per-family medians restricted to families present in BOTH groups. + + Rationale: an unmatched pooled median silently reweights toward whichever + families a group happens to have more replicates of. Earlier in this + project that produced two contradictory cost claims from the same corpus. + + Cost data is MISSING exactly when a cell was never graded, and looping is the + dominant cause of that. So missingness is not random: it is concentrated in + the runs that would have been most expensive. A family is therefore only + treated as cost-comparable when BOTH sides retain >=3 cells with cost data + AND >=50% of that side's cells in the family. Uncovered families are still + printed, but excluded from the summary ratio. + """ + fa = set(r["family"] for r in rows_a) + fb = set(r["family"] for r in rows_b) + shared = sorted(fa & fb) + per = [] + for f in shared: + ca = [r for r in rows_a if r["family"] == f] + cb = [r for r in rows_b if r["family"] == f] + va = [r[field] for r in ca if r[field] is not None] + vb = [r[field] for r in cb if r[field] is not None] + ma, mb = median(va), median(vb) + cov_a = len(va) / len(ca) if ca else 0.0 + cov_b = len(vb) / len(cb) if cb else 0.0 + ok = (len(va) >= 3 and len(vb) >= 3 and cov_a >= 0.5 and cov_b >= 0.5) + per.append(dict(family=f, n_a=len(va), n_b=len(vb), + cells_a=len(ca), cells_b=len(cb), + coverage_a=cov_a, coverage_b=cov_b, comparable=ok, + median_a=ma, median_b=mb, + delta=(mb - ma) if (ma is not None and mb is not None) else None, + ratio=(mb / ma) if (ma not in (None, 0) and mb is not None) else None)) + good = [p for p in per if p["comparable"]] + deltas = [p["delta"] for p in good if p["delta"] is not None] + ratios = [p["ratio"] for p in good if p["ratio"] is not None] + all_ratios = [p["ratio"] for p in per if p["ratio"] is not None] + pooled_a = median([r[field] for r in rows_a if r["family"] in shared and r[field] is not None]) + pooled_b = median([r[field] for r in rows_b if r["family"] in shared and r[field] is not None]) + tot_a = sum(1 for r in rows_a if r["family"] in shared) + tot_b = sum(1 for r in rows_b if r["family"] in shared) + have_a = sum(1 for r in rows_a if r["family"] in shared and r[field] is not None) + have_b = sum(1 for r in rows_b if r["family"] in shared and r[field] is not None) + return dict(shared_families=shared, per_family=per, + n_comparable_families=len(good), + comparable_families=[p["family"] for p in good], + excluded_families=[p["family"] for p in per if not p["comparable"]], + median_of_family_deltas=median(deltas) if deltas else None, + median_of_family_ratios=median(ratios) if ratios else None, + median_of_family_ratios_ALL=median(all_ratios) if all_ratios else None, + pooled_median_a=pooled_a, pooled_median_b=pooled_b, + pooled_is_censored=(have_a < tot_a or have_b < tot_b), + coverage_a=have_a / tot_a if tot_a else None, + coverage_b=have_b / tot_b if tot_b else None, + cells_a=tot_a, cells_b=tot_b, with_data_a=have_a, with_data_b=have_b, + n_families=len(shared)) + + +def tokens_per_success(rows, families=None): + """Total completion tokens spent / number of PASSing cells. + + Denominator is passes among the SAME cell set. Cells with no token + accounting (all of them ungraded, hence non-passing) are excluded from + the numerator and reported separately -- this makes the figure a lower + bound on true cost per success. + """ + sel = rows if families is None else [r for r in rows if r["family"] in families] + toks = [r["completion_tokens"] for r in sel if r["completion_tokens"] is not None] + passes = sum(1 for r in sel if r["passed"] == 1) + miss = len(sel) - len(toks) + return dict(total_tokens=sum(toks), n_cells_with_tokens=len(toks), + n_cells=len(sel), n_missing_tokens=miss, + token_coverage=len(toks) / len(sel) if sel else None, + passes=passes, + tokens_per_success=(sum(toks) / passes) if passes else None, + # Tokens burned by cells that produced no grade at all: pure waste + # that IS captured when the tokens were recorded. + tokens_on_ungraded=sum(r["completion_tokens"] for r in sel + if r["graded"] == 0 and r["completion_tokens"] is not None), + is_lower_bound=miss > 0, + lower_bound_note=("%d of %d cell%s no token accounting (all of them " + "ungraded), so total tokens and tokens-per-success are " + "UNDERSTATED for this arm." + % (miss, len(sel), " has" if miss == 1 else "s have")) + if miss else None) + + +# -------------------------------------------------------------------------- +# SENSITIVITY SCORINGS +# -------------------------------------------------------------------------- + +def order_key(r): + return (r["arm"], r["replicate"], r["cell"]) + + +def depth_matched(rows_a, rows_b): + """Equal replicates per family: for each shared family take the first + k = min(n_a, n_b) cells from each side, ordered deterministically by + (arm, replicate, cell). No randomness, fully reproducible.""" + fa = defaultdict(list) + fb = defaultdict(list) + for r in rows_a: + fa[r["family"]].append(r) + for r in rows_b: + fb[r["family"]].append(r) + out_a, out_b = [], [] + for f in sorted(set(fa) & set(fb)): + k = min(len(fa[f]), len(fb[f])) + out_a += sorted(fa[f], key=order_key)[:k] + out_b += sorted(fb[f], key=order_key)[:k] + return out_a, out_b + + +def best_of_n(rows): + """Per family: 1 if ANY cell in the group passed, else 0.""" + fam = defaultdict(list) + for r in rows: + fam[r["family"]].append(r) + return {f: (1 if any(r["passed"] == 1 for r in v) else 0) for f, v in fam.items()} + + +def sensitivity(rows_a, rows_b): + out = OrderedDict() + + # 1. loops-as-failure = all-cells pass rate (ungraded/looped cells score 0) + out["loops_as_failure"] = dict( + definition="All cells in the group; passed==1 is success. Loops and ungraded " + "cells score 0. This is the headline scoring.", + stat="binomial (Wilson/Newcombe/Fisher)", + **cmp_block(rows_a, rows_b, IS_PASS)) + + # 2. loops-excluded + ea = [r for r in rows_a if r["looped_freq30"] == 0] + eb = [r for r in rows_b if r["looped_freq30"] == 0] + out["loops_excluded"] = dict( + definition="Cells with looped_freq30==1 dropped entirely, then pass rate over " + "the remainder. Charitable to whichever model loops more. NOTE: cells " + "that are ungraded WITHOUT having looped are retained here and score 0.", + stat="binomial (Wilson/Newcombe/Fisher)", + **cmp_block(ea, eb, IS_PASS)) + + # 2b. graded-only: drops loops AND ungraded cells. This is the pure + # "when it produced something gradeable, how good was it" view. + ga = [r for r in rows_a if r["graded"] == 1] + gb = [r for r in rows_b if r["graded"] == 1] + out["graded_only"] = dict( + definition="Only graded==1 cells. Removes every delivery failure (loops and " + "ungraded alike) and measures output quality conditional on delivery. " + "Maximally charitable to a model that fails by not delivering.", + stat="binomial (Wilson/Newcombe/Fisher)", + **cmp_block(ga, gb, IS_PASS)) + + # 3. loops-as-half-credit (NOT binomial -> mean-difference CI) + sa = [1.0 if r["passed"] == 1 else (0.5 if r["looped_freq30"] == 1 else 0.0) for r in rows_a] + sb = [1.0 if r["passed"] == 1 else (0.5 if r["looped_freq30"] == 1 else 0.0) for r in rows_b] + d, lo, hi = z_diff_mean(sb, sa) # sign convention: 3.8 minus 3.6 + out["loops_half_credit"] = dict( + definition="Score 1.0 for pass, 0.5 for a non-passing looped cell, 0.0 otherwise; " + "compare MEAN scores. Scores are not 0/1 so Wilson and Fisher do not " + "apply; CI is a z-interval on the difference of means.", + stat="z-interval on difference of means -- NOT Wilson/Fisher, NOT Welch-t", + a=dict(mean=sum(sa) / len(sa) if sa else None, n=len(sa)), + b=dict(mean=sum(sb) / len(sb) if sb else None, n=len(sb)), + diff=d, diff_lo=lo, diff_hi=hi, fisher_p=None) + + # 4. best-of-N per family + ba, bb = best_of_n(rows_a), best_of_n(rows_b) + shared = sorted(set(ba) & set(bb)) + xa = sum(ba[f] for f in shared) + xb = sum(bb[f] for f in shared) + n = len(shared) + pa, la, ua = wilson(xa, n) + pb, lb, ub = wilson(xb, n) + fp = fisher_exact_two_sided(xa, n - xa, xb, n - xb) + dd, dlo, dhi = newcombe_diff(xb, n, xa, n) # sign convention: 3.8 minus 3.6 + out["best_of_n_per_family"] = dict( + definition="One Bernoulli trial per SHARED family: 1 if any cell in that group/family " + "passed. Measures 'can the model ever do this task', not reliability. " + "n = number of families, so it is structurally low-powered.", + stat="binomial over families (Wilson/Newcombe/Fisher)", + a=dict(x=xa, n=n, p=pa, lo=la, hi=ua), + b=dict(x=xb, n=n, p=pb, lo=lb, hi=ub), + fisher_p=fp, diff=dd, diff_lo=dlo, diff_hi=dhi, + per_family={f: dict(a=ba[f], b=bb[f]) for f in shared}) + + # 5. first-replicate-only + ra = [r for r in rows_a if r["replicate"] == 1] + rb = [r for r in rows_b if r["replicate"] == 1] + out["first_replicate_only"] = dict( + definition="Only rows with replicate==1 (one run per family per arm). Removes any " + "weighting from unequal repeat depth; retains one row per arm, so n is " + "families x arms, not families.", + stat="binomial (Wilson/Newcombe/Fisher)", + **cmp_block(ra, rb, IS_PASS)) + + # 6. depth-matched + da, db = depth_matched(rows_a, rows_b) + out["depth_matched"] = dict( + definition="Per shared family, the first k=min(n_a,n_b) cells from each side ordered " + "by (arm, replicate, cell). Both sides then have identical family x depth " + "composition.", + stat="binomial (Wilson/Newcombe/Fisher)", + **cmp_block(da, db, IS_PASS)) + + return out + + +def family_breakdown(rows_a, rows_b): + fa = set(r["family"] for r in rows_a) + fb = set(r["family"] for r in rows_b) + out = [] + for f in sorted(fa | fb): + A = [r for r in rows_a if r["family"] == f] + B = [r for r in rows_b if r["family"] == f] + ga = [r for r in A if r["graded"] == 1] + gb = [r for r in B if r["graded"] == 1] + rec = dict(family=f, + n_a=len(A), pass_a=sum(1 for r in A if r["passed"] == 1), + graded_a=len(ga), loop_a=sum(1 for r in A if r["looped_freq30"] == 1), + n_b=len(B), pass_b=sum(1 for r in B if r["passed"] == 1), + graded_b=len(gb), loop_b=sum(1 for r in B if r["looped_freq30"] == 1)) + rec["rate_a"] = rec["pass_a"] / rec["n_a"] if rec["n_a"] else None + rec["rate_b"] = rec["pass_b"] / rec["n_b"] if rec["n_b"] else None + rec["delta"] = (rec["rate_b"] - rec["rate_a"]) if (rec["rate_a"] is not None and rec["rate_b"] is not None) else None + if rec["n_a"] and rec["n_b"]: + rec["fisher_p"] = fisher_exact_two_sided( + rec["pass_a"], rec["n_a"] - rec["pass_a"], + rec["pass_b"], rec["n_b"] - rec["pass_b"]) + else: + rec["fisher_p"] = None + out.append(rec) + return out + + +# -------------------------------------------------------------------------- +# EFFORT LADDER (3.8 thinking only) +# -------------------------------------------------------------------------- + +def effort_ladder(): + out = [] + think38 = [r for r in ROWS if r["model"] == "3.8" and r["mode"] == "think"] + for regime in [None, "T0.3/p0.8/pp0", "T1/p0.95/pp0"]: + sel0 = think38 if regime is None else [r for r in think38 if r["regime"] == regime] + for eff in ["low", "medium", "xhigh"]: + sel = [r for r in sel0 if r["effort"] == eff] + if not sel: + continue + graded = [r for r in sel if r["graded"] == 1] + toks = [r["completion_tokens"] for r in sel if r["completion_tokens"] is not None] + el = [r["elapsed_s"] for r in sel if r["elapsed_s"] is not None] + pa = rate_block(sel, IS_PASS) + pg = rate_block(graded, IS_PASS) + out.append(dict(regime=regime or "ALL (pooled)", effort=eff, n=len(sel), + families=len(set(r["family"] for r in sel)), + n_graded=len(graded), + loop_freq30=sum(1 for r in sel if r["looped_freq30"] == 1), + pass_all=pa, pass_graded=pg, + median_tokens=median(toks), n_tokens=len(toks), + median_elapsed=median(el))) + # pairwise low vs xhigh within each regime + pw = [] + for regime in ["T0.3/p0.8/pp0", "T1/p0.95/pp0", None]: + sel0 = think38 if regime is None else [r for r in think38 if r["regime"] == regime] + for e1, e2 in [("low", "medium"), ("medium", "xhigh"), ("low", "xhigh")]: + s1 = [r for r in sel0 if r["effort"] == e1] + s2 = [r for r in sel0 if r["effort"] == e2] + if not s1 or not s2: + continue + c = cmp_block(s1, s2, IS_PASS) + pw.append(dict(regime=regime or "ALL (pooled)", lo=e1, hi=e2, **c)) + return out, pw + + +# -------------------------------------------------------------------------- +# GRADER-DEFECT EXPOSURE +# -------------------------------------------------------------------------- + +def defect_exposure(): + """How much of each group's cell budget sits in a family with a verified + grader defect. Numbers below are AS-GRADED; the overlay is separate.""" + out = {} + for gid, g in GROUPS.items(): + rows = g["rows"] + n = len(rows) + d1 = [r for r in rows if r["family"] in LENGTH_GATED] + d2 = [r for r in rows if r["family"] in PM] + d3 = [r for r in rows if r["family"] in TRIAGE] + out[gid] = dict( + n=n, + d1_length_gated=dict(n=len(d1), passes=sum(1 for r in d1 if r["passed"] == 1), + graded=sum(1 for r in d1 if r["graded"] == 1)), + d2_p3_pm=dict(n=len(d2), passes=sum(1 for r in d2 if r["passed"] == 1), + graded=sum(1 for r in d2 if r["graded"] == 1)), + d3_p2_triage=dict(n=len(d3), passes=sum(1 for r in d3 if r["passed"] == 1), + graded=sum(1 for r in d3 if r["graded"] == 1)), + union=dict(n=sum(1 for r in rows + if r["family"] in set(LENGTH_GATED) | set(TRIAGE)), + passes=sum(1 for r in rows + if r["family"] in set(LENGTH_GATED) | set(TRIAGE) + and r["passed"] == 1))) + return out + + +def defect_free_subset(): + """Pass rate restricted to families with NO verified grader defect: + everything except the 4 length-gated phase-3 families and p2_triage.""" + bad = set(LENGTH_GATED) | set(TRIAGE) + clean = [f for f in FAMILIES if f not in bad] + out = {} + for pid, ga, gb, desc in PAIRS: + ra = [r for r in GROUPS[ga]["rows"] if r["family"] in clean] + rb = [r for r in GROUPS[gb]["rows"] if r["family"] in clean] + out[pid] = dict(clean_families=clean, **cmp_block(ra, rb, IS_PASS)) + return out + + +# -------------------------------------------------------------------------- +# POWER SCREEN +# -------------------------------------------------------------------------- + +def power_note(block): + """Flag comparisons too imprecise to carry a claim. + + Rule applied uniformly: a comparison is reported as UNDERPOWERED if either + arm has n < 30, or if the Newcombe 95% CI on the difference is wider than + 0.30 (30 percentage points) -- i.e. the interval cannot separate a small + effect from a large one in either direction. + """ + a, b = block.get("a", {}), block.get("b", {}) + na, nb = a.get("n", 0) or 0, b.get("n", 0) or 0 + lo, hi = block.get("diff_lo"), block.get("diff_hi") + reasons = [] + if na < 30 or nb < 30: + reasons.append("n<30 in an arm (n_a=%d, n_b=%d)" % (na, nb)) + if lo is not None and hi is not None and (hi - lo) > 0.30: + reasons.append("Newcombe CI width %.1f pp" % (100 * (hi - lo))) + straddles = (lo is not None and hi is not None and lo < 0 < hi) + return dict(underpowered=bool(reasons), reasons=reasons, ci_straddles_zero=straddles) + + +# -------------------------------------------------------------------------- +# BUILD +# -------------------------------------------------------------------------- + +R = OrderedDict() +_LOOP_BOTH = sum(1 for r in ROWS if r["looped_run30"] == 1 and r["looped_freq30"] == 1) +_LOOP_FREQ_ONLY = sum(1 for r in ROWS if r["looped_freq30"] == 1 and r["looped_run30"] == 0) +_LOOP_RUN_ONLY = sum(1 for r in ROWS if r["looped_run30"] == 1 and r["looped_freq30"] == 0) +_COST_MISSING = [r for r in ROWS if r["completion_tokens"] is None] +R["provenance"] = dict( + source_csv=CSV_PATH, + frozen_at=FROZEN_AT, + n_cells=len(ROWS), + seeds=sorted(set(r["seed"] for r in ROWS)), + note="Model/quant/sampler identity taken verbatim from the frozen CSV, which sourced it " + "from receipt.json. No live re-scan. No identity inferred from directory names.", + columns_used=["family", "model", "quant", "mode", "effort", "regime", "replicate", "arm", + "graded", "passed", "terminal", "looped_freq30", "looped_run30", + "completion_tokens", "elapsed_s"], + passed_semantics="passed==1 corresponds to verdict in {PASS, STRUCTURAL_PASS}; every " + "ungraded cell (graded==0) has passed==0.", + loop_nesting="looped_run30==1 implies looped_freq30==1 (%d cells both, %d freq-only, " + "%d run-only)." % (_LOOP_BOTH, _LOOP_FREQ_ONLY, _LOOP_RUN_ONLY), + loop_nesting_counts=dict(both=_LOOP_BOTH, freq_only=_LOOP_FREQ_ONLY, + run_only=_LOOP_RUN_ONLY), + cost_missing=dict(n=len(_COST_MISSING), + all_ungraded=all(r["graded"] == 0 for r in _COST_MISSING)), + depth_range=(lambda c: dict(min=min(c.values()), max=max(c.values())))( + {(gid, r["family"]): sum(1 for x in g["rows"] if x["family"] == r["family"]) + for gid, g in GROUPS.items() for r in g["rows"]}), +) +R["groups"] = {gid: dict(label=g["label"], note=g["note"], + key=dict(zip(["model", "quant", "mode", "regime"], g["key"])), + **group_summary(g["rows"])) for gid, g in GROUPS.items()} + +R["comparisons"] = OrderedDict() +for pid, ga, gb, desc in PAIRS: + ra, rb = GROUPS[ga]["rows"], GROUPS[gb]["rows"] + gra = [r for r in ra if r["graded"] == 1] + grb = [r for r in rb if r["graded"] == 1] + ent = dict( + pair=[ga, gb], description=desc, + label_a=GROUPS[ga]["label"], label_b=GROUPS[gb]["label"], + delivery=dict( + terminal=cmp_block(ra, rb, IS_TERMINAL), + ungraded=cmp_block(ra, rb, IS_UNGRADED), + loop_freq30=cmp_block(ra, rb, IS_LOOP_F), + loop_run30=cmp_block(ra, rb, IS_LOOP_R)), + quality=dict( + pass_graded=cmp_block(gra, grb, IS_PASS), + pass_all=cmp_block(ra, rb, IS_PASS)), + family_breakdown=family_breakdown(ra, rb), + cost=dict( + tokens=cost_by_family(ra, rb, "completion_tokens"), + elapsed=cost_by_family(ra, rb, "elapsed_s")), + sensitivity=sensitivity(ra, rb), + ) + shared = set(ent["cost"]["tokens"]["shared_families"]) + ent["cost"]["tokens_per_success_a"] = tokens_per_success(ra, shared) + ent["cost"]["tokens_per_success_b"] = tokens_per_success(rb, shared) + ent["power"] = dict( + pass_all=power_note(ent["quality"]["pass_all"]), + pass_graded=power_note(ent["quality"]["pass_graded"]), + loop_freq30=power_note(ent["delivery"]["loop_freq30"]), + loop_run30=power_note(ent["delivery"]["loop_run30"]), + ungraded=power_note(ent["delivery"]["ungraded"]), + ) + for sk, sv in ent["sensitivity"].items(): + if sv.get("diff_lo") is not None: + ent["power"]["sens_" + sk] = power_note(sv) + R["comparisons"][pid] = ent + +ladder, ladder_pw = effort_ladder() +R["effort_ladder"] = dict(rows=ladder, pairwise=ladder_pw, + note="Effort is populated only for 3.8 thinking cells; all 3.6 cells " + "and all no-think cells have an empty effort field.") + +# --- Q8 control --- +q8 = GROUPS["Q8"]["rows"] +q8_fams = set(r["family"] for r in q8) +a38 = GROUPS["A38"]["rows"] +a36 = GROUPS["A36"]["rows"] +a38m = [r for r in a38 if r["family"] in q8_fams] +a36m = [r for r in a36 if r["family"] in q8_fams] +Q8_GRADED = sum(1 for r in q8 if r["graded"] == 1) +Q8_TERMINAL = sum(1 for r in q8 if r["terminal"] == 1) +Q8_WITH_TOKENS = sum(1 for r in q8 if r["completion_tokens"] is not None) + +OVERLAY_MANIFEST = "/home/michael/pr-staging/overlay/manifest.json" + + +def post_freeze_drift(q8_rows): + """Post-freeze grading drift for the Q8_0 arm, read from the overlay + manifest's post_freeze_divergence ledger (never from a live re-scan of the + corpus, and never folded into any statistic -- disclosure only).""" + try: + with open(OVERLAY_MANIFEST) as fh: + man = json.load(fh) + div = man.get("post_freeze_divergence", []) + except (OSError, ValueError): + return None + frozen = {r["cell"]: r["family"] for r in q8_rows} + ent = [d for d in div if d.get("cell") in frozen] + return dict( + source=OVERLAY_MANIFEST, + n_cells=len(ent), + n_pass=sum(1 for d in ent if d.get("on_disk_verdict") == "PASS"), + n_fail=sum(1 for d in ent if d.get("on_disk_verdict") == "FAIL"), + cells=[dict(cell=d["cell"], family=frozen[d["cell"]], + on_disk_verdict=d.get("on_disk_verdict")) for d in ent], + note="Verdicts written to disk AFTER the freeze; excluded from every " + "number in this document and disclosed as drift.") + + +Q8_DRIFT = post_freeze_drift(q8) +q8_graded = [r for r in q8 if r["graded"] == 1] +a38m_graded = [r for r in a38m if r["graded"] == 1] +_q8_reps = defaultdict(int) +for r in q8: + _q8_reps[r["family"]] += 1 +R["quant_control"] = dict( + note="Q8_0 arm at freeze #2 is %d cells over all %d families: 2 replicates on the " + "%d phase-1/phase-2 families, 1 on the %d phase-3 families." + % (len(q8), len(q8_fams), + sum(1 for v in _q8_reps.values() if v == 2), + sum(1 for v in _q8_reps.values() if v == 1)), + quality_provisional=True, + quality_note=( + "%d of the %d Q8_0 cells are graded in the frozen CSV (freeze #2), so the arm " + "now carries PROVISIONAL quality information: as-graded pass-given-delivery is " + "%d/%d. The Wilson interval on that rate spans tens of percentage points, so no " + "quality conclusion is drawn -- the graded outcomes are published as data, not " + "as a quantization verdict. Delivery metrics (terminal, loop rates) cover all " + "%d cells and are the arm's primary evidence." + % (Q8_GRADED, len(q8), sum(1 for r in q8_graded if r["passed"] == 1), + Q8_GRADED, len(q8))), + n_graded=Q8_GRADED, + n_pass_graded=sum(1 for r in q8_graded if r["passed"] == 1), + n_fail_graded=sum(1 for r in q8_graded if r["passed"] == 0), + n_terminal=Q8_TERMINAL, + n_with_tokens=Q8_WITH_TOKENS, + pass_graded=rate_block(q8_graded, IS_PASS), + pass_all=rate_block(q8, IS_PASS), + loop_freq30=rate_block(q8, IS_LOOP_F), + loop_run30=rate_block(q8, IS_LOOP_R), + post_freeze_drift=Q8_DRIFT, + q8_families=sorted(q8_fams), + reportable_metrics=["terminal", "looped_freq30", "looped_run30", + "completion_tokens", "elapsed_s", + "pass_graded (provisional, n=%d)" % Q8_GRADED], + unscored_qualitative=dict( + cell="p3_doc_qwen38q8-nothink-matched_v2", + status="quarantined in-flight at freeze #2; NOT a row in the frozen CSV; " + "excluded from every rate and every denominator in this document", + observation="139 iterations rewriting brief.md, context grown to ~228k tokens " + "when quarantined", + note="UNSCORED qualitative evidence only. It documents that the Q8_0 arm can " + "also exhibit a rewrite-loop failure shape (distinct from the " + "identical-call loop the loop metrics count); it carries no weight in any " + "rate above and is not part of the loop_freq30/loop_run30 counts."), + vs_38_q4_same_sampler=dict( + comparator="A38 (3.8 Q4 no-think T0.3/p0.8/pp0) restricted to the %d Q8 families" + % len(q8_fams), + loop_freq30=cmp_block(a38m, q8, IS_LOOP_F), + loop_run30=cmp_block(a38m, q8, IS_LOOP_R), + terminal=cmp_block(a38m, q8, IS_TERMINAL), + pass_graded=dict( + note="PROVISIONAL: graded-only pass rate, Q8 n=%d. Underpowered by the " + "power screen; direction only." % len(q8_graded), + **cmp_block(a38m_graded, q8_graded, IS_PASS)), + tokens=cost_by_family(a38m, q8, "completion_tokens"), + elapsed=cost_by_family(a38m, q8, "elapsed_s")), + vs_36_q4_same_sampler=dict( + comparator="A36 (3.6 Q4 no-think T0.3/p0.8/pp0) restricted to the %d Q8 families" + % len(q8_fams), + loop_freq30=cmp_block(a36m, q8, IS_LOOP_F), + loop_run30=cmp_block(a36m, q8, IS_LOOP_R), + terminal=cmp_block(a36m, q8, IS_TERMINAL), + tokens=cost_by_family(a36m, q8, "completion_tokens"), + elapsed=cost_by_family(a36m, q8, "elapsed_s")), + q8_cells=[dict(cell=r["cell"], family=r["family"], replicate=r["replicate"], + verdict=r["verdict"], + graded=r["graded"], passed=r["passed"], terminal=r["terminal"], + loop_f=r["looped_freq30"], loop_r=r["looped_run30"], + tokens=r["completion_tokens"], elapsed=r["elapsed_s"]) for r in + sorted(q8, key=lambda r: (r["family"], r["replicate"]))], +) + +R["defect_exposure"] = defect_exposure() +R["defect_free_subset"] = defect_free_subset() + +# family-level as-graded pass rates pooled by model, for the defect narrative +R["family_pass_asgraded"] = {} +for f in FAMILIES: + rec = {} + for m in ["3.6", "3.8"]: + sel = [r for r in ROWS if r["family"] == f and r["model"] == m] + g = [r for r in sel if r["graded"] == 1] + rec[m] = dict(n=len(sel), graded=len(g), passes=sum(1 for r in sel if r["passed"] == 1)) + R["family_pass_asgraded"][f] = rec + +# Everything above is pure computation. The WRITE is guarded so that importing +# this module (test_stats.py imports it for the estimators) cannot clobber +# results.json -- in particular it must not drop the defect_diagnostics block +# that defect_diag.py appends after this script runs. +# Pipeline order is: mmbt_results.py -> defect_diag.py -> make_md.py +if __name__ == "__main__": + with open(OUT_JSON, "w") as fh: + json.dump(R, fh, indent=2, sort_keys=False) + + print("wrote", OUT_JSON) + print("cells:", len(ROWS)) + for pid, ga, gb, d in PAIRS: + q = R["comparisons"][pid]["quality"]["pass_all"] + print(pid, ga, gb, "pass_all %d/%d vs %d/%d d=%+.3f [%.3f,%.3f] p=%.4g" % ( + q["a"]["x"], q["a"]["n"], q["b"]["x"], q["b"]["n"], + q["diff"], q["diff_lo"], q["diff_hi"], q["fisher_p"])) diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/test_stats.py b/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/test_stats.py new file mode 100755 index 00000000..c27d49e2 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/test_stats.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Validation of the pure-python estimators used for the MMBT results tables. + +Each test checks an estimator against its DEFINING EQUATION or an exact +independent enumeration, not against a remembered published digit string. +""" +import itertools +import os +import math +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from mmbt_results import (wilson, fisher_exact_two_sided, newcombe_diff, + _hypergeom_pmf, median, z_diff_mean, Z) + +fails = [] + + +def ck(name, cond, detail=""): + print(("PASS " if cond else "FAIL ") + name + (" " + detail if detail else "")) + if not cond: + fails.append(name) + + +# ---------------------------------------------------------------- Wilson +# DEFINING PROPERTY: the Wilson limits are the two roots p of +# (p_hat - p)^2 = z^2 * p(1-p)/n +# So evaluating the score statistic at lo and at hi must return exactly z. +print("== Wilson: roots of the score equation ==") +for (x, n) in [(81, 121), (36, 72), (95, 120), (37, 81), (5, 11), (0, 11), (11, 11), (1, 100)]: + p, lo, hi = wilson(x, n) + ok = True + detail = [] + for lim in (lo, hi): + if lim <= 0 or lim >= 1: + continue # boundary root at 0 or 1 is degenerate + score = abs(p - lim) / math.sqrt(lim * (1 - lim) / n) + detail.append("%.10f" % score) + if abs(score - Z) > 1e-9: + ok = False + ck("wilson %d/%d score==z at both limits" % (x, n), ok, "scores=" + ",".join(detail)) + +ck("wilson 0/11 lower bound is exactly 0", wilson(0, 11)[1] == 0.0) +ck("wilson 11/11 upper bound is exactly 1", wilson(11, 11)[2] == 1.0) +_viol = [(x, n) for n in range(1, 200) for x in range(n + 1) + if not (wilson(x, n)[1] <= wilson(x, n)[0] <= wilson(x, n)[2])] +ck("wilson interval always brackets p_hat (exactly, incl. x=0 and x=n)", + not _viol, "violations=%d %s" % (len(_viol), _viol[:5])) +ck("wilson never leaves [0,1]", + all(0 <= wilson(x, n)[1] and wilson(x, n)[2] <= 1 + for n in range(1, 60) for x in range(n + 1))) + +# ---------------------------------------------------------------- hypergeometric +print("\n== Hypergeometric pmf ==") +for (r1, r2, c1) in [(4, 4, 4), (121, 72, 117), (10, 12, 7), (81, 120, 60)]: + lo = max(0, c1 - r2) + hi = min(r1, c1) + s = sum(_hypergeom_pmf(x, r1, r2, c1) for x in range(lo, hi + 1)) + ck("pmf sums to exactly 1 (r1=%d r2=%d c1=%d)" % (r1, r2, c1), s == 1, "sum=%s" % s) + +# Independent brute-force: probability that a random c1-subset of r1+r2 labelled +# items contains exactly `a` items from group 1, counted by explicit enumeration. +print("\n== Hypergeometric vs brute-force enumeration ==") +r1, r2, c1 = 5, 6, 4 +items = ["g1"] * r1 + ["g2"] * r2 +counts = {} +tot = 0 +for combo in itertools.combinations(range(r1 + r2), c1): + a = sum(1 for i in combo if items[i] == "g1") + counts[a] = counts.get(a, 0) + 1 + tot += 1 +ok = all(abs(float(_hypergeom_pmf(a, r1, r2, c1)) - counts[a] / tot) < 1e-15 for a in counts) +ck("pmf matches enumeration over C(11,4) subsets", ok) + +# ---------------------------------------------------------------- Fisher +print("\n== Fisher exact ==") +# Fisher's tea-tasting: 4 cups each way, 3 correct. Exact two-sided p = 17/35. +p_tea = fisher_exact_two_sided(3, 1, 1, 3) +ck("tea-tasting [[3,1],[1,3]] == 17/35", abs(p_tea - 17 / 35) < 1e-12, + "got %.10f expected %.10f" % (p_tea, 17 / 35)) +# Perfect separation 4/4 vs 0/4: only the two extreme tables are <= P(obs). +p_sep = fisher_exact_two_sided(4, 0, 0, 4) +ck("[[4,0],[0,4]] == 2/70", abs(p_sep - 2 / 70) < 1e-12, "got %.10f" % p_sep) +ck("identical rows give p == 1.0", abs(fisher_exact_two_sided(5, 5, 5, 5) - 1.0) < 1e-12) +ck("degenerate margin gives p == 1.0", fisher_exact_two_sided(0, 5, 0, 5) == 1.0) +ck("fisher is symmetric under row swap", + all(abs(fisher_exact_two_sided(a, b, c, d) - fisher_exact_two_sided(c, d, a, b)) < 1e-12 + for a, b, c, d in [(8, 3, 2, 9), (81, 40, 36, 36), (1, 20, 7, 4)])) +ck("fisher is symmetric under column swap", + all(abs(fisher_exact_two_sided(a, b, c, d) - fisher_exact_two_sided(b, a, d, c)) < 1e-12 + for a, b, c, d in [(8, 3, 2, 9), (81, 40, 36, 36), (1, 20, 7, 4)])) +ck("fisher p in (0,1] everywhere", + all(0 < fisher_exact_two_sided(a, b, c, d) <= 1.0000000001 + for a in range(6) for b in range(6) for c in range(6) for d in range(6))) + +# Large-sample agreement with the chi-square approximation (no scipy: compare the +# chi-square statistic's normal tail via erfc, which is in the stdlib). +def chisq_p(a, b, c, d): + n = a + b + c + d + num = n * (a * d - b * c) ** 2 + den = (a + b) * (c + d) * (a + c) * (b + d) + if den == 0: + return 1.0 + x2 = num / den + return math.erfc(math.sqrt(x2 / 2.0)) # 2-sided normal tail, chi2_1 = Z^2 + +for (a, b, c, d) in [(400, 200, 350, 250), (81, 40, 36, 36), (300, 300, 260, 340)]: + pf = fisher_exact_two_sided(a, b, c, d) + pc = chisq_p(a, b, c, d) + ck("fisher ~ chi-square at large n [[%d,%d],[%d,%d]]" % (a, b, c, d), + abs(math.log10(pf) - math.log10(pc)) < 0.35, + "fisher=%.5g chisq=%.5g" % (pf, pc)) + +# ---------------------------------------------------------------- Newcombe +print("\n== Newcombe method 10 ==") +# DEFINING PROPERTY: built from the two Wilson intervals via +# lower = (p1-p2) - sqrt((p1-l1)^2 + (u2-p2)^2) +# upper = (p1-p2) + sqrt((u1-p1)^2 + (p2-l2)^2) +for (x1, n1, x2, n2) in [(36, 72, 81, 121), (37, 81, 95, 120), (5, 11, 5, 11), (0, 20, 20, 20)]: + p1, l1, u1 = wilson(x1, n1) + p2, l2, u2 = wilson(x2, n2) + exp_lo = (p1 - p2) - math.sqrt((p1 - l1) ** 2 + (u2 - p2) ** 2) + exp_hi = (p1 - p2) + math.sqrt((u1 - p1) ** 2 + (p2 - l2) ** 2) + d, lo, hi = newcombe_diff(x1, n1, x2, n2) + ck("newcombe %d/%d - %d/%d matches construction" % (x1, n1, x2, n2), + abs(d - (p1 - p2)) < 1e-12 and abs(lo - max(-1.0, exp_lo)) < 1e-12 + and abs(hi - min(1.0, exp_hi)) < 1e-12) + +ck("newcombe brackets the point difference", + all(newcombe_diff(x1, 40, x2, 50)[1] <= newcombe_diff(x1, 40, x2, 50)[0] + <= newcombe_diff(x1, 40, x2, 50)[2] for x1 in range(0, 41, 5) for x2 in range(0, 51, 5))) +ck("newcombe antisymmetric under group swap", + all(abs(newcombe_diff(a, 40, b, 50)[0] + newcombe_diff(b, 50, a, 40)[0]) < 1e-12 + and abs(newcombe_diff(a, 40, b, 50)[1] + newcombe_diff(b, 50, a, 40)[2]) < 1e-12 + for a in (0, 7, 20, 40) for b in (0, 13, 25, 50))) +ck("newcombe stays in [-1,1]", + all(-1 <= newcombe_diff(x1, 30, x2, 30)[1] and newcombe_diff(x1, 30, x2, 30)[2] <= 1 + for x1 in range(31) for x2 in range(31))) +# Newcombe CI excluding 0 should broadly track a significant Fisher p. +agree = 0 +tot = 0 +for x1 in range(0, 41, 2): + for x2 in range(0, 41, 2): + d, lo, hi = newcombe_diff(x1, 40, x2, 40) + pf = fisher_exact_two_sided(x1, 40 - x1, x2, 40 - x2) + tot += 1 + if (lo > 0 or hi < 0) == (pf < 0.05): + agree += 1 +ck("newcombe zero-exclusion agrees with Fisher p<0.05 (>=90% of grid)", + agree / tot >= 0.90, "agreement=%.3f over %d tables" % (agree / tot, tot)) + +# ---------------------------------------------------------------- misc +print("\n== median / z-interval on difference of means ==") +ck("median odd", median([3, 1, 2]) == 2.0) +ck("median even", median([4, 1, 2, 3]) == 2.5) +ck("median empty is None", median([]) is None) +ck("z_diff_mean on identical samples gives d=0", abs(z_diff_mean([1, 0, 1, 0], [1, 0, 1, 0])[0]) < 1e-12) +ck("z_diff_mean too-small sample returns None", z_diff_mean([1], [0])[0] is None) + +print("\n%d checks failed" % len(fails)) +if fails: + print("FAILED:", fails) + sys.exit(1) +print("ALL VALIDATION CHECKS PASSED") diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/validate_fixes.py b/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/validate_fixes.py new file mode 100755 index 00000000..117a98ab --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/validate_fixes.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +"""Post-edit validation for the claims + limitations component (freeze #2). + +Every expectation below is pinned to the freeze-2 dataset +(/home/michael/mmbt-frozen-dataset-v2.csv, 802 cells, 2026-08-16T14:23:09Z) +and the freeze-2 overlay digest. The freeze-1 constants this file used to +carry are now in the stale list.""" +import csv, json, glob, sys, collections + +CLAIMS = "/home/michael/pr-staging/claims-additions.yaml" +LIMITS = "/home/michael/pr-staging/known-limitations-additions.md" +failures = [] + +def check(name, cond, detail=""): + print(("PASS " if cond else "FAIL ") + name + ((" | " + detail) if detail else "")) + if not cond: + failures.append(name) + +# ---- 1. PyYAML parse ---- +import yaml +with open(CLAIMS, encoding="utf-8") as f: + doc = yaml.safe_load(f) +check("yaml parses", isinstance(doc, dict)) +check("claims count", len(doc.get("claims", [])) == 7, str(len(doc.get("claims", [])))) +check("retracted count", len(doc.get("retracted", [])) == 5, str(len(doc.get("retracted", [])))) +ids = [c["id"] for c in doc["claims"]] + [r["id"] for r in doc["retracted"]] +check("ids unique", len(ids) == len(set(ids))) + +cl = {c["id"]: c for c in doc["claims"]} +rt = {r["id"]: r for r in doc["retracted"]} +claims_txt = open(CLAIMS, encoding="utf-8").read() +limits_txt = open(LIMITS, encoding="utf-8").read() + +# ---- 2. stale numbers gone (early drafts AND superseded freeze-1 figures) ---- +stale = ["91/114", "91/105", "90/107", "79.8%", "86.7%", "84.1%", "108/120", + "44/61", "72.1%", "~18pp", "59/59", "19 to 22", "3.8 9-12", + "12/12 exact agreement", "sign of the\n difference flips", + # freeze-1 figures superseded by freeze #2 + "39/46", "34/41", "47/61", "40/54", "36/46", "36/43", "104/108", + "108/110", "46/72", "61/81", "58/58", "16 to 37", "18 to 21", + "34/37", "55 of the 58", "148 of 148", "86 of 89", + "138 measurements", "35 gate-invalidated", "64 cells fail", + "57 of those 64", "EXISTENCE ONLY", "3/11", "11-cell", + "26 of 746", "46 labelled", "3.8 5-11", "0/11 on p1_bugfix"] +# NOT in the stale list: "17/21 against 4/37" is the historical formerly_claimed +# quote inside the p3_pm retraction (retractions preserve the original wording), +# and "n=11" is the current effort-ladder xhigh subset size. +for s in stale: + check("stale gone: %r" % s, s not in claims_txt) +for s in ["5-12", "3.8 5-11", "(15/291", "10/265", "(0/64", "of 683", + "all 273", "59 cells", "110 vs 72", "120 vs 81", "121 vs 72", + "of those 11", "11-cell"]: + check("stale gone in limits: %r" % s, s not in limits_txt) + +# ---- 3. recompute ground truth from frozen CSV + shipped overlay ---- +rows = list(csv.DictReader(open("/home/michael/mmbt-frozen-dataset-v2.csv"))) +frozen = {r["cell"]: r for r in rows} +graded = [r for r in rows if r["graded"] == "1"] +check("frozen cells 802", len(rows) == 802, str(len(rows))) +check("frozen graded 733", len(graded) == 733, str(len(graded))) +ov = {} +for p in glob.glob("/home/michael/pr-staging/overlay/*/logs/*/grade.corrected.json"): + j = json.load(open(p)) + ov[j["cell"]] = j +man = json.load(open("/home/michael/pr-staging/overlay/manifest.json")) +print("overlay digest now:", man["overlay_digest"]) +check("overlay digest unchanged since analysis", + man["overlay_digest"] == "e332cd2c78ad94fe264aed7d31e6c64f5273cd9598f889dc1a8fac539e971351") +check("overlay stamped with freeze #2", + man.get("frozen_dataset_stamp") == "2026-08-16T14:23:09Z" + and man.get("frozen_dataset", "").endswith("-v2.csv")) +check("post-freeze ledger empty (static corpus)", + len(man.get("post_freeze_divergence", [1])) == 0) + +def corr(r): # treatment A: D1 gate-invalidated counted as non-failure + j = ov.get(r["cell"]) + if j is None: + return r["passed"] == "1" + if "corrected_verdict" in j: + return j["corrected_verdict"] == "PASS" + return bool(j.get("gate_invalidated")) +def d1cell(r): + j = ov.get(r["cell"]) + return bool(j and "D1" in j["defects_applied"]) + +def rates(model, mode, regime, excl_triage=False): + rs = [r for r in graded if r["model"] == model and r["quant"] == "UD-Q4_K_XL" + and r["mode"] == mode and r["regime"] == regime] + if excl_triage: + rs = [r for r in rs if r["family"] != "p2_triage"] + a = (sum(1 for r in rs if corr(r)), len(rs)) + rb = [r for r in rs if not d1cell(r)] + b = (sum(1 for r in rb if corr(r)), len(rb)) + raw = (sum(1 for r in rs if r["passed"] == "1"), len(rs)) + return raw, a, b + +exp = { + ("3.6", "no-think", "T0.3/p0.8/pp0", False): ((81, 114), (98, 114), (96, 112)), + ("3.8", "no-think", "T0.3/p0.8/pp0", False): ((51, 65), (56, 65), (51, 60)), + ("3.6", "no-think", "T0.3/p0.8/pp0", True): ((81, 105), (89, 105), (87, 103)), + ("3.8", "no-think", "T0.3/p0.8/pp0", True): ((44, 58), (49, 58), (44, 53)), + ("3.6", "think", "T0.3/p0.8/pp0", False): ((67, 107), (99, 107), (85, 93)), + ("3.8", "think", "T0.3/p0.8/pp0", False): ((33, 48), (43, 48), (36, 41)), + ("3.6", "think", "T0.3/p0.8/pp0", True): ((67, 98), (90, 98), (76, 84)), + ("3.8", "think", "T0.3/p0.8/pp0", True): ((29, 44), (39, 44), (32, 37)), + ("3.6", "no-think", "T1/p0.95/pp0", False): ((95, 120), (110, 120), (106, 116)), + ("3.8", "no-think", "T0.7/p0.8/pp1.5", False): ((48, 79), (61, 79), (51, 69)), + ("3.6", "think", "T1/p0.95/pp0", False): ((105, 120), (116, 120), (113, 117)), + ("3.8", "think", "T1/p0.95/pp0", False): ((54, 72), (72, 72), (55, 55)), +} +for (m, mode, reg, ex), want in exp.items(): + got = rates(m, mode, reg, ex) + check("rates %s %s %s excl=%s" % (m, mode, reg, ex), got == want, str(got)) + +# numbers present in claim text +eq = cl["bench.qwen38-vs-qwen36.equivalent-conditional-on-delivery-matched-sampler"] +for frag in ["98/114 (86.0%)", "56/65 (86.2%)", "89/105 (84.8%)", "49/58 (84.5%)", + "99/107 (92.5%)", "43/48 (89.6%)", "90/98 (91.8%)", "39/44 (88.6%)", + "62 D1 gate-invalidated"]: + check("equivalence text has %s" % frag, frag in eq["text"]) +check("equivalence: no sign-flip framing", "sign" in eq["text"] and "flips" not in eq["text"]) +cavs = "\n".join(eq["caveats"]) +for frag in ["110/120 (91.7%)", "61/79 (77.2%)", "~15pp", "106/116", "51/69", + "96/112 (85.7%)", "51/60 (85.0%)", "85/93 (91.4%)", "36/41 (87.8%)", + "81/114", "51/65", "62 verdicts counter-dependent"]: + check("equivalence caveats have %s" % frag, frag in cavs) + +# ---- 4. D1 claim vs census/overlay ---- +d1 = [r for r in graded if d1cell(r)] +check("D1 total 62", len(d1) == 62, str(len(d1))) +byfam = collections.Counter(r["family"] for r in d1) +bymod = collections.Counter(r["model"] for r in d1) +check("D1 by family 24/23/15", + byfam == collections.Counter({"p3_doc": 24, "p3_writing": 23, "p3_business": 15}), str(byfam)) +check("D1 by model 23/39", bymod == collections.Counter({"3.6": 23, "3.8": 39}), str(bymod)) +wg = cl["bench.grader.word-gate-counter-disagreement"] +for frag in ["71 cells fail", "p3_doc 26, p3_writing 24, p3_business 21", "3.8 47 and 3.6 24", + "62 of those 71", "p3_doc 24, p3_writing 23, p3_business 15", "3.8 39 and 3.6 23", + "gate-invalidated", "152 of 152", "93 of 95"]: + check("word-gate text has %r" % frag[:40], frag in wg["text"]) +wcavs = "\n".join(wg["caveats"]) +for frag in ["all 147 measurements", "LC_ALL=C.UTF-8", + "39 gate-invalidated single-deliverable cells (24 p3_doc, 15 p3_business)", + "1 to 54 words"]: + check("word-gate caveats have %r" % frag[:40], frag in wcavs) + +# ---- 5. triage retraction vs frozen ---- +tri = [r for r in graded if r["family"] == "p2_triage"] +t36 = [r for r in tri if r["model"] == "3.6"]; t38 = [r for r in tri if r["model"] == "3.8"] +check("triage frozen 64", len(tri) == 64, str(len(tri))) +check("triage 3.6 17->38", (sum(1 for r in t36 if r["passed"] == "1"), + sum(1 for r in t36 if corr(r)), len(t36)) == (17, 38, 38)) +check("triage 3.8 23->26", (sum(1 for r in t38 if r["passed"] == "1"), + sum(1 for r in t38 if corr(r)), len(t38)) == (23, 26, 26)) +check("triage 3.8 includes 1 Q8 cell", + sum(1 for r in t38 if r["quant"] == "Q8_0") == 1) +check("triage flips 24", sum(1 for r in tri if corr(r) and r["passed"] != "1") == 24) +accs = {ov[r["cell"]]["evidence"]["D3"]["category_accuracy"] for r in tri} +check("triage accuracy 0.867 all 64", accs == {0.867}, str(accs)) +tr = rt["bench.p2_triage.family-result"] +for frag in ["64/64", "All 64 frozen graded", "17 to 38", "23 to 26", "24 verdict flips", + "admitted as graded at freeze #2"]: + check("triage retraction has %r" % frag[:40], frag in tr["text"] + tr["reason"]) + +# ---- 6. p3_pm retraction vs overlay ---- +pm = [r for r in graded if r["family"] == "p3_pm"] +p36 = [r for r in pm if r["model"] == "3.6"]; p38 = [r for r in pm if r["model"] == "3.8"] +check("p3_pm 3.6 4->35/38", (sum(1 for r in p36 if r["passed"] == "1"), + sum(1 for r in p36 if corr(r)), len(p36)) == (4, 35, 38)) +check("p3_pm 3.8 21->26/26", (sum(1 for r in p38 if r["passed"] == "1"), + sum(1 for r in p38 if corr(r)), len(p38)) == (21, 26, 26)) +d2 = [r for r in graded if ov.get(r["cell"]) and "D2" in ov[r["cell"]]["defects_applied"]] +check("D2 touched 39", len(d2) == 39, str(len(d2))) +check("D2 flips 36", sum(1 for r in d2 if corr(r) != (r["passed"] == "1")) == 36) +pmr = rt["bench.p3_pm.qwen38-outscores-qwen36"] +for frag in ["61 of the 64 graded p3_pm cells pass", "35/38", "26/26", + "credits R3 on 39 cells", "flips 36 verdicts", "25/64", "4/38", "21/26"]: + check("p3_pm retraction has %r" % frag[:40], frag in pmr["text"] + pmr["reason"]) + +# ---- 7. win counts for nine-three retraction ---- +def wins(mode, use_corr): + out = {"3.6": 0, "3.8": 0, "tie": 0, "skip": 0} + for fam in sorted({r["family"] for r in graded}): + res = {} + for m in ("3.6", "3.8"): + rs = [r for r in graded if r["model"] == m and r["quant"] == "UD-Q4_K_XL" + and r["mode"] == mode and r["regime"] == "T0.3/p0.8/pp0" and r["family"] == fam] + res[m] = None if not rs else (sum(1 for r in rs if (corr(r) if use_corr else r["passed"] == "1")) / len(rs)) + if res["3.6"] is None or res["3.8"] is None: + out["skip"] += 1 + continue + out["3.6" if res["3.6"] > res["3.8"] else ("3.8" if res["3.8"] > res["3.6"] else "tie")] += 1 + return out +check("no-think raw 4/5/3", wins("no-think", False) == {"3.6": 4, "3.8": 5, "tie": 3, "skip": 0}, + str(wins("no-think", False))) +check("no-think corr 4/3/5", wins("no-think", True) == {"3.6": 4, "3.8": 3, "tie": 5, "skip": 0}, + str(wins("no-think", True))) +check("think raw 4/5/3", wins("think", False) == {"3.6": 4, "3.8": 5, "tie": 3, "skip": 0}, + str(wins("think", False))) +check("think corr 4/2/6", wins("think", True) == {"3.6": 4, "3.8": 2, "tie": 6, "skip": 0}, + str(wins("think", True))) +nn = rt["bench.family-paired.nine-three"] +for frag in ["4 (3.6) / 5 (3.8) / 3 tied", "4 / 3 / 5", "4 / 5 / 3 raw", "4 / 2 / 6", + "all twelve computable at freeze #2"]: + check("nine-three has %r" % frag, frag in nn["reason"]) + +# ---- 8. replicate depth + Q8 rate claim + cross-file agreement ---- +cnt = collections.Counter() +for r in rows: + if r["quant"] == "UD-Q4_K_XL": + cnt[(r["model"], r["mode"], r["regime"], r["family"])] += 1 +def rng(model, mode, reg): + v = [n for (m, mo, rg, f), n in cnt.items() if (m, mo, rg) == (model, mode, reg)] + return min(v), max(v) +check("3.8 nothink matched depth 7-13", rng("3.8", "no-think", "T0.3/p0.8/pp0") == (7, 13)) +check("claims says 3.8 7-13", "3.8 7-13" in claims_txt) +check("limits says 7-13", "Qwen3.8 at the same sampler 7-13" in limits_txt) +check("both say 9-19", "9-19" in claims_txt and "9-19" in limits_txt) +check("both say 4-6 think", "3.8 4-6" in claims_txt and "4-6" in limits_txt) + +# Q8 arm: rate-claim ground truth +q8 = [r for r in rows if r["quant"] == "Q8_0"] +q8g = [r for r in q8 if r["graded"] == "1"] +check("q8 19 cells / 12 families", (len(q8), len({r["family"] for r in q8})) == (19, 12)) +check("q8 loop 6/19", sum(1 for r in q8 if r["looped_freq30"] == "1") == 6) +check("q8 graded 8 = 5 PASS + 3 FAIL", + (len(q8g), sum(1 for r in q8g if r["passed"] == "1")) == (8, 5)) +q8c = cl["bench.qwen38.q8-reproduces-nothink-loop"] +for frag in ["6 of 19", "31.6%", "[15.4%, 54.0%]", "110, 109, 81, 80 and 71", + "provisional RATE claim", "Fisher p = 1.0"]: + check("q8 claim text has %r" % frag, frag in q8c["text"]) +q8cavs = "\n".join(q8c["caveats"]) +for frag in ["5 PASS, 3 FAIL", "[30.6%, 86.3%]", "p3_doc_qwen38q8-nothink-matched_v2", + "139 iterations", "~228k", "UNSCORED", "no matched Qwen3.6 Q8_0 arm"]: + check("q8 claim caveats have %r" % frag, frag in q8cavs) + +# quarantined rewrite-loop cell must NOT be a frozen row +check("quarantined q8 cell not in CSV", + "p3_doc_qwen38q8-nothink-matched_v2" not in frozen) + +# shared constants across both files +for frag in ["802 cells", "2026-08-16T14:23:09Z"]: + check("both files: %r" % frag, frag in claims_txt and frag in limits_txt) +for a, b in [("5 PASS, 3 FAIL", "5 PASS, 3 FAIL"), ("110, 109, 81, 80", "110, 109, 81, 80")]: + check("q8 grades/runs note both files", a in claims_txt and b in limits_txt) + +print() +print("FAILURES:", len(failures)) +for f in failures: + print(" -", f) +sys.exit(1 if failures else 0) diff --git a/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/verify_overlay.py b/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/verify_overlay.py new file mode 100755 index 00000000..a0c600d4 --- /dev/null +++ b/benchmarks/qwen36-vs-qwen38-27b-2026-08/tooling/verify_overlay.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Verification harness: non-mutation, idempotence, write-guard, invariant.""" +import csv, hashlib, json, subprocess, sys, shutil +from pathlib import Path + +SCRIPT = "/home/michael/pr-staging/tooling/apply_grade_corrections.py" +OUT = Path("/home/michael/pr-staging/overlay") +ROOT = Path("/home/michael") + +def sha(p): + h = hashlib.sha256() + with open(p, "rb") as f: + for c in iter(lambda: f.read(1 << 20), b""): + h.update(c) + return h.hexdigest() + +rows = list(csv.DictReader(open("/home/michael/mmbt-frozen-dataset-v2.csv"))) + +# ---- snapshot every protected file BEFORE ---- +protected = [] +for r in rows: + g = ROOT / r["repo"] / "logs" / r["cell"] / "grade.json" + if g.is_file(): + protected.append(g) +for repo in {r["repo"] for r in rows}: + for sub in ("tooling/tasks", "tooling/graders/ground_truth"): + d = ROOT / repo / sub + if d.is_dir(): + protected.extend(sorted(p for p in d.rglob("*") if p.is_file())) +before = {str(p): (sha(p), p.stat().st_mtime_ns) for p in protected} +print(f"protected files snapshotted: {len(before)}") + +# ---- run 1 ---- +if OUT.exists(): + shutil.rmtree(OUT) +r1 = subprocess.run([sys.executable, SCRIPT], capture_output=True, text=True) +print("run1 rc:", r1.returncode) +print("run1 out:", r1.stdout.strip()[:300]) +if r1.stderr.strip(): + print("run1 err:", r1.stderr.strip()[:600]) +files1 = {str(p.relative_to(OUT)): sha(p) for p in OUT.rglob("*.json") if p.name != "manifest.json"} +m1 = json.loads((OUT / "manifest.json").read_text()) + +# ---- run 2 (idempotence) ---- +r2 = subprocess.run([sys.executable, SCRIPT], capture_output=True, text=True) +print("run2 rc:", r2.returncode) +files2 = {str(p.relative_to(OUT)): sha(p) for p in OUT.rglob("*.json") if p.name != "manifest.json"} +m2 = json.loads((OUT / "manifest.json").read_text()) + +print() +print("=== IDEMPOTENCE ===") +print(" per-cell files run1:", len(files1), " run2:", len(files2)) +print(" identical file set :", set(files1) == set(files2)) +diffs = [k for k in files1 if files1.get(k) != files2.get(k)] +print(" byte-differing files:", len(diffs), diffs[:5]) +print(" overlay_digest run1 :", m1["overlay_digest"]) +print(" overlay_digest run2 :", m2["overlay_digest"]) +print(" digest stable :", m1["overlay_digest"] == m2["overlay_digest"]) +print(" generated_at differs:", m1["generated_at"] != m2["generated_at"], "(expected True)") + +# ---- non-mutation ---- +print() +print("=== NON-MUTATION OF RUN EVIDENCE ===") +changed = [p for p, v in before.items() if (sha(Path(p)), Path(p).stat().st_mtime_ns) != v] +print(f" protected files checked : {len(before)}") +print(f" protected files changed : {len(changed)}") +for c in changed[:10]: + print(" CHANGED:", c) +stray = [str(p) for p in ROOT.glob("mmbt-*/logs/*/grade.corrected.json")] +print(f" stray grade.corrected.json inside checkouts : {len(stray)}") + +# ---- write guard ---- +print() +print("=== WRITE GUARD ===") +sys.path.insert(0, "/home/michael/pr-staging/tooling") +import apply_grade_corrections as A +targets = [ + ROOT / "mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v2/grade.json", + ROOT / "mmbt-q36-card/tooling/tasks/task_triage.md", + ROOT / "mmbt-q36-card/tooling/graders/ground_truth/phase2_triage.json", + ROOT / "mmbt-q36-card/logs/p3_pm_qwen36-nothink-card_v2/receipt.json", +] +for t in targets: + try: + A.guarded_write(t, "SHOULD NEVER BE WRITTEN") + print(" !! NOT REFUSED:", t) + except A.ProtectedPathError as e: + print(" refused OK:", t.relative_to(ROOT), "->", str(e).split(":")[-1].strip()[:60]) +# and confirm a legitimate target is allowed +ok = OUT / "_guard_probe.json" +A.guarded_write(ok, "{}\n") +print(" allowed OK:", ok.name, ok.is_file()) +ok.unlink() + +print() +print("=== MANIFEST INVARIANT ===") +print(" invariant_checked :", m1["invariant_checked"]) +print(" invariant_violations:", len(m1["invariant_violations"])) +print(" post_freeze_divergence:", len(m1["post_freeze_divergence"])) +print(" frozen dataset sha256:", m1["frozen_dataset_sha256"]) diff --git a/claims.yaml b/claims.yaml index 7060027a..9a98f23d 100644 --- a/claims.yaml +++ b/claims.yaml @@ -19,7 +19,7 @@ # entry once it's been published. schema_version: 0.1 -last_updated: "2026-08-02" +last_updated: "2026-08-16" claims: @@ -598,6 +598,196 @@ claims: - "Small modality-specific N; older local entries have not all received the same strict overlay." - "Per-PR factual accuracy is not exhaustively graded; the 75-PR result is artifact completeness and audit substance." + + # ─── Qwen3.6-27B vs Qwen3.8-27B, UD-Q4_K_XL GGUF (benchmarks/qwen36-vs-qwen38-27b-2026-08) ─── + + - id: bench.qwen38-vs-qwen36.nothink-delivery-regression + text: > + In no-think mode, Qwen3.8-27B-UD-Q4_K_XL produces a gradeable deliverable + markedly less often than Qwen3.6-27B-UD-Q4_K_XL on the same 12-family + MMBT suite, same rig, same engine, same quant. With each model at its own + vendor sampling point, 3.6 delivers 120/120 cells (100%) at + T1/p0.95/pp0 and 3.8 delivers 79/96 (82.3%) at T0.7/p0.8/pp1.5; 3.6's + per-family delivery rate is greater than or equal to 3.8's in 12 of 12 + families (5 strict wins, 7 ties). At the matched off-spec sampler + T0.3/p0.8/pp0 the direction is the same and larger: 3.6 114/121 (94.2%) + vs 3.8 65/95 (68.4%). The regression is concentrated, not uniform — at + the matched sampler 3.8 delivers 1/12 on p1_bugfix and 2/13 on + p1_testwrite while matching 3.6 at 100% delivery on all four phase-2 + families (7/7 each). + status: provisional + scope: > + Single rig (Tower2), llama.cpp GGUF serving, UD-Q4_K_XL only, no-think + mode, 12-family MMBT suite, seed=42 throughout. + evidence: "benchmarks/qwen36-vs-qwen38-27b-2026-08/data/mmbt-frozen-dataset-v2.csv (802 cells); benchmarks/qwen36-vs-qwen38-27b-2026-08/findings.md § Delivery reliability" + caveats: + - "Delivery is not quality. A delivered cell can still be wrong; this claim is only about whether a gradeable artifact exists." + - "Replicate depth is unbalanced between the arms (3.6 no-think T0.3 carries 9-19 replicates per family, 3.8 7-13; 3.6 think T0.3 carries 9, 3.8 4-6). Per-family rates therefore rest on different denominators, which are printed per family in findings.md rather than pooled silently." + - "Non-delivering cells are a mixture of operator-terminated runs and runs that ended non-terminal. Do NOT convert this into an 'abort rate' — see [bench.qwen38.nothink-abort-rate] in the retracted block for why the operator label cannot carry that weight." + - "One quant, one engine. Whether the regression survives at FP8/BF16, under vLLM, or at other context settings is uncharacterized." + promote_to_strong_when: + - "An independent reproducer on different hardware publishes the same direction at both models' vendor sampling points" + - "Replicate depth is balanced across arms and per-family rates are recomputed on equal denominators" + - "Non-delivery is classified by a deterministic, published rule rather than an operator kill decision" + + - id: bench.qwen38.q8-reproduces-nothink-loop + text: > + The identical-call-loop failure shape seen in Qwen3.8-27B-UD-Q4_K_XL + no-think also occurs at Q8_0, at a provisional rate whose confidence + interval excludes zero. In the freeze-2 Q8_0 control arm (19 cells over + all 12 families, no-think, matched T0.3/p0.8/pp0), 6 of 19 cells trip the + mechanical loop detector (looped_freq30) — 31.6%, Wilson 95% CI + [15.4%, 54.0%] — with maximum identical-call runs of 110, 109, 81, 80 + and 71 consecutive calls (the sixth trips the frequency detector at + max_freq 121 without a long run). Because the interval excludes zero, + this is now a provisional RATE claim rather than the existence-only + observation it was at freeze #1: the loop is not an artifact of the + UD-Q4_K_XL quantization. Against 3.8-Q4 at the same sampler over the + same 12 families the loop rate is statistically indistinguishable + (Q4 29/95, 30.5%, vs Q8_0 6/19, 31.6%; Fisher p = 1.0) — consistent + with quantization playing no role in the loop, though the Q8_0 interval + is wide. + status: provisional + scope: > + 19 cells (2 replicates on the 7 phase-1/phase-2 families, 1 on the 5 + phase-3 families), Qwen3.8-27B Q8_0, no-think, T0.3/p0.8/pp0, seed=42, + single rig. + evidence: "benchmarks/qwen36-vs-qwen38-27b-2026-08/data/mmbt-frozen-dataset-v2.csv (rows where quant=Q8_0)" + caveats: + - "The rate is provisional. n=19 with 1-2 replicates per family; the Wilson interval [15.4%, 54.0%] bounds the rate away from zero and pins down little else." + - "Quality stays near-uninformative: 8 of 19 cells are graded at freeze #2 (5 PASS, 3 FAIL; graded-only rate 62.5%, Wilson [30.6%, 86.3%]). Of the three FAILs, two (p3_business, p3_doc) are over-length by BOTH word counters — genuine length overruns D1 does not rescue — and one (p3_pm) is the D2 R3 keyword literalism, corrected to PASS by the overlay. No quality verdict is drawn from n=8, and the vs-Q4 graded-only contrast (78.5% vs 62.5%, Fisher p = 0.38) is underpowered by the PR's own power screen." + - "UNSCORED qualitative evidence, excluded from every rate above: p3_doc_qwen38q8-nothink-matched_v2 was quarantined in-flight at freeze #2 (it is not a row in the frozen CSV) after 139 iterations rewriting brief.md with its context grown to ~228k tokens — a REWRITE-loop subclass distinct from the identical-call loop the detector counts. Disclosed because a second loop shape at Q8_0 is qualitatively relevant to the quantization question; it carries no statistical weight anywhere in this PR." + - "There is no matched Qwen3.6 Q8_0 arm, so this is not a quant A/B and cannot attribute the delivery regression to quantization or exonerate it." + promote_to_strong_when: + - "A Q8_0 arm with N>=5 per family, graded, is published alongside a matched UD-Q4_K_XL arm for BOTH models so loop and pass rates can be compared with useful power" + + - id: bench.qwen38-vs-qwen36.equivalent-conditional-on-delivery-matched-sampler + text: > + Conditional on having produced a gradeable deliverable, and after the + three grader corrections published with this PR (D1 word-gate counter + disagreement, D2 p3_pm keyword literalism, D3 p2_triage brief/ground-truth + contradiction — applied from the shipped correction overlay, with the + 62 D1 gate-invalidated cells counted as non-failures), the two models + score close to each other at the matched sampler T0.3/p0.8/pp0. + No-think: 3.6 98/114 (86.0%) vs 3.8 56/65 (86.2%); excluding the + p2_triage family entirely, 3.6 89/105 (84.8%) vs 3.8 49/58 (84.5%). + Think: 3.6 99/107 (92.5%) vs 3.8 43/48 (89.6%); excluding p2_triage, + 3.6 90/98 (91.8%) vs 3.8 39/44 (88.6%). All four contrasts sit within + about three percentage points — 3.6 nominally ahead in three, behind by + 0.2 pp in the fourth — and on 3.8-side denominators of 44-65 cells a + one-to-two-cell swing reverses any of them: no consistent sign, and a + result too small to call in either direction. + status: provisional + scope: > + Matched off-spec sampler T0.3/p0.8/pp0 only, UD-Q4_K_XL, corrected + overlay applied non-destructively, conditional on delivery. + evidence: "benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/manifest.json; benchmarks/qwen36-vs-qwen38-27b-2026-08/findings.md § Quality conditional on delivery" + caveats: + - "THE CONDITIONING IS THE WHOLE PROBLEM. Delivery is exactly where the two models differ, so conditioning on delivery conditions on a non-random subset: 3.8's 65 delivered cells at this sampler are its surviving runs, not a random sample of its attempts. This is survivorship, and it means the claim describes graded work only, never expected outcome per attempt." + - "Equivalence does NOT extend to the vendor-point comparison. There, corrected pass-given-delivery is 3.6 110/120 (91.7%) vs 3.8 61/79 (77.2%) — an ~15pp gap that the grader corrections do not explain and this PR does not resolve; excluding the D1 gate-invalidated cells instead of counting them widens it (3.6 106/116, 91.4%, vs 3.8 51/69, 73.9%)." + - "The corrections change the numbers substantially (raw pass-given-delivery at matched no-think is 3.6 81/114 and 3.8 51/65), so anyone quoting these figures must state that they are overlay-corrected. Raw grades remain published and unmodified." + - "Counting the D1 cells as non-failures is a treatment choice, not a measurement: the overlay marks those 62 verdicts counter-dependent — not a valid FAIL, not a verified PASS. Excluding them from the denominators instead gives no-think 3.6 96/112 (85.7%) vs 3.8 51/60 (85.0%) and think 3.6 85/93 (91.4%) vs 3.8 36/41 (87.8%) — same within-noise margins, so the reading does not hinge on the treatment." + - "D1 is a disagreement between two defensible word counters, not a repair of a wrong one. See [bench.grader.word-gate-counter-disagreement]." + + - id: bench.grader.word-gate-counter-disagreement + text: > + The four length-gated phase-3 graders (business_memo, doc_synthesis, + project_mgmt, writing_editing) count words as + len(re.findall(r"\b\w+\b", text)), which splits hyphenated compounds and + comma-grouped numerals into multiple words. Both models budget with shell + `wc -w`: a literal `wc -w` call appears in 152 of 152 Qwen3.6 and 93 of 95 + Qwen3.8 length-gated phase-3 transcripts. Across the frozen corpus, + 71 cells fail on length alone (every non-length criterion satisfied) — + by family, p3_doc 26, p3_writing 24, p3_business 21; by model, 3.8 47 + and 3.6 24. 62 of those 71 have every gated deliverable at or under its + ceiling by `wc -w` (by family, p3_doc 24, p3_writing 23, p3_business 15; + by model, 3.8 39 and 3.6 23); the overlay marks exactly these 62 as + gate-invalidated — not a valid FAIL, not a verified PASS — and the other + 9, over by both counters (the Q8_0 control's two length FAILs among + them), keep their FAILs. Neither counter is + ground truth: `wc -w` counts markdown bullets + and table pipes as words, the regex counts "mobile-responsive" as two. + The finding is that two reasonable counters disagree by more than the + margin at a hard threshold, not that one of them is wrong. + status: provisional + scope: > + 4 length-gated phase-3 graders, both models, all sampler arms in the + frozen dataset. + evidence: "tooling/graders/phase3_{business_memo,doc_synthesis,project_mgmt,writing_editing}_grade.py; benchmarks/qwen36-vs-qwen38-27b-2026-08/overlay/manifest.json" + caveats: + - "The overlay's recount runs shell GNU `wc -w` pinned to LC_ALL=C.UTF-8 (the sandbox image's locale) and records a pure-Python recount beside it for every measured deliverable; the two agree on all 147 measurements in the shipped overlay. It is not otherwise validated against other wc implementations or locales." + - "Overshoot on the 39 gate-invalidated single-deliverable cells (24 p3_doc, 15 p3_business) ranges from 1 to 54 words by the grader's own counter, so this is not uniformly a hairline effect." + - "Correcting the counter does not make a length gate a good measure of writing discipline. A model that lands at exactly 700 by the counter it was told to use has satisfied the instruction, not demonstrated concision." + promote_to_strong_when: + - "The task briefs specify the counter explicitly and a re-run under the specified counter reproduces the gate behavior" + + - id: bench.qwen38.thinking-restores-delivery + text: > + Enabling thinking mode removes most of Qwen3.8-27B's no-think delivery + regression. At the matched sampler T0.3/p0.8/pp0, 3.8 delivers 48/51 + (94.1%) in think mode against 65/95 (68.4%) in no-think; at T1/p0.95/pp0 + it delivers 72/72 (100%) in think mode against 79/96 (82.3%) no-think at + its own vendor point. Qwen3.6 delivers at or near ceiling in think mode at + both samplers (107/108 and 120/120), so thinking closes a gap that only + one of the two models had. + status: provisional + scope: > + UD-Q4_K_XL, think vs no-think within Qwen3.8 at T0.3/p0.8/pp0 and + T1/p0.95/pp0; single rig. + evidence: "benchmarks/qwen36-vs-qwen38-27b-2026-08/data/mmbt-frozen-dataset-v2.csv; benchmarks/qwen36-vs-qwen38-27b-2026-08/findings.md § Delivery reliability" + caveats: + - "The think-vs-no-think contrast at T1/p0.95/pp0 crosses a sampler boundary for 3.8 (its no-think vendor point is T0.7/p0.8/pp1.5, its think cells at T0.7 do not exist), so the cleanest within-model contrast is the matched T0.3 pair: 48/51 vs 65/95." + - "Every Qwen3.8 think cell carries a reasoning_effort setting; the 51 matched-sampler think cells are a mixture of 14 low, 24 medium and 13 xhigh. This arm is a mixture, not a single configuration." + - "Thinking is not free — see [bench.qwen38.reasoning-effort-cost-ladder] for the token cost, which is substantial and is not netted against the delivery gain anywhere in this PR." + + - id: bench.qwen38-vs-qwen36.thinking-quality-head-to-head.held + text: > + A cross-model thinking-mode quality comparison is held. The only sampler + at which both models have think cells and could be compared directly is + T1/p0.95/pp0, which is Qwen3.6's vendor sampling point and not Qwen3.8's; + the corpus contains zero Qwen3.8 think cells at Qwen3.8's own vendor point + (T0.7/p0.8/pp1.5). The two think arms are also not the same kind of + object: every Qwen3.8 think cell carries a reasoning_effort setting and is + therefore an effort mixture, while Qwen3.6's shipped chat template + contains no reasoning_effort variable at all, so its think arm is a single + configuration. The underlying numbers are published (corrected + pass-given-delivery at T1/p0.95/pp0, with D1 gate-invalidated cells + counted as non-failures: 3.6 116/120, 3.8 72/72) but no + conclusion is drawn from them. + status: held + scope: think mode, cross-model, UD-Q4_K_XL + evidence: "benchmarks/qwen36-vs-qwen38-27b-2026-08/data/mmbt-frozen-dataset-v2.csv; benchmarks/qwen36-vs-qwen38-27b-2026-08/findings.md § What we did not run" + + - id: bench.qwen38.reasoning-effort-cost-ladder + text: > + Qwen3.8-27B's reasoning_effort ladder is not monotone in cost. Pooled over + the families that carry all three effort levels, median completion tokens + at T0.3/p0.8/pp0 are low 17,246 (n=13), medium 10,898 (n=22) and + xhigh 29,744 (n=11) across 11 families — low costs roughly 58% more median + tokens than medium, and exceeds medium in 9 of those 11 families. The + mechanism is visible in the shipped Qwen3.8 chat template: it initializes + reasoning_instructions to the empty string, sets a "think carefully, + validate key assumptions, consider plausible alternatives" instruction for + xhigh and a "keep your thinking brief and focused, moving directly to the + conclusion" instruction for low, and sets NOTHING for medium. "Medium" is + the un-instructed baseline; "low" is the baseline plus an injected brevity + instruction that does not, in these runs, buy brevity. xhigh is the + template's default when reasoning_effort is unset, and "high" is aliased + to xhigh. + status: provisional + scope: > + Qwen3.8-27B-UD-Q4_K_XL think mode only, 123 effort-labelled cells + (26 low / 36 medium / 61 xhigh), 1-4 replicates per family per level. + evidence: "benchmarks/qwen36-vs-qwen38-27b-2026-08/data/mmbt-frozen-dataset-v2.csv (effort column); Qwen3.8-27B chat template, reasoning_instructions block; tooling/harness.py --reasoning-effort (wired via chat_template_kwargs)" + caveats: + - "THE NON-MONOTONICITY DOES NOT REPRODUCE AT THE SECOND SAMPLER. At T1/p0.95/pp0 the pooled medians are low 14,514 and medium 14,358 across 12 families — within 1% — and low exceeds medium in only 7 of 12 families. The effect is established at T0.3/p0.8/pp0 and is absent at T1/p0.95/pp0." + - "Replicate depth is 1-2 cells per family per effort level at T0.3 and 1 per family at T1 for low/medium. Family-matched medians were used precisely because pooled means are dominated by a handful of runaway cells, but the underlying n is thin and this claim should not be quoted as a measured cost multiplier." + - "The template reading is a source-level mechanism, not a controlled ablation. No run in this corpus toggles the injected instruction text while holding reasoning_effort fixed, so 'low costs more because of the brevity instruction' is the most parsimonious reading of the template, not a demonstrated cause." + - "xhigh is monotonically the most expensive level at both samplers (median 29,744 and 38,506 tokens), which is the one part of the ladder that behaves as its name suggests." + promote_to_strong_when: + - "An ablation holds reasoning_effort at medium while injecting the low-effort instruction text, and reproduces the cost inversion" + - "The T0.3 result reproduces at N>=5 per family per effort level, and the T1 null is either reproduced or resolved" + retracted: - id: hw.q8.27b.tower2-decode-narrows-at-long-ctx @@ -623,3 +813,139 @@ retracted: mostly-prefill work, total-tok/s lands near the prefill rate (~187) even though decode-streaming is ~48 tok/s. Replaced by the three-column headline (prefill / decode / TTFT) with no composite. See AUDIT B19. + + - id: bench.qwen38.nothink-abort-rate + text: > + There is no defensible per-model "abort rate" in this corpus. Runs that + did not deliver are counted and reported as a delivery rate instead — see + [bench.qwen38-vs-qwen36.nothink-delivery-regression] — with the mixture of + end states described rather than collapsed into a single figure. + formerly_claimed: > + A per-model no-think "abort rate", computed as the fraction of cells + carrying the identical-call-loop operator label. + status: retracted + retracted_in: "benchmarks/qwen36-vs-qwen38-27b-2026-08 (replaced by the delivery-rate framing)" + reason: > + The label is an operator kill decision, not a consistent rule, so a rate + built on it measures operator intervention as much as model behavior. All + 58 cells carrying the identical-call-loop label have a label.json and no + summary.json and no workspace_final.tar.gz — they were terminated before + the harness wrote one. Cross-tabbed against the mechanical repetition + signal recorded in the frozen dataset (looped_freq30 / looped_run30), the + label and the signal disagree on 24 of 802 cells: 19 cells show the + mechanical signal yet ran to completion, were graded, and carry no label + (p1_bugfix_qwen38-nothink-card_v2 reaches a maximum identical-call + frequency of 332 and is graded), while 5 labelled cells show no + repetition at all (p3_market_qwen36-nothink-offspec_v9 has max_freq 1 + and max_run 1). Both error directions are present, so the label is + neither a superset nor a subset of the mechanical signal and cannot be + treated as either. + + - id: bench.p3_pm.qwen38-outscores-qwen36 + text: > + p3_pm does not meaningfully separate the two models. Under the repo's + own semantic-equivalence rules, applied from the shipped overlay, 61 of + the 64 graded p3_pm cells pass — Qwen3.6 35/38 and Qwen3.8 26/26 + (the Q8_0 control's cell among them). The + three residual Qwen3.6 FAILs are content misses (risk/decision recall + below threshold even with R3 credited), not lexical artifacts; a + three-cell residual is not the 4/38-versus-21/26 separation the raw + grades showed, and the family should not appear in any per-family win + count built on raw grades. + formerly_claimed: > + Qwen3.8 materially outscores Qwen3.6 on the p3_pm (project-management) + family, 17/21 against 4/37. + status: retracted + retracted_in: "benchmarks/qwen36-vs-qwen38-27b-2026-08 (grader defect D2)" + reason: > + The raw gap was lexical, not substantive. phase3_project_mgmt_grade.py + matches risk item R3 against the literal strings "legal unresponsive", + "legal silent", "legal hasn't" and "blocking on legal". Qwen3.6 writes + "legal has not responded" — semantically identical, literally unmatched. + The corpus contains its own control: p3_pm_qwen36-nothink-card_v4 writes + "legal hasn't responded to draft [wk6]" and PASSES, while card_v2 and + card_v8 write "legal has not responded to draft [wk6]" in the same table + row of an otherwise equivalent report and FAIL. The repo already shipped + the fix — tooling/correct_gemma4_project_mgmt_grades.py, unit-tested, + whose R3 rule is described as "legal/private-beta contract delay uses an + equivalent non-contracted phrase" — but hardcoded to gemma4 cell names, so + it never ran on these cells. Applying those same published rules + unmodified credits R3 on 39 cells and flips 36 verdicts, taking the + family from 25/64 (3.6 4/38, 3.8 21/26) to 61/64 (3.6 35/38, + 3.8 26/26); the three cells still failing miss on risk/decision recall + even with R3 credited. Raw grade.json files are unchanged and remain + the primary record. + + - id: bench.p2_triage.family-result + text: > + p2_triage carries no signal in this campaign in either direction. Once + the brief/ground-truth contradiction is repaired, every frozen graded + cell passes (64/64), and category accuracy is identical (0.867) in all + 64 cells regardless of model, quant, mode or sampler. The family is + reported as instrument-limited and excluded from aggregate comparisons. + formerly_claimed: > + A p2_triage family result separating the models, with Qwen3.8 passing + 19/22 against Qwen3.6's 16/37. + status: retracted + retracted_in: "benchmarks/qwen36-vs-qwen38-27b-2026-08 (grader defect D3)" + reason: > + The task brief and the ground truth contradict each other, and the models + were penalized for obeying the brief. tooling/tasks/task_triage.md defines + urgency "n/a" for noise/spam; ground_truth/phase2_triage.json labels the + three spam tickets (004, 009, 021) "low"; phase2_triage_grade.py compares + by exact string. All 64 frozen graded p2_triage cells answered "n/a" + for all three tickets — both models, every quant (the Q8_0 control cell + included, admitted as graded at freeze #2), every mode, every sampler — + and every one therefore lost exactly 0.100 of a 30-ticket urgency + accuracy scored against a 0.700 threshold. Crediting the + three brief-compliant answers moves every cell over the line: 64/64 + PASS, 24 verdict flips, Qwen3.6 17 to 38 and Qwen3.8 23 to 26. Whether + a cell landed above or below threshold was decided by a uniform defect, + not by triage skill. + + - id: bench.family-paired.nine-three + text: > + No family-paired win count is published. The per-family rates are printed + with their denominators in findings.md and are left as twelve separate + small results. + formerly_claimed: > + A 9-3 family-paired split between the two models. + status: retracted + retracted_in: "benchmarks/qwen36-vs-qwen38-27b-2026-08" + reason: > + The count is not reproducible from the frozen dataset at any sampler and + is not stable under correction. At the matched sampler T0.3/p0.8/pp0 + the family-level win count on raw grades is 4 (3.6) / 5 (3.8) / 3 tied + (all twelve computable at freeze #2 — 3.8 now has a graded p1_bugfix + cell), and 4 / 3 / 5 after correction; in + think mode at the same sampler it is 4 / 5 / 3 raw and 4 / 2 / 6 + corrected. Two of the twelve families were contributing wins largely on + the strength of the grader defects retracted + above ([bench.p3_pm.qwen38-outscores-qwen36] and + [bench.p2_triage.family-result]). Beyond irreproducibility, a win count + over 12 families discards every denominator — cells per family range from + 4 to 19 across the arms — and converts twelve small, differently-powered + results into one number that looks decisive and is not. + + - id: bench.thinking.default-vs-default + text: > + There is no default-versus-default thinking comparison in this corpus. The + cross-model thinking question is held — see + [bench.qwen38-vs-qwen36.thinking-quality-head-to-head.held]. + formerly_claimed: > + A thinking-mode head-to-head between the two models described as + "default vs default". + status: retracted + retracted_in: "benchmarks/qwen36-vs-qwen38-27b-2026-08" + reason: > + The label misdescribes what was run. The only sampler at which both models + have think cells is T1/p0.95/pp0, which is Qwen3.6's vendor sampling point + and not Qwen3.8's. Qwen3.8's vendor sampler is T0.7/p0.8/pp1.5, and the + corpus contains zero Qwen3.8 think cells there — its think arms exist only + at T0.3/p0.8/pp0 (51 cells) and T1/p0.95/pp0 (72 cells). Calling that + comparison "default vs default" reads as both models at their own vendor + settings, when in fact it is both models at one model's settings. + Separately, the two arms are not the same kind of configuration: every + Qwen3.8 think cell carries a reasoning_effort value, so that arm is a + mixture of low/medium/xhigh, while Qwen3.6's chat template has no + reasoning_effort variable at all. diff --git a/tooling/bench_autopilot.py b/tooling/bench_autopilot.py index 83135b61..359dfea1 100755 --- a/tooling/bench_autopilot.py +++ b/tooling/bench_autopilot.py @@ -28,6 +28,7 @@ """ from __future__ import annotations import argparse, json, os, signal, statistics, subprocess, sys, time +import re from pathlib import Path HOME = Path(os.path.expanduser("~")) @@ -44,6 +45,7 @@ HEARTBEAT = STATE_DIR / "heartbeat.json" SCORECARD = STATE_DIR / "scorecard.md" # addition B NOTIFY_SH = HOME / "dream-fleet-test" / "lib" / "notify.sh" +CLEANUP_IMAGE = "sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662" HEARTBEAT_FRESH_SECS = 120 SUBSTANCE_CHECK_SECS = 300 @@ -79,6 +81,12 @@ "arms": [{"label": "397b-nothink", "thinking": "off"}, {"label": "397b-think", "thinking": "on"}], "stuck_secs": 1200, # transcript frozen this long = truly hung (kill it) + # A transcript is intentionally quiet while the harness waits for a bash + # tool. The harness owns that command's timeout and adds a 15 s Docker + # cleanup guard, so the supervisor must not apply the shorter inference + # watchdog to this phase. 3,900 s covers the documented 3,600 s ceiling + # plus teardown/jitter while still bounding a genuinely wedged harness. + "tool_stuck_secs": 3900, "endpoint_grace_secs": 180 # endpoint must load within this on (re)launch } @@ -152,6 +160,26 @@ def sh(cmd, **kw): return subprocess.run(cmd, shell=isinstance(cmd, str), capture_output=True, text=True, **kw) +def cleanup_pattern(parent: Path, pattern: str) -> None: + """Remove root-owned benchmark scratch through a pinned, networkless helper.""" + allowed_parents = {Path("/tmp").resolve(), (TOOLING / "workspace").resolve()} + resolved_parent = parent.resolve() + if resolved_parent not in allowed_parents: + raise RuntimeError(f"cleanup parent is not approved: {resolved_parent}") + if not re.fullmatch(r"[A-Za-z0-9._*-]{1,160}", pattern) or "/" in pattern or ".." in pattern: + raise RuntimeError("cleanup pattern is unsafe") + result = sh([ + "docker", "run", "--rm", "--network", "none", "--read-only", + "-e", f"CLEANUP_PATTERN={pattern}", + "-v", f"{resolved_parent}:/cleanup:rw", + CLEANUP_IMAGE, + "sh", "-c", + 'find /cleanup -mindepth 1 -maxdepth 1 -name "$CLEANUP_PATTERN" -exec rm -rf -- {} \\;', + ]) + if result.returncode != 0: + raise RuntimeError(f"root-owned scratch cleanup failed: {result.stderr.strip()[:300]}") + + # --- pushover --------------------------------------------------------------- def notify(title: str, msg: str, priority: int = 0): """Best-effort Pushover via the fleet notify.sh helper. Never raises.""" @@ -352,7 +380,7 @@ def cell_done(run_name: str) -> bool: # final workspace archive because SIGTERM bypasses harness teardown. try: label = json.loads((d / "label.json").read_text()) - return (label.get("primary") == "identical-call-loop" + return (label.get("primary") in {"identical-call-loop", "stuck-in-research"} and (d / "receipt.json").exists() and (d / "transcript.jsonl").exists()) except Exception: @@ -476,6 +504,40 @@ def current_cell_info(cfg=None): return {"cell": tp.parent.name, "iter": it, "frozen_secs": int(time.time() - mt)} +def _last_transcript_row(transcript: Path): + """Return the newest JSONL object without rereading a potentially huge log.""" + try: + with transcript.open("rb") as handle: + handle.seek(0, os.SEEK_END) + size = handle.tell() + handle.seek(max(0, size - 65536)) + tail = handle.read().decode("utf-8", errors="replace") + for line in reversed(tail.splitlines()): + if line.strip(): + row = json.loads(line) + return row if isinstance(row, dict) else None + except (OSError, json.JSONDecodeError): + pass + return None + + +def watchdog_stuck_policy(cfg: dict, transcript: Path): + """Return ``(limit_seconds, phase)`` for the active harness phase. + + Model rows ending in ``tool_calls`` precede tool execution; the matching + tool row is written only after that command exits. During that interval + neither transcript mtime nor inference-slot counters move. Applying the + normal 1,200 s inference watchdog there can therefore kill a valid command + whose harness timeout is 1,800 or 3,600 s. + """ + inference_limit = int(cfg["stuck_secs"]) + row = _last_transcript_row(transcript) + if row and row.get("type") == "model" and row.get("finish_reason") == "tool_calls": + tool_limit = int(cfg.get("tool_stuck_secs", 3900)) + return max(inference_limit, tool_limit), "pending-tool" + return inference_limit, "inference-or-idle" + + def kill_stuck(cell: str): """Kill a truly-hung cell's harness PID + sandbox so the chain advances.""" log(f"STUCK: killing hung cell {cell}") @@ -827,7 +889,7 @@ def git(*args, check=False): log(f"publish: git step failed (non-fatal): {e}") -def benchmark_environment(cfg, base_env=None): +def benchmark_environment(cfg, base_env=None, arm=None): """Return the child environment carrying a model's pinned sampling point. ``run_microbench.sh`` intentionally keeps historical defaults, so model-card @@ -840,13 +902,20 @@ def benchmark_environment(cfg, base_env=None): "benchmark_temperature": "BENCH_TEMP", "benchmark_top_p": "BENCH_TOP_P", "benchmark_top_k": "BENCH_TOP_K", + "benchmark_min_p": "BENCH_MIN_P", + "benchmark_presence_penalty": "BENCH_PRESENCE_PENALTY", + "benchmark_repeat_penalty": "BENCH_REPEAT_PENALTY", + "benchmark_seed": "BENCH_SEED", "benchmark_max_output_tokens_cap": "BENCH_MAX_OUTPUT_TOKENS_CAP", + "benchmark_sandbox_gpus": "BENCH_SANDBOX_GPUS", + "preserve_thinking": "BENCH_PRESERVE_THINKING", + "reasoning_effort_location": "BENCH_REASONING_EFFORT_LOCATION", "serving_manifest": "BENCH_SERVING_MANIFEST", } for config_key, env_key in sampling_env.items(): - value = cfg.get(config_key) + value = arm.get(config_key, cfg.get(config_key)) if arm else cfg.get(config_key) if value is not None: - run_env[env_key] = str(value) + run_env[env_key] = str(value).lower() if isinstance(value, bool) else str(value) else: run_env.pop(env_key, None) return run_env @@ -855,10 +924,12 @@ def benchmark_environment(cfg, base_env=None): def run_arm_with_supervision(cfg, arm, target, started_at): """Shard one arm deterministically across configured replica endpoints.""" label, thinking = arm["label"], arm["thinking"] - ids = sh("docker ps -aq --filter name=bench-sandbox-").stdout.split() + # Scope to THIS arm only. A global wipe kills concurrent campaigns' live cells + # mid-run; harness.py names sandboxes per-cell so parallel runs can coexist. + ids = sh(f"docker ps -aq --filter name=bench-sandbox-.*{label}").stdout.split() if ids: sh(["docker", "rm", "-f", *ids]) - sh(f"sudo rm -rf {TOOLING}/workspace/*{label}_v* 2>/dev/null") + cleanup_pattern(TOOLING / "workspace", f"*{label}_v*") ports = configured_ports(cfg) lane_count = len(ports) log(f"RUN arm {label} (thinking={thinking}) target N={target} lanes={ports}") @@ -866,7 +937,7 @@ def run_arm_with_supervision(cfg, arm, target, started_at): handles = [] try: for lane_index, port in enumerate(ports): - run_env = benchmark_environment(cfg) + run_env = benchmark_environment(cfg, arm=arm) run_env["BENCH_LANE_INDEX"] = str(lane_index) run_env["BENCH_LANE_COUNT"] = str(lane_count) log_path = STATE_DIR / ( @@ -877,7 +948,8 @@ def run_arm_with_supervision(cfg, arm, target, started_at): handles.append(handle) proc = subprocess.Popen( ["bash", str(SCRIPTS / "run_microbench.sh"), cfg["model"], str(port), - label, str(target), "", thinking, str(cfg["max_model_len"])], + label, str(target), str(arm.get("reasoning_effort", "")), + thinking, str(cfg["max_model_len"])], stdout=handle, stderr=subprocess.STDOUT, env=run_env) procs.append((lane_index, port, proc)) @@ -922,10 +994,12 @@ def run_arm_with_supervision(cfg, arm, target, started_at): watch.update(cell=cell, signal=signal, last_progress=now) continue frozen_secs = int(now - watch["last_progress"]) - if frozen_secs > cfg["stuck_secs"]: + stuck_limit, phase = watchdog_stuck_policy(cfg, transcript) + if frozen_secs > stuck_limit: log( f"STUCK: lane port {port} cell {cell} showed no transcript " - f"or server-slot progress for {frozen_secs}s" + f"or server-slot progress for {frozen_secs}s " + f"(phase={phase}, limit={stuck_limit}s)" ) kill_stuck(cell) watch.update(cell=None, signal=None, last_progress=now) @@ -934,7 +1008,7 @@ def run_arm_with_supervision(cfg, arm, target, started_at): finally: for handle in handles: handle.close() - sh(f"sudo rm -rf /tmp/grade_*{label}_v* 2>/dev/null") + cleanup_pattern(Path("/tmp"), f"grade_*{label}_v*") g = sh(["bash", str(SCRIPTS / "grade_microbench.sh"), label]) (STATE_DIR / f"grade-{label}.log").write_text(g.stdout + g.stderr) s = sh(["bash", str(SCRIPTS / "summarize.sh"), label]) diff --git a/tooling/bench_dashboard.py b/tooling/bench_dashboard.py index 13a1f810..8f473dbc 100644 --- a/tooling/bench_dashboard.py +++ b/tooling/bench_dashboard.py @@ -33,14 +33,15 @@ from collections import Counter from pathlib import Path -HOME = Path(os.path.expanduser("~")) -LOGS = HOME / "bench" / "logs" +REPO = Path(__file__).resolve().parent.parent +LOGS = REPO / "logs" STATE = Path("/tmp/bench-autopilot") STATUS = STATE / "status.json" LOG = STATE / "autopilot.log" -# Glob that matches a cell dir for any arm/rep (e.g. p1_bugfix_397b-nothink_v3). -CELL_GLOB = "p*_397b-*_v*" +# Glob that matches a cell dir for any model arm/rep. A dedicated campaign +# checkout keeps this generic scan isolated from unrelated historical logs. +CELL_GLOB = "p[1-3]_*_v*" # N=10 no-think baseline pass counts per task (totals to 82/120). BASELINE_N10_NOTHINK = { @@ -536,6 +537,7 @@ def load_status(): def main(): + global LOGS ap = argparse.ArgumentParser() ap.add_argument("--watch", action="store_true") ap.add_argument("--html", default=None) @@ -550,7 +552,10 @@ def main(): help="per-task cumulative-window pass-rate table (v1-3/5/10/20) per arm") ap.add_argument("--flips", action="store_true", help="per-task flip map: cells differing from the task/arm modal verdict") + ap.add_argument("--logs", default=str(LOGS), + help="per-cell logs directory (default: logs in this checkout)") args = ap.parse_args() + LOGS = Path(os.path.expanduser(args.logs)).resolve() def once(): status = load_status() # None if missing/partial — every renderer handles None diff --git a/tooling/bench_report.py b/tooling/bench_report.py index 726022ab..ab1b1f15 100644 --- a/tooling/bench_report.py +++ b/tooling/bench_report.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """bench_report — standalone MMBT findings-markdown generator. -Reads ~/bench/logs/_