benchmark analysis: HITL pass-through, all-predictions audit, per-field breakdown

Tooling additions used to prepare steering-committee briefing materials.

- hitl-planner: track per-document sample index on each prediction; compute
  per-document pass-through (zero-flag rate) at each target recall and emit
  hitl-passthrough.csv alongside the existing per-category / combined CSVs.
- report-errors: add --include-all-predictions to emit all-predictions.csv
  (every cell, matched + non-matched), same schema as wrong-by-category.csv.
  Share wrapper passes the flag through.
- hitl-per-category-plots: new diagnostic plotting tool that emits one
  two-panel PNG per category (recall-vs-workload + threshold-vs-workload)
  to make threshold cliffs in small-error categories (e.g. name, date)
  visually obvious. Share wrapper included.
- verify-all-predictions: aggregate-only verification over all-predictions.csv
  (counts, accuracy per category, no values). Also emits per-field-breakdown
  with reviewable counts so the "fields with content per doc" figure is
  traceable to specific fields.

Author: strukalex

scripts/benchmark analysis/hitl-per-category-plots-share.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# Generate per-category HITL detail plots from a UNC share benchmark JSON.
+# Same streaming pattern as hitl-planner-share.sh.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+FIFO_DIR="$(mktemp -d -t hitl-plots-fifo-XXXXXX)"
+SHM_DIR="$(mktemp -d -p /dev/shm hitl-plots-XXXXXX 2>/dev/null \
+            || mktemp -d -t hitl-plots-XXXXXX)"
+POWERSHELL="/mnt/c/Windows/System32/WindowsPowerShell/v1.0/powershell.exe"
+WRITER_PIDS=()
+
+cleanup() {
+    for pid in "${WRITER_PIDS[@]}"; do kill "$pid" 2>/dev/null || true; done
+    rm -rf "$FIFO_DIR" "$SHM_DIR"
+}
+trap cleanup EXIT
+
+INPUT=""
+OUT_DIR=""
+EXTRA=()
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --out-dir) OUT_DIR="$2"; shift 2 ;;
+        --categories|--docs-count|--exclude-missing-in-categories|--skip-trivial-predictions-in-categories)
+            EXTRA+=("$1" "$2"); shift 2 ;;
+        *) if [[ -z "$INPUT" ]]; then INPUT="$1"; shift
+           else echo "error: unexpected arg: $1" >&2; exit 2; fi ;;
+    esac
+done
+[[ -z "$INPUT" || -z "$OUT_DIR" ]] && { echo "usage: $0 <input.json> --out-dir <dir> [options]" >&2; exit 2; }
+
+INPUT_PATH="$INPUT"
+if [[ "$INPUT" == \\\\* || "$INPUT" == //* ]]; then
+    INPUT_FIFO="$FIFO_DIR/input.json.fifo"
+    mkfifo "$INPUT_FIFO"
+    ( "$POWERSHELL" -NoProfile -Command \
+        "\$b = [System.IO.File]::ReadAllBytes('$INPUT'); \
+         \$o = [System.Console]::OpenStandardOutput(); \
+         \$o.Write(\$b, 0, \$b.Length); \$o.Close()" \
+        > "$INPUT_FIFO" ) &
+    WRITER_PIDS+=("$!")
+    INPUT_PATH="$INPUT_FIFO"
+    echo "input ← $INPUT (streamed)" >&2
+fi
+
+python3 "${SCRIPT_DIR}/hitl-per-category-plots.py" \
+    "$INPUT_PATH" \
+    --out-dir "$SHM_DIR" \
+    "${EXTRA[@]}"
+
+for pid in "${WRITER_PIDS[@]}"; do wait "$pid" 2>/dev/null || true; done
+
+if [[ "$OUT_DIR" == \\\\* || "$OUT_DIR" == //* ]]; then
+    SHM_WIN=$(wslpath -w "$SHM_DIR")
+    "$POWERSHELL" -NoProfile -Command \
+        "if (-not (Test-Path -LiteralPath '$OUT_DIR')) { New-Item -ItemType Directory -Path '$OUT_DIR' -Force | Out-Null }; \
+         Get-ChildItem -LiteralPath '$SHM_WIN' -File | ForEach-Object { Copy-Item -LiteralPath \$_.FullName -Destination '$OUT_DIR' -Force }" >&2
+    echo "outputs → $OUT_DIR" >&2
+else
+    mkdir -p "$OUT_DIR"
+    cp "${SHM_DIR}"/* "$OUT_DIR/"
+    echo "outputs → $OUT_DIR" >&2
+fi
+echo "done." >&2

scripts/benchmark analysis/hitl-per-category-plots.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+"""Generate individual per-category HITL plots showing the full continuous
+threshold sweep. One PNG per category, plus one combined overview.
+
+Usage:
+    python hitl-per-category-plots.py <input.json> \
+        --out-dir <dir> \
+        --categories income_amounts,sin,phone,name,date \
+        [--exclude-missing-in-categories income_amounts] \
+        [--skip-trivial-predictions-in-categories income_amounts] \
+        [--docs-count 99]
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from importlib import import_module
+
+planner = import_module("hitl-planner")
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+
+def plot_single_category(
+    cat: str,
+    sweep_rows: list[dict],
+    full_curve: list[dict],
+    targets: list[float],
+    out_path: Path,
+) -> None:
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5.5))
+
+    xs = [r["reviews_per_100_docs"] for r in full_curve]
+    ys = [r["recall"] * 100 for r in full_curve]
+    ts = [r["threshold"] for r in full_curve]
+    colour = planner.CATEGORY_COLOURS.get(cat, "#333")
+
+    # Left panel: recall vs reviews (log scale) — same as the combined chart
+    ax1.plot(xs, ys, color=colour, linewidth=2)
+    for r in sweep_rows:
+        if r["threshold"] is None:
+            continue
+        ax1.scatter([r["reviews_per_100_docs"]], [r["recall_actual"] * 100],
+                    color=colour, edgecolor="black", s=70, zorder=5)
+        ax1.annotate(
+            f"T={r['threshold']:.2f}\n{r['predictions_reviewable_flagged']}/{r['predictions_reviewable_total']} flagged",
+            xy=(r["reviews_per_100_docs"], r["recall_actual"] * 100),
+            xytext=(8, -8), textcoords="offset points", fontsize=8,
+        )
+
+    for tgt in targets:
+        ax1.axhline(tgt * 100, color="grey", linestyle=":", linewidth=0.7, alpha=0.5)
+
+    ax1.set_xscale("log")
+    ax1.set_xlim(left=1)
+    ax1.set_ylim(0, 105)
+    ax1.set_xlabel("Reviews per 100 documents (log scale)")
+    ax1.set_ylabel("Errors caught (%)")
+    ax1.set_title(f"{cat} — recall vs review workload")
+    ax1.grid(alpha=0.3, which="both")
+
+    # Right panel: threshold vs reviews (linear) — shows the cliff
+    ax2.plot(ts, xs, color=colour, linewidth=2)
+    for r in sweep_rows:
+        if r["threshold"] is None:
+            continue
+        ax2.scatter([r["threshold"]], [r["reviews_per_100_docs"]],
+                    color=colour, edgecolor="black", s=70, zorder=5)
+        ax2.annotate(
+            f"{int(r['target_recall']*100)}% recall",
+            xy=(r["threshold"], r["reviews_per_100_docs"]),
+            xytext=(6, 6), textcoords="offset points", fontsize=8,
+        )
+
+    ax2.set_xlabel("Confidence threshold T")
+    ax2.set_ylabel("Reviews per 100 documents")
+    ax2.set_title(f"{cat} — threshold vs review workload")
+    ax2.grid(alpha=0.3)
+
+    n_err = sweep_rows[0]["errors_total"]
+    n_reviewable = sweep_rows[0]["predictions_reviewable_total"]
+    fig.suptitle(
+        f"{cat}: {n_err} errors, {n_reviewable} reviewable predictions across 99 docs",
+        fontsize=12, fontweight="bold",
+    )
+    fig.tight_layout()
+    fig.savefig(out_path, dpi=150)
+    plt.close(fig)
+
+
+def main(argv: list[str]) -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("input")
+    ap.add_argument("--out-dir", required=True, type=Path)
+    ap.add_argument("--categories", default="income_amounts,sin,phone,name,date")
+    ap.add_argument("--docs-count", type=int, default=99)
+    ap.add_argument("--exclude-missing-in-categories", default="")
+    ap.add_argument("--skip-trivial-predictions-in-categories", default="")
+    args = ap.parse_args(argv)
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+
+    allowlist = [c.strip() for c in args.categories.split(",") if c.strip()]
+    allowlist = [c for c in allowlist if c in planner.CATEGORY_ORDER]
+
+    exclude_missing = {c.strip() for c in args.exclude_missing_in_categories.split(",") if c.strip()}
+    skip_trivial = {c.strip() for c in args.skip_trivial_predictions_in_categories.split(",") if c.strip()}
+
+    preds = planner.load_predictions(Path(args.input))
+    print(f"loaded {len(preds)} predictions", file=sys.stderr)
+
+    by_cat_preds: dict[str, list] = {}
+    for p in preds:
+        if p.category in allowlist:
+            by_cat_preds.setdefault(p.category, []).append(p)
+
+    for cat in allowlist:
+        cat_preds = by_cat_preds.get(cat, [])
+        if not cat_preds:
+            continue
+        cat_preds = planner.filter_predictions_for_category(cat_preds, cat, cat in exclude_missing)
+        cat_skip = cat in skip_trivial
+        sweep = planner.sweep_for_category(cat_preds, planner.TARGET_RECALLS, args.docs_count, skip_trivial=cat_skip)
+        curve = planner.sweep_full_curve(cat_preds, args.docs_count, step=0.005, skip_trivial=cat_skip)
+
+        out_path = args.out_dir / f"hitl-detail-{cat}.png"
+        plot_single_category(cat, sweep, curve, planner.TARGET_RECALLS, out_path)
+        print(f"  {cat} → {out_path}", file=sys.stderr)
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

scripts/benchmark analysis/hitl-planner.py

@@ -111,6 +111,7 @@ class Prediction:
     predicted_is_empty: bool      # the model returned null / empty for this cell
     expected_is_empty: bool       # ground truth is null / empty for this cell
     predicted_is_trivial: bool    # predicted is empty or "looks like 0" — see _predicted_looks_trivial
+    sample_index: int = 0         # ordinal index of the source sample (no PII)
 
 
 TARGET_RECALLS = [0.50, 0.70, 0.80, 0.90, 0.95, 0.99]
@@ -161,7 +162,7 @@ def _predicted_looks_trivial(value: Any) -> bool:
 def load_predictions(path: Path) -> list[Prediction]:
     raw = json.loads(path.read_text("utf-8"))
     out: list[Prediction] = []
-    for sample in raw.get("perSampleResults") or []:
+    for sample_idx, sample in enumerate(raw.get("perSampleResults") or []):
         for det in sample.get("evaluationDetails") or []:
             field = det.get("field")
             conf = det.get("confidence")
@@ -179,6 +180,7 @@ def load_predictions(path: Path) -> list[Prediction]:
                 predicted_is_empty=pred_empty,
                 expected_is_empty=exp_empty,
                 predicted_is_trivial=pred_trivial,
+                sample_index=sample_idx,
             ))
     return out
 
@@ -508,6 +510,93 @@ def plot_hitl_curves(
     plt.close(fig)
 
 
+# ---------------------------------------------------------------------------
+# Per-document pass-through analysis
+# ---------------------------------------------------------------------------
+
+
+def compute_passthrough(
+    by_cat_preds: dict[str, list[Prediction]],
+    by_category_sweep: dict[str, list[dict]],
+    targets: list[float],
+    docs_count: int,
+    exclude_missing: set[str],
+    skip_trivial: set[str],
+) -> list[dict]:
+    """For each target recall, compute the fraction of documents that have
+    zero reviewable flagged predictions across ALL HITL-scoped categories.
+    A document passes through only if none of its in-scope predictions are
+    flagged at the combined per-category thresholds for that recall level.
+    """
+    cats_present = [c for c in CATEGORY_ORDER if c in by_category_sweep]
+
+    per_cat_filtered: dict[str, list[Prediction]] = {}
+    for cat in cats_present:
+        cat_preds = by_cat_preds.get(cat, [])
+        per_cat_filtered[cat] = filter_predictions_for_category(
+            cat_preds, cat, cat in exclude_missing
+        )
+
+    rows: list[dict] = []
+    for idx, target in enumerate(targets):
+        per_cat_thresholds: dict[str, float] = {}
+        for cat in cats_present:
+            sweep_row = by_category_sweep[cat][idx]
+            t = sweep_row["threshold"]
+            per_cat_thresholds[cat] = t if t is not None else 0.0
+
+        doc_flags: dict[int, int] = {i: 0 for i in range(docs_count)}
+        for cat in cats_present:
+            t = per_cat_thresholds[cat]
+            use_skip_trivial = cat in skip_trivial
+            reviewable_fn = _reviewable_skip_trivial if use_skip_trivial else _reviewable_default
+            for p in per_cat_filtered[cat]:
+                if p.confidence < t and reviewable_fn(p):
+                    doc_flags[p.sample_index] = doc_flags.get(p.sample_index, 0) + 1
+
+        flag_counts = list(doc_flags.values())
+        zero_flag_docs = sum(1 for c in flag_counts if c == 0)
+        one_flag_docs = sum(1 for c in flag_counts if c == 1)
+        two_flag_docs = sum(1 for c in flag_counts if c == 2)
+        three_plus_docs = sum(1 for c in flag_counts if c >= 3)
+
+        rows.append({
+            "target_recall": target,
+            "docs_total": docs_count,
+            "docs_zero_flags": zero_flag_docs,
+            "docs_one_flag": one_flag_docs,
+            "docs_two_flags": two_flag_docs,
+            "docs_three_plus_flags": three_plus_docs,
+            "passthrough_pct": zero_flag_docs * 100 / docs_count if docs_count else 0.0,
+            "avg_flags_per_doc": sum(flag_counts) / docs_count if docs_count else 0.0,
+            "max_flags": max(flag_counts) if flag_counts else 0,
+        })
+    return rows
+
+
+def write_passthrough_csv(rows: list[dict], out_path: Path) -> None:
+    header = [
+        "target_recall", "docs_total",
+        "docs_zero_flags", "docs_one_flag", "docs_two_flags", "docs_three_plus_flags",
+        "passthrough_pct", "avg_flags_per_doc", "max_flags",
+    ]
+    with out_path.open("w", newline="", encoding="utf-8") as f:
+        w = csv.writer(f)
+        w.writerow(header)
+        for r in rows:
+            w.writerow([
+                f"{r['target_recall']:.2f}",
+                r["docs_total"],
+                r["docs_zero_flags"],
+                r["docs_one_flag"],
+                r["docs_two_flags"],
+                r["docs_three_plus_flags"],
+                f"{r['passthrough_pct']:.1f}",
+                f"{r['avg_flags_per_doc']:.2f}",
+                r["max_flags"],
+            ])
+
+
 # ---------------------------------------------------------------------------
 # Entrypoint
 # ---------------------------------------------------------------------------
@@ -610,8 +699,14 @@ def main(argv: list[str]) -> int:
         args.out_dir / "hitl-curves.png", args.engine_label,
     )
 
+    passthrough_rows = compute_passthrough(
+        by_cat_preds, by_category_sweep, TARGET_RECALLS,
+        args.docs_count, exclude_missing, skip_trivial,
+    )
+    write_passthrough_csv(passthrough_rows, args.out_dir / "hitl-passthrough.csv")
+
     print(f"\nWrote outputs to {args.out_dir}/", file=sys.stderr)
-    print("  CSVs:  hitl-per-category.csv, hitl-combined.csv", file=sys.stderr)
+    print("  CSVs:  hitl-per-category.csv, hitl-combined.csv, hitl-passthrough.csv", file=sys.stderr)
     print("  PNG:   hitl-curves.png", file=sys.stderr)
     # Summary line
     last = by_category_sweep

scripts/benchmark analysis/report-errors-share.sh

@@ -33,9 +33,11 @@ trap cleanup EXIT
 
 ENGINE_ARGS=()
 OUT_DIR=""
+EXTRA_FLAGS=()
 while [[ $# -gt 0 ]]; do
     case "$1" in
         --out-dir) OUT_DIR="$2"; shift 2 ;;
+        --include-all-predictions) EXTRA_FLAGS+=("$1"); shift ;;
         -h|--help)
             sed -n 's/^# //p; s/^#//p' "$0" | sed -n '1,18p'
             exit 0 ;;
@@ -74,7 +76,8 @@ done
 # Run the report generator.
 python3 "${SCRIPT_DIR}/report-errors.py" \
     "${STAGED[@]}" \
-    --out-dir "$SHM_DIR"
+    --out-dir "$SHM_DIR" \
+    "${EXTRA_FLAGS[@]}"
 
 # Wait for input writers.
 for pid in "${WRITER_PIDS[@]}"; do