feat(report): add threshold sweep metrics

AlisinaDevelo · AlisinaDevelo · commit 4fce240c663f · 2026-06-24T11:37:56.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,7 @@
 - `GET /models` serving endpoint for safe loaded-model metadata.
 - `examples/smoke_pairs.csv` and CI coverage for the train-to-report workflow.
 - JSON output for `quorabust-report --format json`.
+- Precision, recall, F1, and configurable threshold sweeps in model reports.
 
 ### Fixed
 - Capped NumPy below 2.5 so Mypy can parse dependency stubs with the configured
diff --git a/README.md b/README.md
@@ -71,9 +71,10 @@ quorabust-report \
 ```
 
 The report includes artifact metadata, persisted training/eval metrics, optional holdout
-metrics, and a threshold confusion matrix. Use `--format json` for machine-readable CI or
-release artifacts. Use a real held-out CSV for comparable model claims; the command
-accepts the same `question1`, `question2`, `is_duplicate` column contract as training.
+metrics, a confusion matrix, and a precision/recall/F1 threshold sweep. Use
+`--format json` for machine-readable CI or release artifacts. Use a real held-out CSV for
+comparable model claims; the command accepts the same `question1`, `question2`,
+`is_duplicate` column contract as training.
 See [docs/REPORTING.md](docs/REPORTING.md) for the CI smoke workflow and
 real-evaluation checklist.
 
diff --git a/docs/REPORTING.md b/docs/REPORTING.md
@@ -20,6 +20,7 @@ quorabust-report \
   --model /tmp/quorabust-smoke.pkl \
   --artifact-label quorabust-smoke.pkl \
   --eval-csv examples/smoke_pairs.csv \
+  --thresholds 0.3,0.5,0.7 \
   --out /tmp/quorabust-smoke-model-card.md
 
 quorabust-report \
@@ -33,6 +34,10 @@ quorabust-report \
 The smoke dataset proves the command path works. It is not a benchmark and should not be
 used for public model-quality claims.
 
+Reports include precision, recall, F1, accuracy, and predicted-positive rate at the
+selected threshold plus a threshold sweep. Use `--thresholds` to compare operating
+points before choosing one.
+
 ## Real Evaluation
 
 For comparable numbers, generate the report from a held-out CSV that was not used for
@@ -49,6 +54,7 @@ quorabust-report \
   --model models/quorabust.pkl \
   --artifact-label quorabust-tfidf-v1.pkl \
   --eval-csv data/processed/holdout.csv \
+  --thresholds 0.2,0.3,0.4,0.5,0.6,0.7,0.8 \
   --out reports/quorabust-tfidf-v1.md
 ```
 
diff --git a/docs/SAMPLE_MODEL_CARD.md b/docs/SAMPLE_MODEL_CARD.md
@@ -29,7 +29,7 @@ Scores pairs of short natural-language questions and returns the probability tha
 | n_eval | 0 |
 | seed | 7 |
 | quorabust_version | 0.3.2 |
-| git_revision | 669dd7dffb715e2d5d84c7ec3ce8315219115057 |
+| git_revision | bb7c63ccdf8b22f8ed963dd312013eb2bd6f6719 |
 | csv_sha256 | 0a7b4ad6c6bb12fd038d5d7673eaced0646a8914c52dbdf502d2476b29ad6132 |
 
 ## Persisted Evaluation
@@ -47,6 +47,9 @@ Scores pairs of short natural-language questions and returns the probability tha
 | n | 32 |
 | threshold | 0.5000 |
 | accuracy | 0.9375 |
+| precision | 0.9444 |
+| recall | 0.9444 |
+| f1 | 0.9444 |
 | log_loss | 0.1958 |
 | roc_auc | 0.9841 |
 | positive_rate | 0.5625 |
@@ -59,6 +62,14 @@ Scores pairs of short natural-language questions and returns the probability tha
 | actual 0 | 13 | 1 |
 | actual 1 | 1 | 17 |
 
+## Threshold Sweep
+
+| threshold | precision | recall | f1 | accuracy | predicted_positive_rate |
+| --- | --- | --- | --- | --- | --- |
+| 0.3000 | 0.8182 | 1.0000 | 0.9000 | 0.8750 | 0.6875 |
+| 0.5000 | 0.9444 | 0.9444 | 0.9444 | 0.9375 | 0.5625 |
+| 0.7000 | 1.0000 | 0.8333 | 0.9091 | 0.9062 | 0.4688 |
+
 ## Serving Contract
 
 `POST /predict` accepts `question1` and `question2` arrays of equal length and returns `proba_duplicate` in the same order. `GET /metrics` exposes Prometheus counters and latency histograms.
@@ -72,8 +83,7 @@ Performance depends on the training data distribution and threshold. Re-run this
 ```bash
 pip install -e ".[dev]"
 quorabust-train  --csv examples/smoke_pairs.csv --out /tmp/quorabust-smoke.pkl --eval-fraction 0 --seed 7
-quorabust-report --model /tmp/quorabust-smoke.pkl --artifact-label quorabust-smoke.pkl --eval-csv examples/smoke_pairs.csv --out docs/SAMPLE_MODEL_CARD.md
+quorabust-report --model /tmp/quorabust-smoke.pkl --artifact-label quorabust-smoke.pkl --eval-csv examples/smoke_pairs.csv --thresholds 0.3,0.5,0.7 --out docs/SAMPLE_MODEL_CARD.md
 ```
 
 For a real model card, run the same `train`/`report` commands against the Kaggle Quora Question Pairs CSV instead of the smoke fixture.
-
diff --git a/src/quorabust/report.py b/src/quorabust/report.py
@@ -9,7 +9,15 @@
 
 import numpy as np
 import pandas as pd
-from sklearn.metrics import accuracy_score, confusion_matrix, log_loss, roc_auc_score
+from sklearn.metrics import (
+    accuracy_score,
+    confusion_matrix,
+    f1_score,
+    log_loss,
+    precision_score,
+    recall_score,
+    roc_auc_score,
+)
 
 from quorabust.model import predict_proba_duplicate
 from quorabust.persist import load_classifier
@@ -75,46 +83,95 @@ def _persisted_metrics_from_meta(meta: dict[str, Any]) -> dict[str, float]:
     }
 
 
-def evaluate_holdout(
+def _parse_thresholds(raw: str) -> list[float]:
+    out: list[float] = []
+    for part in raw.split(","):
+        value = part.strip()
+        if not value:
+            continue
+        try:
+            threshold = float(value)
+        except ValueError as exc:
+            raise ValueError(f"Invalid threshold: {value}") from exc
+        if not 0.0 < threshold < 1.0:
+            raise ValueError("thresholds must be between 0 and 1")
+        out.append(threshold)
+    if not out:
+        raise ValueError("at least one threshold is required")
+    return out
+
+
+def _metrics_at_threshold(y: np.ndarray, proba: np.ndarray, threshold: float) -> dict[str, Any]:
+    pred = (proba >= threshold).astype(int)
+    tn, fp, fn, tp = confusion_matrix(y, pred, labels=[0, 1]).ravel()
+    return {
+        "threshold": float(threshold),
+        "accuracy": float(accuracy_score(y, pred)),
+        "precision": float(precision_score(y, pred, zero_division=0)),
+        "recall": float(recall_score(y, pred, zero_division=0)),
+        "f1": float(f1_score(y, pred, zero_division=0)),
+        "tn": int(tn),
+        "fp": int(fp),
+        "fn": int(fn),
+        "tp": int(tp),
+        "predicted_positive_rate": float(np.mean(pred)),
+    }
+
+
+def _holdout_proba(
     builder: Any,
     clf: Any,
     df: pd.DataFrame,
-    *,
-    threshold: float = 0.5,
-) -> dict[str, Any]:
-    """Evaluate a loaded artifact against a labeled Quora-style dataframe."""
+) -> tuple[np.ndarray, np.ndarray]:
     y = df["is_duplicate"].astype(int).to_numpy()
     proba = predict_proba_duplicate(
         builder,
         clf,
         df["question1"].astype(str).tolist(),
         df["question2"].astype(str).tolist(),
     )[:, 1]
-    pred = (proba >= threshold).astype(int)
-    tn, fp, fn, tp = confusion_matrix(y, pred, labels=[0, 1]).ravel()
+    return y, proba
+
 
+def evaluate_holdout(
+    builder: Any,
+    clf: Any,
+    df: pd.DataFrame,
+    *,
+    threshold: float = 0.5,
+) -> dict[str, Any]:
+    """Evaluate a loaded artifact against a labeled Quora-style dataframe."""
+    y, proba = _holdout_proba(builder, clf, df)
+    threshold_metrics = _metrics_at_threshold(y, proba, threshold)
     metrics: dict[str, Any] = {
         "n": int(len(df)),
-        "threshold": float(threshold),
-        "accuracy": float(accuracy_score(y, pred)),
         "log_loss": float(log_loss(y, proba, labels=[0, 1])),
-        "tn": int(tn),
-        "fp": int(fp),
-        "fn": int(fn),
-        "tp": int(tp),
         "positive_rate": float(np.mean(y)),
-        "predicted_positive_rate": float(np.mean(pred)),
+        **threshold_metrics,
     }
     if len(np.unique(y)) > 1:
         metrics["roc_auc"] = float(roc_auc_score(y, proba))
     return metrics
 
 
+def threshold_sweep(
+    builder: Any,
+    clf: Any,
+    df: pd.DataFrame,
+    *,
+    thresholds: list[float],
+) -> list[dict[str, Any]]:
+    """Evaluate precision/recall tradeoffs across decision thresholds."""
+    y, proba = _holdout_proba(builder, clf, df)
+    return [_metrics_at_threshold(y, proba, threshold) for threshold in thresholds]
+
+
 def render_model_card(
     *,
     artifact: str,
     meta: dict[str, Any],
     holdout_metrics: dict[str, Any] | None = None,
+    sweep_metrics: list[dict[str, Any]] | None = None,
 ) -> str:
     """Render artifact metadata and optional holdout metrics as Markdown."""
     artifact_rows = [
@@ -162,6 +219,9 @@ def render_model_card(
             "n",
             "threshold",
             "accuracy",
+            "precision",
+            "recall",
+            "f1",
             "log_loss",
             "roc_auc",
             "positive_rate",
@@ -184,6 +244,36 @@ def render_model_card(
                 _markdown_table(["", "predicted 0", "predicted 1"], confusion_rows),
             ]
         )
+        if sweep_metrics:
+            sweep_rows = [
+                [
+                    row["threshold"],
+                    row["precision"],
+                    row["recall"],
+                    row["f1"],
+                    row["accuracy"],
+                    row["predicted_positive_rate"],
+                ]
+                for row in sweep_metrics
+            ]
+            parts.extend(
+                [
+                    "",
+                    "## Threshold Sweep",
+                    "",
+                    _markdown_table(
+                        [
+                            "threshold",
+                            "precision",
+                            "recall",
+                            "f1",
+                            "accuracy",
+                            "predicted_positive_rate",
+                        ],
+                        sweep_rows,
+                    ),
+                ]
+            )
 
     parts.extend(
         [
@@ -213,6 +303,7 @@ def build_report_payload(
     artifact: str,
     meta: dict[str, Any],
     holdout_metrics: dict[str, Any] | None = None,
+    sweep_metrics: list[dict[str, Any]] | None = None,
 ) -> dict[str, Any]:
     """Build a machine-readable report payload for CI and model comparisons."""
     payload: dict[str, Any] = {
@@ -252,6 +343,8 @@ def build_report_payload(
                 "predicted_1": holdout_metrics["tp"],
             },
         }
+        if sweep_metrics:
+            payload["threshold_sweep"] = sweep_metrics
     return payload
 
 
@@ -272,6 +365,11 @@ def main(argv: list[str] | None = None) -> int:
         default=0.5,
         help="Decision threshold for the optional confusion matrix",
     )
+    parser.add_argument(
+        "--thresholds",
+        default="0.3,0.5,0.7",
+        help="Comma-separated thresholds for the optional holdout sweep",
+    )
     parser.add_argument(
         "--artifact-label",
         default=None,
@@ -292,20 +390,33 @@ def main(argv: list[str] | None = None) -> int:
     if not 0.0 < args.threshold < 1.0:
         print("--threshold must be between 0 and 1", file=sys.stderr)
         return 1
+    try:
+        thresholds = _parse_thresholds(args.thresholds)
+    except ValueError as exc:
+        print(str(exc), file=sys.stderr)
+        return 1
 
     builder, clf, meta = load_classifier(args.model)
     holdout_metrics = None
+    sweep_metrics = None
     if args.eval_csv is not None:
         if not args.eval_csv.is_file():
             print(f"File not found: {args.eval_csv}", file=sys.stderr)
             return 1
         try:
+            eval_df = _load_eval_csv(args.eval_csv)
             holdout_metrics = evaluate_holdout(
                 builder,
                 clf,
-                _load_eval_csv(args.eval_csv),
+                eval_df,
                 threshold=args.threshold,
             )
+            sweep_metrics = threshold_sweep(
+                builder,
+                clf,
+                eval_df,
+                thresholds=thresholds,
+            )
         except ValueError as exc:
             print(str(exc), file=sys.stderr)
             return 1
@@ -317,6 +428,7 @@ def main(argv: list[str] | None = None) -> int:
                 artifact=artifact,
                 meta=meta,
                 holdout_metrics=holdout_metrics,
+                sweep_metrics=sweep_metrics,
             ),
             indent=2,
             sort_keys=True,
@@ -326,6 +438,7 @@ def main(argv: list[str] | None = None) -> int:
             artifact=artifact,
             meta=meta,
             holdout_metrics=holdout_metrics,
+            sweep_metrics=sweep_metrics,
         )
     if args.out is None:
         print(report)
diff --git a/tests/test_report.py b/tests/test_report.py
