feat(report): compare backend artifacts

Author: AlisinaDevelo

CHANGELOG.md

@@ -17,6 +17,7 @@
 - Holdout-selected `decision_threshold` metadata from `quorabust-train`.
 - `quorabust-validate-report` release gate for machine-readable model-card JSON.
 - Optional `--feature-backend cross-encoder` for modern transformer pair scoring.
+- `quorabust-report --compare-model label=path` for same-holdout backend comparisons.
 
 ### Fixed
 - Capped NumPy below 2.5 so Mypy can parse dependency stubs with the configured

README.md

@@ -86,6 +86,8 @@ comparable model claims; the command accepts the same `question1`, `question2`,
 `is_duplicate` column contract as training.
 Use `quorabust-validate-report --require-holdout --require-calibration` to fail release
 jobs when a JSON model card is missing required review fields.
+Use repeated `--compare-model label=path` flags to compare TF-IDF, embedding, and
+cross-encoder artifacts against the same holdout split.
 See [docs/REPORTING.md](docs/REPORTING.md) for the CI smoke workflow and
 real-evaluation checklist.
 

docs/REPORTING.md

@@ -93,6 +93,24 @@ quorabust-validate-report \
   --require-calibration
 ```
 
+Use repeated `--compare-model label=path` arguments when you need to compare trained
+backends against the same holdout split:
+
+```bash
+quorabust-report \
+  --model models/quorabust-tfidf.pkl \
+  --artifact-label quorabust-tfidf.pkl \
+  --compare-model tfidf=models/quorabust-tfidf.pkl \
+  --compare-model embedding=models/quorabust-embedding.pkl \
+  --compare-model cross=models/quorabust-cross.pkl \
+  --eval-csv data/processed/holdout.csv \
+  --format json \
+  --out reports/backend-comparison.json
+```
+
+The comparison rows are sorted by F1. Treat the table as meaningful only when all
+artifacts use the same holdout CSV, threshold, and metric code.
+
 When `quorabust-train` has a holdout split, it stores a selected `decision_threshold` in
 artifact metadata. The threshold is chosen from `--thresholds` by maximizing
 `--threshold-metric` (default `f1`). Serving uses that artifact threshold unless the

src/quorabust/report.py

@@ -390,6 +390,58 @@ def render_model_card(
     return "\n".join(parts)
 
 
+def _comparison_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    return sorted(rows, key=lambda row: row.get("f1", 0.0), reverse=True)
+
+
+def render_comparison_report(rows: list[dict[str, Any]]) -> str:
+    """Render comparable artifact metrics as a Markdown table."""
+    metric_rows = [
+        [
+            row["artifact"],
+            row.get("feature_backend", ""),
+            row.get("threshold", ""),
+            row.get("f1", ""),
+            row.get("precision", ""),
+            row.get("recall", ""),
+            row.get("accuracy", ""),
+            row.get("roc_auc", ""),
+            row.get("log_loss", ""),
+        ]
+        for row in _comparison_rows(rows)
+    ]
+    return "\n".join(
+        [
+            "# Quorabust Model Comparison",
+            "",
+            "## Backend Comparison",
+            "",
+            _markdown_table(
+                [
+                    "artifact",
+                    "feature_backend",
+                    "threshold",
+                    "f1",
+                    "precision",
+                    "recall",
+                    "accuracy",
+                    "roc_auc",
+                    "log_loss",
+                ],
+                metric_rows,
+            ),
+            "",
+            "## Caveats",
+            "",
+            (
+                "Compare rows only when every artifact was evaluated against the same "
+                "holdout CSV, threshold policy, and metric code."
+            ),
+            "",
+        ]
+    )
+
+
 def build_report_payload(
     *,
     artifact: str,
@@ -447,6 +499,50 @@ def build_report_payload(
     return payload
 
 
+def _parse_compare_model(raw: str) -> tuple[str, Path]:
+    if "=" not in raw:
+        raise ValueError("--compare-model must use label=path")
+    label, value = raw.split("=", 1)
+    label = label.strip()
+    path = Path(value.strip())
+    if not label:
+        raise ValueError("--compare-model label cannot be empty")
+    if not path.is_file():
+        raise ValueError(f"File not found: {path}")
+    return label, path
+
+
+def _comparison_metrics(
+    label: str,
+    path: Path,
+    eval_df: pd.DataFrame,
+    *,
+    threshold: float,
+    calibration_bins: int,
+) -> dict[str, Any]:
+    builder, clf, meta = load_classifier(path)
+    metrics = evaluate_holdout(
+        builder,
+        clf,
+        eval_df,
+        threshold=threshold,
+        calibration_bins=calibration_bins,
+    )
+    return {
+        "artifact": label,
+        "feature_backend": meta.get("feature_backend", ""),
+        "threshold": metrics.get("threshold"),
+        "accuracy": metrics.get("accuracy"),
+        "precision": metrics.get("precision"),
+        "recall": metrics.get("recall"),
+        "f1": metrics.get("f1"),
+        "roc_auc": metrics.get("roc_auc"),
+        "log_loss": metrics.get("log_loss"),
+        "positive_rate": metrics.get("positive_rate"),
+        "predicted_positive_rate": metrics.get("predicted_positive_rate"),
+    }
+
+
 def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(
         description="Generate a model report for a Quorabust artifact.",
@@ -480,6 +576,12 @@ def main(argv: list[str] | None = None) -> int:
         default=None,
         help="Public artifact label to print instead of the local model path",
     )
+    parser.add_argument(
+        "--compare-model",
+        action="append",
+        default=[],
+        help="Compare another artifact on the same eval CSV, formatted as label=path",
+    )
     parser.add_argument(
         "--format",
         choices=["markdown", "json"],
@@ -507,6 +609,7 @@ def main(argv: list[str] | None = None) -> int:
     builder, clf, meta = load_classifier(args.model)
     holdout_metrics = None
     sweep_metrics = None
+    comparison_metrics = None
     if args.eval_csv is not None:
         if not args.eval_csv.is_file():
             print(f"File not found: {args.eval_csv}", file=sys.stderr)
@@ -526,22 +629,37 @@ def main(argv: list[str] | None = None) -> int:
                 eval_df,
                 thresholds=thresholds,
             )
+            if args.compare_model:
+                comparison_metrics = [
+                    _comparison_metrics(
+                        label,
+                        path,
+                        eval_df,
+                        threshold=args.threshold,
+                        calibration_bins=args.calibration_bins,
+                    )
+                    for label, path in (_parse_compare_model(raw) for raw in args.compare_model)
+                ]
         except ValueError as exc:
             print(str(exc), file=sys.stderr)
             return 1
+    elif args.compare_model:
+        print("--compare-model requires --eval-csv", file=sys.stderr)
+        return 1
 
     artifact = args.artifact_label or str(args.model.resolve())
     if args.format == "json":
-        report = json.dumps(
-            build_report_payload(
-                artifact=artifact,
-                meta=meta,
-                holdout_metrics=holdout_metrics,
-                sweep_metrics=sweep_metrics,
-            ),
-            indent=2,
-            sort_keys=True,
+        payload = build_report_payload(
+            artifact=artifact,
+            meta=meta,
+            holdout_metrics=holdout_metrics,
+            sweep_metrics=sweep_metrics,
         )
+        if comparison_metrics is not None:
+            payload["comparison"] = _comparison_rows(comparison_metrics)
+        report = json.dumps(payload, indent=2, sort_keys=True)
+    elif comparison_metrics is not None:
+        report = render_comparison_report(comparison_metrics)
     else:
         report = render_model_card(
             artifact=artifact,

tests/test_report.py

@@ -9,6 +9,7 @@
     calibration_summary,
     evaluate_holdout,
     main,
+    render_comparison_report,
     render_model_card,
     threshold_sweep,
 )
@@ -145,6 +146,37 @@ def test_threshold_sweep_returns_tradeoff_rows(tmp_path):
     assert all(0.0 <= row["recall"] <= 1.0 for row in rows)
 
 
+def test_render_comparison_report_sorts_by_f1():
+    report = render_comparison_report(
+        [
+            {
+                "artifact": "tfidf",
+                "feature_backend": "tfidf",
+                "threshold": 0.5,
+                "accuracy": 0.7,
+                "precision": 0.7,
+                "recall": 0.7,
+                "f1": 0.7,
+                "roc_auc": 0.75,
+                "log_loss": 0.5,
+            },
+            {
+                "artifact": "cross",
+                "feature_backend": "cross-encoder",
+                "threshold": 0.5,
+                "accuracy": 0.9,
+                "precision": 0.9,
+                "recall": 0.9,
+                "f1": 0.9,
+                "roc_auc": 0.95,
+                "log_loss": 0.2,
+            },
+        ]
+    )
+    assert report.index("| cross |") < report.index("| tfidf |")
+    assert "## Backend Comparison" in report
+
+
 def test_report_cli_writes_model_card(tmp_path):
     model, _, _ = _artifact(tmp_path)
     eval_csv = tmp_path / "eval.csv"
@@ -208,6 +240,41 @@ def test_report_cli_writes_json_payload(tmp_path):
     assert len(payload["threshold_sweep"]) == 3
 
 
+def test_report_cli_writes_comparison_json(tmp_path):
+    model, _, _ = _artifact(tmp_path)
+    eval_csv = tmp_path / "eval.csv"
+    _df().to_csv(eval_csv, index=False)
+    out = tmp_path / "COMPARE.json"
+
+    assert (
+        main(
+            [
+                "--model",
+                str(model),
+                "--compare-model",
+                f"tfidf={model}",
+                "--eval-csv",
+                str(eval_csv),
+                "--format",
+                "json",
+                "--out",
+                str(out),
+            ]
+        )
+        == 0
+    )
+
+    payload = json.loads(out.read_text(encoding="utf-8"))
+    assert payload["artifact"] == str(model.resolve())
+    assert payload["comparison"][0]["artifact"] == "tfidf"
+    assert payload["comparison"][0]["feature_backend"] == "tfidf"
+
+
+def test_report_cli_rejects_comparison_without_eval_csv(tmp_path):
+    model, _, _ = _artifact(tmp_path)
+    assert main(["--model", str(model), "--compare-model", f"tfidf={model}"]) == 1
+
+
 def test_report_cli_rejects_bad_threshold(tmp_path):
     model, _, _ = _artifact(tmp_path)
     assert main(["--model", str(model), "--threshold", "1.5"]) == 1