feat(report): support json output

Author: AlisinaDevelo

.github/workflows/ci.yml

@@ -44,4 +44,11 @@ jobs:
             --artifact-label quorabust-smoke.pkl \
             --eval-csv examples/smoke_pairs.csv \
             --out /tmp/quorabust-smoke-model-card.md
+          quorabust-report \
+            --model /tmp/quorabust-smoke.pkl \
+            --artifact-label quorabust-smoke.pkl \
+            --eval-csv examples/smoke_pairs.csv \
+            --format json \
+            --out /tmp/quorabust-smoke-model-card.json
           test -s /tmp/quorabust-smoke-model-card.md
+          test -s /tmp/quorabust-smoke-model-card.json

CHANGELOG.md

@@ -7,6 +7,7 @@
   optional labeled holdout CSVs.
 - `GET /models` serving endpoint for safe loaded-model metadata.
 - `examples/smoke_pairs.csv` and CI coverage for the train-to-report workflow.
+- JSON output for `quorabust-report --format json`.
 
 ### Fixed
 - Capped NumPy below 2.5 so Mypy can parse dependency stubs with the configured

README.md

@@ -71,10 +71,11 @@ quorabust-report \
 ```
 
 The report includes artifact metadata, persisted training/eval metrics, optional holdout
-metrics, and a threshold confusion matrix. Use a real held-out CSV for comparable model
-claims; the command accepts the same `question1`, `question2`, `is_duplicate` column
-contract as training. See [docs/REPORTING.md](docs/REPORTING.md) for the CI smoke
-workflow and real-evaluation checklist.
+metrics, and a threshold confusion matrix. Use `--format json` for machine-readable CI or
+release artifacts. Use a real held-out CSV for comparable model claims; the command
+accepts the same `question1`, `question2`, `is_duplicate` column contract as training.
+See [docs/REPORTING.md](docs/REPORTING.md) for the CI smoke workflow and
+real-evaluation checklist.
 
 ### Load a saved model
 

docs/REPORTING.md

@@ -21,6 +21,13 @@ quorabust-report \
   --artifact-label quorabust-smoke.pkl \
   --eval-csv examples/smoke_pairs.csv \
   --out /tmp/quorabust-smoke-model-card.md
+
+quorabust-report \
+  --model /tmp/quorabust-smoke.pkl \
+  --artifact-label quorabust-smoke.pkl \
+  --eval-csv examples/smoke_pairs.csv \
+  --format json \
+  --out /tmp/quorabust-smoke-model-card.json
 ```
 
 The smoke dataset proves the command path works. It is not a benchmark and should not be
@@ -48,3 +55,6 @@ quorabust-report \
 Record the dataset source, split method, command, commit SHA, and date next to any
 published result. Do not compare artifacts unless they use the same holdout split and
 threshold.
+
+Use `--format json` when you want CI or release tooling to compare metrics without
+scraping Markdown.

src/quorabust/report.py

@@ -15,6 +15,16 @@
 from quorabust.persist import load_classifier
 
 _REQUIRED_COLUMNS = {"question1", "question2", "is_duplicate"}
+_METADATA_KEYS = [
+    "feature_backend",
+    "feature_schema",
+    "n_train",
+    "n_eval",
+    "seed",
+    "quorabust_version",
+    "git_revision",
+    "csv_sha256",
+]
 
 
 def _package_version() -> str:
@@ -53,6 +63,18 @@ def _markdown_table(headers: list[str], rows: list[list[Any]]) -> str:
     return "\n".join(out)
 
 
+def _metadata_from_meta(meta: dict[str, Any]) -> dict[str, Any]:
+    return {k: meta[k] for k in _METADATA_KEYS if k in meta}
+
+
+def _persisted_metrics_from_meta(meta: dict[str, Any]) -> dict[str, float]:
+    return {
+        k.removeprefix("eval_"): float(v)
+        for k, v in sorted(meta.items())
+        if k.startswith("eval_") and isinstance(v, int | float)
+    }
+
+
 def evaluate_holdout(
     builder: Any,
     clf: Any,
@@ -100,22 +122,8 @@ def render_model_card(
         ["generated_by", f"Quorabust {_package_version()}"],
     ]
 
-    metadata_keys = [
-        "feature_backend",
-        "feature_schema",
-        "n_train",
-        "n_eval",
-        "seed",
-        "quorabust_version",
-        "git_revision",
-        "csv_sha256",
-    ]
-    metadata_rows = [[k, meta[k]] for k in metadata_keys if k in meta]
-    persisted_metric_rows = [
-        [k.removeprefix("eval_"), meta[k]]
-        for k in sorted(meta)
-        if k.startswith("eval_") and isinstance(meta[k], int | float)
-    ]
+    metadata_rows = [[k, v] for k, v in _metadata_from_meta(meta).items()]
+    persisted_metric_rows = [[k, v] for k, v in _persisted_metrics_from_meta(meta).items()]
 
     parts = [
         "# Quorabust Model Card",
@@ -200,9 +208,56 @@ def render_model_card(
     return "\n".join(parts)
 
 
+def build_report_payload(
+    *,
+    artifact: str,
+    meta: dict[str, Any],
+    holdout_metrics: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Build a machine-readable report payload for CI and model comparisons."""
+    payload: dict[str, Any] = {
+        "artifact": artifact,
+        "generated_by": f"Quorabust {_package_version()}",
+        "intended_use": (
+            "Scores pairs of short natural-language questions and returns the probability "
+            "that the pair is semantically duplicate."
+        ),
+        "training_metadata": _metadata_from_meta(meta),
+        "persisted_evaluation": _persisted_metrics_from_meta(meta),
+        "serving_contract": {
+            "predict": "POST /predict",
+            "metrics": "GET /metrics",
+            "input": {"question1": "list[str]", "question2": "list[str]"},
+            "output": {"proba_duplicate": "list[float]"},
+        },
+        "caveats": [
+            "Performance depends on the training data distribution and threshold.",
+            "Re-run on a current holdout set before comparing artifacts.",
+        ],
+    }
+    if holdout_metrics is not None:
+        payload["holdout_evaluation"] = {
+            k: v
+            for k, v in holdout_metrics.items()
+            if k not in {"tn", "fp", "fn", "tp"}
+        }
+        payload["confusion_matrix"] = {
+            "labels": ["not_duplicate", "duplicate"],
+            "actual_0": {
+                "predicted_0": holdout_metrics["tn"],
+                "predicted_1": holdout_metrics["fp"],
+            },
+            "actual_1": {
+                "predicted_0": holdout_metrics["fn"],
+                "predicted_1": holdout_metrics["tp"],
+            },
+        }
+    return payload
+
+
 def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(
-        description="Generate a Markdown model card for a Quorabust artifact.",
+        description="Generate a model report for a Quorabust artifact.",
     )
     parser.add_argument("--model", type=Path, required=True, help="Saved .pkl artifact")
     parser.add_argument(
@@ -222,7 +277,13 @@ def main(argv: list[str] | None = None) -> int:
         default=None,
         help="Public artifact label to print instead of the local model path",
     )
-    parser.add_argument("--out", type=Path, default=None, help="Write Markdown here")
+    parser.add_argument(
+        "--format",
+        choices=["markdown", "json"],
+        default="markdown",
+        help="Report format to print or write",
+    )
+    parser.add_argument("--out", type=Path, default=None, help="Write report here")
     args = parser.parse_args(argv)
 
     if not args.model.is_file():
@@ -249,16 +310,28 @@ def main(argv: list[str] | None = None) -> int:
             print(str(exc), file=sys.stderr)
             return 1
 
-    card = render_model_card(
-        artifact=args.artifact_label or str(args.model.resolve()),
-        meta=meta,
-        holdout_metrics=holdout_metrics,
-    )
+    artifact = args.artifact_label or str(args.model.resolve())
+    if args.format == "json":
+        report = json.dumps(
+            build_report_payload(
+                artifact=artifact,
+                meta=meta,
+                holdout_metrics=holdout_metrics,
+            ),
+            indent=2,
+            sort_keys=True,
+        )
+    else:
+        report = render_model_card(
+            artifact=artifact,
+            meta=meta,
+            holdout_metrics=holdout_metrics,
+        )
     if args.out is None:
-        print(card)
+        print(report)
     else:
         args.out.parent.mkdir(parents=True, exist_ok=True)
-        args.out.write_text(card, encoding="utf-8")
+        args.out.write_text(report + "\n", encoding="utf-8")
         print(f"wrote {args.out.resolve()}")
     return 0
 

tests/test_report.py

@@ -1,8 +1,10 @@
+import json
+
 import pandas as pd
 
 from quorabust.model import train_duplicate_classifier
 from quorabust.persist import save_classifier
-from quorabust.report import evaluate_holdout, main, render_model_card
+from quorabust.report import build_report_payload, evaluate_holdout, main, render_model_card
 
 
 def _df():
@@ -47,6 +49,27 @@ def test_render_model_card_includes_metadata_and_persisted_metrics():
     assert "| log_loss | 0.6100 |" in card
 
 
+def test_build_report_payload_is_machine_readable():
+    payload = build_report_payload(
+        artifact="model.pkl",
+        meta={"feature_backend": "tfidf", "eval_accuracy": 0.75, "csv": "/private/train.csv"},
+        holdout_metrics={
+            "n": 10,
+            "threshold": 0.5,
+            "accuracy": 0.8,
+            "tn": 4,
+            "fp": 1,
+            "fn": 1,
+            "tp": 4,
+        },
+    )
+    assert payload["artifact"] == "model.pkl"
+    assert payload["training_metadata"] == {"feature_backend": "tfidf"}
+    assert payload["persisted_evaluation"]["accuracy"] == 0.75
+    assert payload["confusion_matrix"]["actual_1"]["predicted_1"] == 4
+    assert "csv" not in payload["training_metadata"]
+
+
 def test_evaluate_holdout_returns_confusion_counts(tmp_path):
     _, builder, clf = _artifact(tmp_path)
     metrics = evaluate_holdout(builder, clf, _df(), threshold=0.5)
@@ -83,6 +106,36 @@ def test_report_cli_writes_model_card(tmp_path):
     assert "## Confusion Matrix" in card
 
 
+def test_report_cli_writes_json_payload(tmp_path):
+    model, _, _ = _artifact(tmp_path)
+    eval_csv = tmp_path / "eval.csv"
+    _df().to_csv(eval_csv, index=False)
+    out = tmp_path / "MODEL_CARD.json"
+
+    assert (
+        main(
+            [
+                "--model",
+                str(model),
+                "--artifact-label",
+                "smoke-model.pkl",
+                "--eval-csv",
+                str(eval_csv),
+                "--format",
+                "json",
+                "--out",
+                str(out),
+            ]
+        )
+        == 0
+    )
+
+    payload = json.loads(out.read_text(encoding="utf-8"))
+    assert payload["artifact"] == "smoke-model.pkl"
+    assert payload["holdout_evaluation"]["n"] == 30
+    assert payload["confusion_matrix"]["labels"] == ["not_duplicate", "duplicate"]
+
+
 def test_report_cli_rejects_bad_threshold(tmp_path):
     model, _, _ = _artifact(tmp_path)
     assert main(["--model", str(model), "--threshold", "1.5"]) == 1