benchmark_report.py

"""Summarize verification report and exit non-zero on verification failure."""

from __future__ import annotations

import argparse
import json
from pathlib import Path


def parse_args() -> argparse.Namespace:
    """Parse command-line arguments."""
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--report-path",
        type=Path,
        default=Path("artifacts/report.json"),
        help="Path to report generated by the ServerApp.",
    )
    return parser.parse_args()


def main() -> int:
    """Print benchmark-ready verification summary."""
    args = parse_args()
    report = json.loads(args.report_path.read_text(encoding="utf-8"))

    verification_rounds = report.get("verification_rounds", [])
    if not verification_rounds:
        print("No verification rounds found in report.")
        return 1

    failed = [item for item in verification_rounds if not item.get("passed", False)]

    print("round\tnum_replies\tmax_abs_diff\tpassed")
    for item in verification_rounds:
        print(
            f"{item['round']}\t{item['num_replies']}\t"
            f"{item['max_abs_diff']:.8e}\t{int(item['passed'])}"
        )

    if failed:
        print(f"Verification failed in {len(failed)} rounds.")
        return 2

    max_diff = max(float(item["max_abs_diff"]) for item in verification_rounds)
    print(f"All rounds verified. Max observed absolute difference: {max_diff:.8e}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())