gen_report.py

#!/usr/bin/env python3
# gen_report.py
import argparse
import os
import pandas as pd
from datetime import datetime

REPORT_PATH = "results/summary.md"

def load(csv_path: str) -> pd.DataFrame:
    df = pd.read_csv(csv_path)
    # required columns
    required = {
        "model","device","precision","batch",
        "p50_ms","p90_ms","p99_ms","throughput_sps","iters",
        "torch","python","machine"
    }
    missing = required - set(df.columns)
    if missing:
        raise SystemExit(f"[error] CSV missing columns: {missing}")
    # normalize types
    df["batch"] = df["batch"].astype(int)
    for c in ["p50_ms","p90_ms","p99_ms","throughput_sps"]:
        df[c] = df[c].astype(float)
    return df

def fmt(v, n=2):
    try:
        return f"{float(v):.{n}f}"
    except Exception:
        return str(v)

def best_latency(df_model):
    # best = minimum p50_ms
    return df_model.loc[df_model["p50_ms"].idxmin()]

def best_throughput(df_model):
    # best = maximum throughput
    return df_model.loc[df_model["throughput_sps"].idxmax()]

def cpu_mps_speedup_rows(df):
    rows = []
    for model in sorted(df["model"].unique()):
        batches = sorted(df[df.model == model]["batch"].unique())
        for b in batches:
            cpu = df[(df.model == model) & (df.batch == b) & (df.device == "cpu") & (df.precision == "fp32")]
            mps16 = df[(df.model == model) & (df.batch == b) & (df.device == "mps") & (df.precision == "fp16")]
            mps32 = df[(df.model == model) & (df.batch == b) & (df.device == "mps") & (df.precision == "fp32")]
            if not cpu.empty and (not mps16.empty or not mps32.empty):
                cpu_p50 = cpu["p50_ms"].mean()
                if not mps16.empty:
                    mps_p50 = mps16["p50_ms"].mean(); pair = "cpu-fp32 vs mps-fp16"
                else:
                    mps_p50 = mps32["p50_ms"].mean(); pair = "cpu-fp32 vs mps-fp32"
                speed = cpu_p50 / mps_p50 if (cpu_p50 > 0 and mps_p50 > 0) else None
                if speed:
                    rows.append({
                        "model": model,
                        "batch": int(b),
                        "cpu_p50_ms": round(cpu_p50,2),
                        "mps_p50_ms": round(mps_p50,2),
                        "speedup_x": round(speed,2),
                        "pair": pair
                    })
    return pd.DataFrame(rows)

def section_header(title, level=2):
    return f"\n{'#'*level} {title}\n"

def write_report(df: pd.DataFrame, out_md: str, plots_dir="results/plots"):
    os.makedirs(os.path.dirname(out_md), exist_ok=True)

    # Global metadata
    torch_v = ", ".join(sorted(df["torch"].astype(str).unique()))
    py_v    = ", ".join(sorted(df["python"].astype(str).unique()))
    machines= ", ".join(sorted(df["machine"].astype(str).unique()))

    lines = []
    lines.append("# 🧪 Torch-MPS-Bench — Summary\n")
    lines.append(f"_Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}_\n")
    lines.append(f"- **PyTorch**: {torch_v}\n- **Python**: {py_v}\n- **System**: {machines}\n")
    lines.append("---\n")

    # Overall CPU→MPS speedup table
    sp = cpu_mps_speedup_rows(df)
    if not sp.empty:
        lines.append(section_header("CPU → MPS Latency Speedup (P50)"))
        lines.append(sp.to_markdown(index=False))
        lines.append("\n> Pairing preference: cpu-fp32 vs mps-fp16 (if available), else cpu-fp32 vs mps-fp32.\n")

    # Per-model sections
    for model in sorted(df["model"].unique()):
        sub = df[df.model == model].copy()
        lines.append(section_header(f"Model: {model}"))

        # Best configs
        best_lat = best_latency(sub)
        best_thr = best_throughput(sub)
        lines.append("**Best Latency (P50)**\n")
        lines.append(f"- Device: `{best_lat.device}`  Precision: `{best_lat.precision}`  Batch: `{int(best_lat.batch)}`  "
                     f"P50: **{fmt(best_lat.p50_ms)} ms**, P90: {fmt(best_lat.p90_ms)} ms, P99: {fmt(best_lat.p99_ms)} ms\n")
        lines.append("**Best Throughput**\n")
        lines.append(f"- Device: `{best_thr.device}`  Precision: `{best_thr.precision}`  Batch: `{int(best_thr.batch)}`  "
                     f"Throughput: **{fmt(best_thr.throughput_sps)} samples/s**  (P50: {fmt(best_thr.p50_ms)} ms)\n")

        # Compact table
        view_cols = ["device","precision","batch","p50_ms","p90_ms","p99_ms","throughput_sps"]
        sub_sorted = sub.sort_values(by=["device","precision","batch"])
        lines.append("\n**All Runs**\n")
        lines.append(sub_sorted[view_cols].to_markdown(index=False))

        # Embed plots if present
        lat_png = os.path.join(plots_dir, f"{model}_latency_p50.png")
        thr_png = os.path.join(plots_dir, f"{model}_throughput.png")
        if os.path.exists(lat_png):
            lines.append(f"\n![{model} Latency (P50)]({lat_png})")
        if os.path.exists(thr_png):
            lines.append(f"\n![{model} Throughput]({thr_png})")

        lines.append("\n---\n")

    with open(out_md, "w") as f:
        f.write("\n".join(lines))

    print(f"[ok] Wrote report → {out_md}")

def main():
    ap = argparse.ArgumentParser(description="Generate Markdown report from benchmark CSV.")
    ap.add_argument("--csv", default="results/bench.csv")
    ap.add_argument("--out", default=REPORT_PATH)
    ap.add_argument("--plots_dir", default="results/plots")
    args = ap.parse_args()

    df = load(args.csv)
    write_report(df, args.out, plots_dir=args.plots_dir)

if __name__ == "__main__":
    main()