Performance metrics

.gitignore

@@ -30,4 +30,5 @@ docs/.ub_cache/
 
 # Temporary files
 *.tmp
-*.bak
+*.bak
+*.pyc

crates/herkos-tests/Cargo.toml

@@ -18,6 +18,9 @@ herkos-core = { version = "0.2.0", path = "../herkos-core" }
 [dev-dependencies]
 criterion = "0.8.2"
 
+[features]
+baseline_benches = []
+
 [[bench]]
 name = "herkos_runtime_benchmark"
 harness = false

crates/herkos-tests/benches/herkos_runtime_benchmark.rs

@@ -9,6 +9,7 @@ fn fibo_5_wasm_bench(c: &mut Criterion) {
     });
 }
 
+#[cfg(feature = "baseline_benches")]
 fn fibo_5_orig_bench(c: &mut Criterion) {
     c.bench_function("fib 5 plain rust", |b| b.iter(|| fibo_orig(black_box(5))));
 }
@@ -20,6 +21,7 @@ fn fibo_20_wasm_bench(c: &mut Criterion) {
     });
 }
 
+#[cfg(feature = "baseline_benches")]
 fn fibo_20_orig_bench(c: &mut Criterion) {
     c.bench_function("fib 20 plain rust", |b| b.iter(|| fibo_orig(black_box(20))));
 }
@@ -33,6 +35,7 @@ fn memsort_100_wasm_bench(c: &mut Criterion) {
     });
 }
 
+#[cfg(feature = "baseline_benches")]
 fn memsort_100_orig_bench(c: &mut Criterion) {
     c.bench_function("memsort 100 plain rust", |b| {
         b.iter(|| mem_fill_sort_sum_orig(black_box(100), black_box(42)))
@@ -48,6 +51,7 @@ fn collatz_27_wasm_bench(c: &mut Criterion) {
     });
 }
 
+#[cfg(feature = "baseline_benches")]
 fn collatz_27_orig_bench(c: &mut Criterion) {
     c.bench_function("collatz 27 plain rust", |b| {
         b.iter(|| collatz_steps_orig(black_box(27)))
@@ -61,6 +65,7 @@ fn collatz_871_wasm_bench(c: &mut Criterion) {
     });
 }
 
+#[cfg(feature = "baseline_benches")]
 fn collatz_871_orig_bench(c: &mut Criterion) {
     c.bench_function("collatz 871 plain rust", |b| {
         b.iter(|| collatz_steps_orig(black_box(871)))
@@ -76,6 +81,7 @@ fn isqrt_wasm_bench(c: &mut Criterion) {
     });
 }
 
+#[cfg(feature = "baseline_benches")]
 fn isqrt_orig_bench(c: &mut Criterion) {
     c.bench_function("isqrt 1000000 plain rust", |b| {
         b.iter(|| isqrt_orig(black_box(1_000_000)))
@@ -89,6 +95,7 @@ fn gcd_wasm_bench(c: &mut Criterion) {
     });
 }
 
+#[cfg(feature = "baseline_benches")]
 fn gcd_orig_bench(c: &mut Criterion) {
     c.bench_function("gcd(46368,28657) plain rust", |b| {
         b.iter(|| gcd_orig(black_box(46368), black_box(28657)))
@@ -104,6 +111,7 @@ fn popcount_wasm_bench(c: &mut Criterion) {
     });
 }
 
+#[cfg(feature = "baseline_benches")]
 fn popcount_orig_bench(c: &mut Criterion) {
     c.bench_function("popcount 0xDEADBEEF plain rust", |b| {
         b.iter(|| popcount_orig(black_box(0xDEADBEEFu32 as i32)))
@@ -119,12 +127,35 @@ fn sum_recursive_100_wasm_bench(c: &mut Criterion) {
     });
 }
 
+#[cfg(feature = "baseline_benches")]
 fn sum_recursive_100_orig_bench(c: &mut Criterion) {
     c.bench_function("sum_recursive 100 plain rust", |b| {
         b.iter(|| sum_recursive_orig(black_box(100)))
     });
 }
 
+#[cfg(not(feature = "baseline_benches"))]
+criterion_group!(
+    benches,
+    // Fibonacci (pure computation)
+    fibo_5_wasm_bench,
+    fibo_20_wasm_bench,
+    // Memory-intensive (bounds-checking overhead)
+    memsort_100_wasm_bench,
+    // Branchy control flow (division + conditionals)
+    collatz_27_wasm_bench,
+    collatz_871_wasm_bench,
+    // Binary search loop (multiplication + comparison)
+    isqrt_wasm_bench,
+    // Euclidean algorithm (modular arithmetic loop)
+    gcd_wasm_bench,
+    // Bitwise tight loop
+    popcount_wasm_bench,
+    // Recursive function call overhead
+    sum_recursive_100_wasm_bench,
+);
+
+#[cfg(feature = "baseline_benches")]
 criterion_group!(
     benches,
     // Fibonacci (pure computation)
@@ -153,4 +184,5 @@ criterion_group!(
     sum_recursive_100_wasm_bench,
     sum_recursive_100_orig_bench,
 );
+
 criterion_main!(benches);

scripts/bench_record.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+COMMENT="${1:-}"
+
+# Ensure we're in the repo root
+if [ ! -f Cargo.toml ]; then
+    echo "ERROR: Cargo.toml not found. Please run this script from the repo root."
+    exit 1
+fi
+
+echo "=== herkos benchmark recorder ==="
+echo "Branch: $(git rev-parse --abbrev-ref HEAD)"
+echo "Commit: $(git rev-parse --short HEAD)"
+[ -n "$COMMENT" ] && echo "Comment: $COMMENT"
+echo
+
+# Clean up stale temp files to avoid duplication
+rm -f /tmp/new_bench_rows.csv
+
+# Run benchmarks
+echo "Running benchmarks (HERKOS_OPTIMIZE=1)..."
+HERKOS_OPTIMIZE=1 cargo bench -p herkos-tests
+
+echo
+echo "Collecting results..."
+
+# Capture metadata
+export BRANCH=$(git rev-parse --abbrev-ref HEAD)
+export COMMIT_SHA=$(git rev-parse HEAD)
+export TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+
+# Parse Criterion output to CSV
+python3 scripts/collect_bench.py target/criterion /tmp/new_bench_rows.csv ${COMMENT:+--comment "$COMMENT"}
+
+echo
+echo "Pushing to metrics branch..."
+
+# Configure git if not already done
+git config user.email "bench@local" || true
+git config user.name "bench" || true
+
+CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+
+# Fetch or create metrics branch
+if git fetch origin metrics 2>/dev/null; then
+    git checkout metrics
+else
+    echo "Creating orphan metrics branch..."
+    git checkout --orphan metrics
+    git rm -rf . --quiet 2>/dev/null || true
+    printf '# metrics branch\n\nManual benchmark history for herkos.\n\nSee bench_history.csv for data.\n' > README.md
+    git add README.md
+    git commit -m "chore: init metrics branch"
+fi
+
+# Append rows to bench_history.csv
+if [ -f bench_history.csv ]; then
+    # Skip header row when appending
+    tail -n +2 /tmp/new_bench_rows.csv >> bench_history.csv
+else
+    # First time: copy entire file with header
+    cp /tmp/new_bench_rows.csv bench_history.csv
+fi
+
+git add bench_history.csv
+if ! git diff --cached --quiet; then
+    git commit -m "bench: results for ${COMMIT_SHA:0:12}"
+    git push origin metrics
+else
+    echo "No changes to bench_history.csv (likely zero benchmarks found)"
+fi
+
+# Return to original branch
+git checkout "$CURRENT_BRANCH"
+
+echo "✓ Done. Pushed to metrics branch."

scripts/collect_bench.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+"""
+collect_bench.py <criterion_dir> <output_csv> [--comment "text"]
+
+Walks Criterion JSON output, extracts timing statistics, and appends
+enriched rows to a CSV file. Creates the file with header on first run.
+
+Environment variables consumed:
+  BRANCH        - git branch name
+  COMMIT_SHA    - full commit SHA
+  TIMESTAMP     - ISO-8601 timestamp
+"""
+
+import sys
+import os
+import json
+import csv
+import pathlib
+import argparse
+
+FIELDS = [
+    "timestamp", "branch", "commit_sha",
+    "benchmark_name",
+    "mean_ns", "stddev_ns", "median_ns", "lower_ns", "upper_ns",
+    "comments",
+]
+
+
+def parse_estimates(path: pathlib.Path) -> dict:
+    """Extract timing statistics from a Criterion estimates.json file."""
+    with path.open() as f:
+        data = json.load(f)
+    return {
+        "mean_ns":   data["mean"]["point_estimate"],
+        "stddev_ns": data["std_dev"]["point_estimate"],
+        "median_ns": data["median"]["point_estimate"],
+        "lower_ns":  data["mean"]["confidence_interval"]["lower_bound"],
+        "upper_ns":  data["mean"]["confidence_interval"]["upper_bound"],
+    }
+
+
+def collect_rows(criterion_dir: pathlib.Path, metadata: dict) -> list[dict]:
+    """Walk criterion_dir and build one row per benchmark."""
+    rows = []
+    for estimates_path in sorted(criterion_dir.rglob("new/estimates.json")):
+        # Path: <criterion_dir>/<group>/<bench_name>/new/estimates.json
+        bench_name = estimates_path.parent.parent.name
+        try:
+            stats = parse_estimates(estimates_path)
+        except (KeyError, json.JSONDecodeError) as e:
+            print(f"WARNING: skipping {estimates_path}: {e}", file=sys.stderr)
+            continue
+        rows.append({**metadata, "benchmark_name": bench_name, **stats})
+    return rows
+
+
+def write_csv(output_path: pathlib.Path, rows: list[dict]) -> None:
+    """Append rows to CSV, writing header only if file is new."""
+    file_exists = output_path.exists() and output_path.stat().st_size > 0
+    with output_path.open("a", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=FIELDS)
+        if not file_exists:
+            writer.writeheader()
+        writer.writerows(rows)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Parse Criterion benchmark results into CSV rows"
+    )
+    parser.add_argument("criterion_dir", help="Root Criterion output directory")
+    parser.add_argument("output_csv", help="Output CSV file path")
+    parser.add_argument(
+        "--comment",
+        default="",
+        help="Optional comment to append to each row"
+    )
+
+    args = parser.parse_args()
+
+    criterion_dir = pathlib.Path(args.criterion_dir)
+    output_csv = pathlib.Path(args.output_csv)
+
+    if not criterion_dir.is_dir():
+        print(f"ERROR: criterion directory not found: {criterion_dir}", file=sys.stderr)
+        return 1
+
+    metadata = {
+        "timestamp":   os.environ.get("TIMESTAMP", ""),
+        "branch":      os.environ.get("BRANCH", ""),
+        "commit_sha":  os.environ.get("COMMIT_SHA", ""),
+        "comments":    args.comment,
+    }
+
+    rows = collect_rows(criterion_dir, metadata)
+
+    if not rows:
+        print("WARNING: no benchmark results found in criterion directory", file=sys.stderr)
+        return 0
+
+    write_csv(output_csv, rows)
+    print(f"Wrote {len(rows)} rows to {output_csv}")
+    for r in rows:
+        print(f"  {r['benchmark_name']}: mean={r['mean_ns']:.1f}ns")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())