Add --repeat to examples

Author: fkondej

.github/scripts/generate_benchmark_report.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+"""Generate benchmark report for GitHub Actions summary from JSON summaries."""
+
+import argparse
+import json
+import urllib.parse
+from pathlib import Path
+from typing import Dict, List, Any
+
+
+def find_latest_all_examples_file(search_dir: Path = None) -> Path:
+    """Find the latest all-examples JSON file."""
+    if search_dir is None:
+        search_dir = Path("benchmark_summaries")
+    
+    if not search_dir.exists():
+        raise FileNotFoundError(f"Directory not found: {search_dir}")
+    
+    all_summaries = list(search_dir.glob("all-examples-*.json"))
+    if not all_summaries:
+        raise FileNotFoundError(f"No all-examples-*.json files found in {search_dir}")
+    
+    # Sort by modification time, newest first
+    return max(all_summaries, key=lambda p: p.stat().st_mtime)
+
+
+def generate_report_from_json(
+    json_file: Path,
+    branch_path: str = "",
+    perfetto_host: str = "https://perfetto.irreducible.com"
+) -> str:
+    """Generate complete benchmark report from a JSON summary file."""
+    output = []
+    
+    if not json_file.exists():
+        output.append(f"JSON file not found: {json_file}")
+        return "\n".join(output)
+    
+    with open(json_file) as f:
+        data = json.load(f)
+    
+    if not data:
+        output.append("No data in summary file.")
+        return "\n".join(output)
+    
+    # Auto-detect machine name from the data
+    machine = data[0].get("machine", "") if data else ""
+    if machine:
+        output.append(f"# Benchmark Report for {machine}")
+        output.append("")
+    
+    # Generate metrics table
+    output.append("## 📈 Benchmark Metrics")
+    output.append("")
+    
+    # Group by circuit
+    circuits: Dict[str, List[Dict[str, Any]]] = {}
+    for entry in data:
+        circuit = entry.get("circuit", "unknown")
+        if circuit not in circuits:
+            circuits[circuit] = []
+        circuits[circuit].append(entry)
+    
+    for circuit in sorted(circuits.keys()):
+        output.append(f"### {circuit}")
+        
+        # Show parameters if available
+        if params := circuits[circuit][0].get("parameters"):
+            output.append(f"**Parameters:** {params}")
+        
+        output.append("")
+        output.append("| Config | Witness (ms) | Prove (ms) | Verify (ms) | Proof Size (bytes) |")
+        output.append("|--------|--------------|------------|-------------|-------------------|")
+        
+        # Sort for consistent display
+        entries = sorted(circuits[circuit], key=lambda x: (
+            x.get('threading', 'single'),
+            not x.get('fusion', False)
+        ))
+        
+        for entry in entries:
+            config = entry.get('threading', 'single')
+            if entry.get('fusion'):
+                config += "-fusion"
+            
+            witness = entry.get('avg_witness_ms', 0)
+            prove = entry.get('avg_prove_ms', 0)
+            verify = entry.get('avg_verify_ms', 0)
+            proof_size = entry.get('avg_proof_size_bytes', 0)
+            
+            output.append(f"| {config} | {witness:.2f} | {prove:.2f} | {verify:.2f} | {proof_size:.0f} |")
+        
+        output.append("")
+    
+    # Generate Perfetto trace links
+    output.append("## 📊 Perfetto Traces")
+    output.append("")
+    
+    # Group by circuit again for trace links
+    for circuit in sorted(circuits.keys()):
+        output.append(f"### {circuit}")
+        
+        if params := circuits[circuit][0].get("parameters"):
+            output.append(f"**Parameters:** {params}")
+        
+        output.append("")
+        
+        for entry in sorted(circuits[circuit], key=lambda x: (
+            x.get('threading', 'single'),
+            not x.get('fusion', False)
+        )):
+            config = entry.get('threading', 'single')
+            if entry.get('fusion'):
+                config += "-fusion"
+            
+            trace_files = entry.get('trace_files', [])
+            if trace_files:
+                links = []
+                for i, trace_path in enumerate(trace_files, 1):
+                    # Build S3 URL from the trace path
+                    # trace_path already contains timestamp: sha256/20250903-092341-80a3f42/filename.perfetto-trace
+                    if branch_path:
+                        # For CI with S3 links
+                        s3_key = f"traces/monbijou/{branch_path}/{trace_path}"
+                        trace_url = f"{perfetto_host}/{s3_key}"
+                        encoded_url = urllib.parse.quote_plus(trace_url)
+                        perfetto_ui_url = f"{perfetto_host}/#!/?url={encoded_url}"
+                        links.append(f"[{i}]({perfetto_ui_url})")
+                    else:
+                        # For local testing, just show trace file names
+                        filename = Path(trace_path).name
+                        links.append(f"[{i}]({filename})")
+                
+                if links:
+                    output.append(f"- **{config}**: {' '.join(links)}")
+        
+        output.append("")
+    
+    return "\n".join(output)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate benchmark report from JSON summary file")
+    parser.add_argument("--json-file", type=Path, default=None,
+                        help="Path to all-examples JSON file (auto-detects latest if not provided)")
+    parser.add_argument("--summaries-dir", type=Path, default=Path("benchmark_summaries"),
+                        help="Directory to search for all-examples JSON files (default: benchmark_summaries)")
+    parser.add_argument("--branch-path", type=str, default="",
+                        help="S3 branch path for Perfetto links (e.g., 'main' or 'branch-feature')")
+    parser.add_argument("--perfetto-host", type=str, default="https://perfetto.irreducible.com",
+                        help="Perfetto host URL")
+    parser.add_argument("--output", type=Path, default=None,
+                        help="Output file (default: stdout)")
+    
+    args = parser.parse_args()
+    
+    # Determine which JSON file to use
+    if args.json_file:
+        json_file = args.json_file
+        print(f"DEBUG: Using specified JSON file: {json_file}")
+    else:
+        try:
+            json_file = find_latest_all_examples_file(args.summaries_dir)
+            print(f"DEBUG: Auto-detected JSON file: {json_file}")
+        except FileNotFoundError as e:
+            print(f"DEBUG: Error finding JSON file: {e}")
+            return 1
+    
+    # Generate the report
+    try:
+        report = generate_report_from_json(
+            json_file,
+            args.branch_path,
+            args.perfetto_host
+        )
+    except Exception as e:
+        print(f"Error generating report: {e}")
+        return 1
+    
+    # Output the report
+    if args.output:
+        args.output.write_text(report)
+        print(f"Report written to {args.output}")
+    else:
+        print(report)
+    
+    return 0
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/requirements.txt

@@ -0,0 +1,2 @@
+# Requirements for run_benchmarks.py
+perfetto>=0.14.0

.github/scripts/run_benchmarks

@@ -0,0 +1,79 @@
+#!/bin/bash
+# Wrapper script for run_benchmarks.py with automatic venv management
+
+set -e  # Exit on error
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+VENV_DIR="$SCRIPT_DIR/.venv"
+REQUIREMENTS_FILE="$SCRIPT_DIR/requirements.txt"
+REQUIREMENTS_HASH_FILE="$VENV_DIR/.requirements_hash"
+PYTHON_SCRIPT="$SCRIPT_DIR/run_benchmarks.py"
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+# Function to calculate hash of requirements.txt
+get_requirements_hash() {
+    if [[ -f "$REQUIREMENTS_FILE" ]]; then
+        if command -v md5sum >/dev/null 2>&1; then
+            md5sum "$REQUIREMENTS_FILE" | cut -d' ' -f1
+        elif command -v md5 >/dev/null 2>&1; then
+            md5 -q "$REQUIREMENTS_FILE"
+        else
+            # Fallback to modification time if no hash command available
+            stat -f "%m" "$REQUIREMENTS_FILE" 2>/dev/null || stat -c "%Y" "$REQUIREMENTS_FILE" 2>/dev/null || echo "0"
+        fi
+    else
+        echo "no_requirements"
+    fi
+}
+
+# Function to check if requirements have changed
+requirements_changed() {
+    if [[ ! -f "$REQUIREMENTS_HASH_FILE" ]]; then
+        return 0  # No hash file means requirements changed (or first run)
+    fi
+    
+    current_hash=$(get_requirements_hash)
+    stored_hash=$(cat "$REQUIREMENTS_HASH_FILE" 2>/dev/null || echo "")
+    
+    [[ "$current_hash" != "$stored_hash" ]]
+}
+
+# Create venv if it doesn't exist
+if [[ ! -d "$VENV_DIR" ]]; then
+    echo -e "${YELLOW}Creating Python virtual environment...${NC}"
+    python3 -m venv "$VENV_DIR"
+    echo -e "${GREEN}Virtual environment created.${NC}"
+fi
+
+# Activate venv
+source "$VENV_DIR/bin/activate"
+
+# Check if requirements have changed and install/update if needed
+if requirements_changed; then
+    echo -e "${YELLOW}Requirements have changed. Installing/updating dependencies...${NC}"
+    
+    # Upgrade pip first
+    pip install --upgrade pip >/dev/null 2>&1
+    
+    if [[ -f "$REQUIREMENTS_FILE" ]]; then
+        pip install -r "$REQUIREMENTS_FILE"
+        
+        # Store the new hash
+        get_requirements_hash > "$REQUIREMENTS_HASH_FILE"
+        echo -e "${GREEN}Dependencies installed successfully.${NC}"
+    else
+        echo -e "${YELLOW}No requirements.txt found. Skipping dependency installation.${NC}"
+    fi
+else
+    echo -e "${GREEN}Dependencies are up to date.${NC}"
+fi
+
+# Run the Python script with all arguments passed to this wrapper
+echo -e "${GREEN}Running benchmark script...${NC}"
+echo "----------------------------------------"
+python "$PYTHON_SCRIPT" "$@"