Make benchmarks easy to use in OSS

- Add __init__.py to make cinderx.benchmarks importable as a package
- Add __main__.py unified runner (python -m cinderx.benchmarks)
- Add symlink from PythonLib/cinderx/benchmarks -> ../../benchmarks
- Fix fastmark.py to find pyperformance via import instead of filesystem
- Add requirements-fastmark.txt and requirements-torchrec.txt
- Add comprehensive README with lightweight/heavyweight/JIT categories
- Add --iterations flag for configurable iteration count
- Update compile_time.py docstring with OSS usage

Author: Subbarao Garlapati

cinderx/PythonLib/cinderx/benchmarks

@@ -0,0 +1 @@
+../../benchmarks

cinderx/benchmarks/README.md

@@ -0,0 +1,104 @@
+# CinderX Benchmarks
+
+Benchmarks for measuring CinderX JIT performance on real-world Python workloads.
+
+## Quick Start
+
+```bash
+pip install cinderx
+python -m cinderx.benchmarks
+python -m cinderx.benchmarks --iterations 10
+```
+
+This runs all benchmarks whose dependencies are available, skipping the rest with
+a helpful message about what to install. Lightweight benchmarks run with 3 iterations
+by default — use `--iterations N` for more:
+
+## Lightweight Benchmarks
+
+These benchmarks have no extra dependencies beyond cinderx itself:
+
+```bash
+python -m cinderx.benchmarks.binary_trees 5
+python -m cinderx.benchmarks.fannkuch 3
+python -m cinderx.benchmarks.nbody 3
+python -m cinderx.benchmarks.richards 3
+python -m cinderx.benchmarks.spectral_norm 3
+```
+
+The numeric argument controls the number of iterations (higher = longer run).
+
+## JIT Compilation Time Benchmark
+
+Measures how long the JIT takes to compile functions (not runtime performance):
+
+```bash
+python -m cinderx.benchmarks.compile_time
+```
+
+For a per-phase breakdown, pass the `jit-time` flag:
+
+```bash
+python -X jit-time='*' -m cinderx.benchmarks.compile_time
+```
+
+## Heavyweight Benchmarks
+
+These require additional dependencies to be installed.
+
+### Full Suite (fastmark)
+
+The `fastmark` benchmark runs the full pyperformance suite with CinderX:
+
+```bash
+pip install -r cinderx/benchmarks/requirements-fastmark.txt
+python -m cinderx.benchmarks.fastmark --cinderx
+```
+
+Options:
+- `--scale N` — work scale factor (default 100, lower = faster)
+- `--json output.json` — save results as JSON
+- `--cinderx` — enable the CinderX JIT
+- `benchmarks...` — run only specific benchmarks (e.g. `richards chaos`)
+
+### TorchRec Benchmarks
+
+PT2 compilation benchmarks for TorchRec models:
+
+```bash
+pip install -r cinderx/benchmarks/requirements-torchrec.txt
+python cinderx/benchmarks/torchrec_pt2/run_with_cinderx.py
+```
+
+See [torchrec_pt2/README.md](torchrec_pt2/README.md) for details.
+
+## Running Without CinderX JIT
+
+To get a baseline comparison without JIT compilation, disable it via environment variable:
+
+```bash
+# With JIT (default)
+python -m cinderx.benchmarks
+
+# Without JIT (baseline)
+CINDERJIT_DISABLE=1 python -m cinderx.benchmarks
+```
+
+## Benchmark Descriptions
+
+| Benchmark | Description |
+|-----------|-------------|
+| `binary_trees` | Allocation-heavy workload building and traversing complete binary trees |
+| `fannkuch` | Combinatorial puzzle exercising array permutations and reversals |
+| `nbody` | N-body gravitational simulation with tight floating-point loops |
+| `richards` | Operating system task scheduler simulation (object-oriented workload) |
+| `spectral_norm` | Numerical computation of the spectral norm of a matrix |
+| `compile_time` | Measures JIT compilation speed (not runtime performance) |
+| `fastmark` | Full pyperformance suite (~60 benchmarks) with CinderX integration |
+| `torchrec_pt2` | TorchRec model compilation benchmarks with PT2 |
+
+## Listing Available Benchmarks
+
+```bash
+python -m cinderx.benchmarks --list
+```

cinderx/benchmarks/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.

cinderx/benchmarks/__main__.py

@@ -0,0 +1,180 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+
+# pyre-ignore-all-errors
+
+"""
+Unified benchmark runner for CinderX.
+
+Usage:
+    python -m cinderx.benchmarks              # Run all available benchmarks
+    python -m cinderx.benchmarks binary_trees  # Run specific benchmark(s)
+    python -m cinderx.benchmarks --list        # List available benchmarks
+"""
+
+import os
+import subprocess
+import sys
+
+# Lightweight benchmarks that only require cinderx itself.
+LIGHTWEIGHT_BENCHMARKS = [
+    "binary_trees",
+    "fannkuch",
+    "nbody",
+    "richards",
+    "spectral_norm",
+]
+
+# JIT compilation time benchmarks (measure compilation speed, not runtime).
+JIT_COMPILATION_BENCHMARKS = [
+    "compile_time",
+]
+
+# Heavyweight benchmarks with extra dependencies.
+HEAVYWEIGHT_BENCHMARKS = {
+    "fastmark": "requirements-fastmark.txt",
+    "torchrec_pt2": "requirements-torchrec.txt",
+}
+
+ALL_BENCHMARK_NAMES = (
+    LIGHTWEIGHT_BENCHMARKS
+    + JIT_COMPILATION_BENCHMARKS
+    + list(HEAVYWEIGHT_BENCHMARKS.keys())
+)
+
+
+def run_benchmark(name, iterations):
+    """Run a single benchmark as a subprocess. Returns True on success."""
+    benchmarks_dir = os.path.dirname(os.path.abspath(__file__))
+
+    if name in LIGHTWEIGHT_BENCHMARKS:
+        script = os.path.join(benchmarks_dir, f"{name}.py")
+        cmd = [sys.executable, script, str(iterations)]
+    elif name == "compile_time":
+        cmd = [sys.executable, "-m", "cinderx.benchmarks.compile_time"]
+    elif name == "fastmark":
+        script = os.path.join(benchmarks_dir, "fastmark.py")
+        cmd = [sys.executable, script, "--cinderx", "--scale", "10"]
+    elif name == "torchrec_pt2":
+        script = os.path.join(benchmarks_dir, "torchrec_pt2", "run_with_cinderx.py")
+        cmd = [sys.executable, script]
+    else:
+        print(f"Unknown benchmark: {name}", file=sys.stderr)
+        return False
+
+    print(f"\n{'='*60}")
+    print(f"Running: {name}")
+    print(f"{'='*60}")
+
+    try:
+        result = subprocess.run(cmd, check=False)
+        if result.returncode != 0:
+            return False
+    except FileNotFoundError:
+        print(f"  Error: could not execute {cmd[0]}", file=sys.stderr)
+        return False
+
+    return True
+
+
+def check_deps(name):
+    """Check if a benchmark's dependencies are available."""
+    if name in LIGHTWEIGHT_BENCHMARKS or name in JIT_COMPILATION_BENCHMARKS:
+        return True
+    elif name == "fastmark":
+        try:
+            import importlib.util
+
+            return importlib.util.find_spec("pyperformance") is not None
+        except ImportError:
+            return False
+    elif name == "torchrec_pt2":
+        try:
+            import importlib.util
+
+            return importlib.util.find_spec("torchrec") is not None
+        except ImportError:
+            return False
+    return True
+
+
+def main():
+    args = sys.argv[1:]
+
+    # Parse --iterations N
+    iterations = 3
+    if "--iterations" in args:
+        idx = args.index("--iterations")
+        try:
+            iterations = int(args[idx + 1])
+            args = args[:idx] + args[idx + 2:]
+        except (IndexError, ValueError):
+            print("Error: --iterations requires an integer argument", file=sys.stderr)
+            sys.exit(1)
+
+    if "--list" in args or "-l" in args:
+        print("Available benchmarks:")
+        print()
+        print("Lightweight (no extra dependencies):")
+        for name in LIGHTWEIGHT_BENCHMARKS:
+            print(f"  {name}")
+        print()
+        print("JIT compilation time:")
+        for name in JIT_COMPILATION_BENCHMARKS:
+            print(f"  {name}")
+        print()
+        print("Heavyweight (extra dependencies required):")
+        for name, req in HEAVYWEIGHT_BENCHMARKS.items():
+            print(f"  {name}  (pip install -r {req})")
+        return
+
+    if "--help" in args or "-h" in args:
+        print(__doc__)
+        print("Options:")
+        print("  --list, -l          List available benchmarks")
+        print("  --iterations N      Number of iterations for lightweight benchmarks (default: 3)")
+        print("  --help, -h          Show this help message")
+        print()
+        print("Examples:")
+        print("  python -m cinderx.benchmarks")
+        print("  python -m cinderx.benchmarks binary_trees fannkuch")
+        print("  python -m cinderx.benchmarks compile_time")
+        return
+
+    benchmarks_to_run = args if args else ALL_BENCHMARK_NAMES
+
+    # Validate names
+    for name in benchmarks_to_run:
+        if name not in ALL_BENCHMARK_NAMES:
+            print(f"Unknown benchmark: {name}", file=sys.stderr)
+            print(f"Run 'python -m cinderx.benchmarks --list' to see available benchmarks")
+            sys.exit(1)
+
+    passed = 0
+    skipped = 0
+    failed = 0
+
+    for name in benchmarks_to_run:
+        if not check_deps(name):
+            req_file = HEAVYWEIGHT_BENCHMARKS.get(name)
+            print(f"\nSkipping {name} — missing dependencies.")
+            if req_file:
+                print(f"  Install with: pip install -r cinderx/benchmarks/{req_file}")
+            skipped += 1
+            continue
+
+        success = run_benchmark(name, iterations)
+        if success:
+            passed += 1
+        else:
+            failed += 1
+
+    print(f"\n{'='*60}")
+    print(f"Results: {passed} passed, {skipped} skipped, {failed} failed")
+    print(f"{'='*60}")
+
+    if failed:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

cinderx/benchmarks/compile_time.py

@@ -11,10 +11,12 @@
 aggregate timing statistics.
 
 Usage:
+    python -m cinderx.benchmarks.compile_time
     buck run //cinderx/benchmarks:compile-time
 
 For a per-phase breakdown of each function's compilation, pass the jit-time
 flag via Python's -X option:
+    python -X jit-time='*' -m cinderx.benchmarks.compile_time
     buck run //cinderx/benchmarks:compile-time -- -X jit-time='*'
 """
 

cinderx/benchmarks/fastmark.py

@@ -33,14 +33,17 @@ def ADD_PATH(path):
     sys.path.append(os.path.join(os.path.dirname(__file__), path))
 
 
-PYPERFORMANCE = os.path.join(os.path.dirname(__file__), "pyperformance")
-
-if not os.path.exists(PYPERFORMANCE):
+try:
+    import pyperformance
+except ImportError:
     print(
-        f"Error: pyperformance directory not found at {PYPERFORMANCE}", file=sys.stderr
+        "Error: pyperformance not found. Install it with: pip install pyperformance==1.14.0",
+        file=sys.stderr,
     )
     sys.exit(1)
 
+PYPERFORMANCE = os.path.dirname(pyperformance.__file__)
+
 BENCHMARKS = os.path.join(PYPERFORMANCE, "data-files", "benchmarks")
 
 ADD_PATH(os.path.join(BENCHMARKS, "bm_regex_compile"))

cinderx/benchmarks/requirements-fastmark.txt

@@ -0,0 +1,13 @@
+pyperformance==1.14.0
+coverage
+docutils
+dulwich
+genshi
+html5lib
+mako
+pyaes
+sqlalchemy<2.0
+sqlglot
+sympy
+tomli
+websockets

cinderx/benchmarks/requirements-torchrec.txt

@@ -0,0 +1,4 @@
+torch>=2.0
+torchrec
+fbgemm-gpu
+click