Skip to content

Add benchmark numbers to dashboard #2260

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 5 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 70 additions & 0 deletions .github/workflows/run_microbenchmarks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
name: Microbenchmarks-Perf-Nightly

on:
pull_request:
push:
tags:
- ciflow/benchmark/*
workflow_dispatch:
schedule:
- cron: '0 3 * * *' # Run daily at 7 AM UTC

jobs:
benchmark:
runs-on: linux.aws.h100
strategy:
matrix:
torch-spec:
- '--pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu126'
steps:
- uses: actions/checkout@v4

- name: Setup miniconda
uses: pytorch/test-infra/.github/actions/setup-miniconda@main
with:
python-version: "3.9"

- name: Run benchmark
shell: bash
run: |
set -eux

# Upgrade pip
${CONDA_RUN} python -m pip install --upgrade pip

${CONDA_RUN} ls
${CONDA_RUN} bash -c 'pwd'
${CONDA_RUN} bash -c 'echo $PYTHONPATH'

# Install dependencies
${CONDA_RUN} pip install ${{ matrix.torch-spec }}
${CONDA_RUN} pip install -r dev-requirements.txt
${CONDA_RUN} pip install .

${CONDA_RUN} ls
${CONDA_RUN} bash -c 'pwd'
${CONDA_RUN} bash -c 'echo $PYTHONPATH'

# Set PYTHONPATH to current directory (.) if not set, and include the benchmarks directory
${CONDA_RUN} export PYTHONPATH="${PYTHONPATH:-$(pwd)}:$(pwd)/benchmarks"

# Create benchmark results directory
mkdir -p ${{ runner.temp }}/benchmark-results

# Run microbenchmarks
# Run microbenchmarks
${CONDA_RUN} bash -c '
export PYTHONPATH="${PYTHONPATH:-$(pwd)}:$(pwd)/benchmarks"
echo "PYTHONPATH is: $PYTHONPATH"
echo "Current directory is: $(pwd)"
python benchmarks/microbenchmarks/run_ci_microbenchmarks.py \
--config benchmarks/microbenchmarks/benchmark_config.yml \
--output "$RUNNER_TEMP/benchmark-results/microbenchmark-results.json"'

- name: Upload the benchmark results to OSS benchmark database for the dashboard
uses: pytorch/test-infra/.github/actions/upload-benchmark-results@main
with:
benchmark-results-dir: ${{ runner.temp }}/benchmark-results
dry-run: false
schema-version: v3
github-token: ${{ secrets.GITHUB_TOKEN }}
19 changes: 19 additions & 0 deletions benchmarks/microbenchmarks/benchmark_config.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# Benchmark configuration for microbenchmarks
benchmark_mode: "inference"
quantization_config_recipe_names: # Will run a baseline inference for model by default, without quantization for comparison
- "int8wo"
- "int8dq"
- "float8dq-tensor"
- "float8wo"
output_dir: "benchmarks/microbenchmarks/results"
model_params:
- name: "small_bf16_linear"
matrix_shapes:
- name: "small_sweep"
min_power: 4
max_power: 4
high_precision_dtype: "torch.bfloat16"
use_torch_compile: true
torch_compile_mode: "max-autotune"
device: "cuda"
model_type: "linear"
149 changes: 149 additions & 0 deletions benchmarks/microbenchmarks/run_ci_microbenchmarks.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
#!/usr/bin/env python3
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.

# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.

"""
CI Microbenchmark Runner for PyTorch OSS Benchmark Database

This script runs microbenchmarks for various quantization types (int8wo, int8dq, float8wo, float8dq)
and outputs results in the format required by the PyTorch OSS benchmark database.
It reuses functionality from benchmark_runner.py and only adds CI-specific code.

Usage:
python run_ci_microbenchmarks.py --config benchmark_config.yml

The YAML file should contain all necessary configuration parameters for the benchmarks.
"""

import argparse
import json
from typing import Any, Dict, List

from benchmarks.microbenchmarks.benchmark_inference import run as run_inference
from benchmarks.microbenchmarks.benchmark_runner import (
load_benchmark_configs,
)
from benchmarks.microbenchmarks.utils import clean_caches


def create_benchmark_result(
benchmark_name: str,
shape: List[int],
metric_name: str,
metric_values: List[float],
quant_type: str,
device: str,
) -> Dict[str, Any]:
"""Create a benchmark result in the PyTorch OSS benchmark database format.

Args:
benchmark_name: Name of the benchmark
shape: List of shape dimensions [M, K, N]
metric_name: Name of the metric
metric_values: List of metric values
quant_type: Quantization type
device: Device type (cuda/cpu)

Returns:
Dictionary containing the benchmark result in the required format
"""
print(
f"Creating benchmark result for {benchmark_name} with shape {shape} and metric {metric_name}"
)

# Map device to benchmark device name
benchmark_device = "h100" if device == "cuda" else device

# Format shape as M-K-N
mkn_name = f"{shape[0]}-{shape[1]}-{shape[2]}" if len(shape) == 3 else "unknown"

return {
"benchmark": (
"micro-benchmark api", # name
"inference", # mode
quant_type, # quantization technique
{},
),
"model": (
mkn_name, # name in M-K-N format
"micro-benchmark custom layer", # type
benchmark_device, # backend (device)
"torchao", # origins
{},
),
"metric": (
f"{metric_name}(wrt bf16)", # name with unit
metric_values, # benchmark_values
0.0, # TODO: Will need to define the target value
{},
),
}


def run_ci_benchmarks(config_path: str) -> List[Dict[str, Any]]:
"""Run benchmarks using configurations from YAML file and return results in OSS format.

Args:
config_path: Path to the benchmark configuration file

Returns:
List of benchmark results in the PyTorch OSS benchmark database format
"""
# Load configuration using existing function
configs = load_benchmark_configs(argparse.Namespace(config=config_path))
results = []

# Run benchmarks for each config
for config in configs:
# Run benchmark using existing function
clean_caches()
result = run_inference(config)

if result is not None:
# Create benchmark result in OSS format
benchmark_result = create_benchmark_result(
benchmark_name="TorchAO Quantization Benchmark",
shape=[config.m, config.k, config.n],
metric_name="speedup",
metric_values=[result.speedup],
quant_type=config.quantization,
device=config.device,
)
results.append(benchmark_result)

return results


def main():
parser = argparse.ArgumentParser(
description="Run microbenchmarks and output results in PyTorch OSS benchmark database format"
)
parser.add_argument(
"--config",
type=str,
required=True,
help="Path to benchmark configuration file",
)
parser.add_argument(
"--output",
type=str,
default="benchmark_results.json",
help="Path to output JSON file",
)
args = parser.parse_args()

# Run benchmarks
results = run_ci_benchmarks(args.config)

# Save results to JSON file
with open(args.output, "w") as f:
json.dump(results, f, indent=2)

print(f"Benchmark results saved to {args.output}")


if __name__ == "__main__":
main()
Loading