Skip to content

Commit bf88db8

Browse files
xuzhao9xuzhao9
authored
[bisect] Add pytorch inductor benchmark bisect (#170)
1 parent c42e1ae commit bf88db8

4 files changed

Lines changed: 295 additions & 0 deletions

File tree

‎.github/scripts/bisect/regression_detector.py‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
"""
2+
PyTorch Regression Detector
3+
Used toghether with TritonParse bisector for automatic bisection.
4+
5+
Envs to control the behavior:
6+
7+
- FUNCTIONAL: Detect performance or functional regression.
8+
- REPRO_CMDLINE: The repro command line to run.
9+
- BASELINE_LOG: The baseline log file to compare with.
10+
- REGRESSION_THRESHOLD: The regression threshold, default to 10%.
11+
12+
Example usage:
13+
14+
REPRO_CMDLINE="python benchmarks/dynamo/timm_models.py --performance --amp --training --cudagraphs --only inception_v3 --inductor" \
15+
BASELINE_LOG="$PWD/bisect_logs/baseline.log" REGRESSION_THRESHOLD="0.1" \
16+
tritonparseoss bisect --triton-dir $HOME/local/pytorch --test-script $PWD/.ci/bisect/regression_detector.py \
17+
--good 34cdf49 --bad 9d49044
18+
19+
"""
20+
21+
import os
22+
import subprocess
23+
from pathlib import Path
24+
25+
# the default regression threshold is 10%
26+
REGRESSION_THRESHOLD = float(os.environ.get("REGRESSION_THRESHOLD", 10.0)) / 100.0
27+
# functional or performance regression
28+
FUNCTIONAL = bool(int(os.environ["FUNCTIONAL"]))
29+
# repro command line
30+
REPRO_CMDLINE = os.environ.get("REPRO_CMDLINE", None)
31+
# baseline log file
32+
BASELINE_LOG = os.environ.get("BASELINE_LOG", None)
33+
# pytorch root dir
34+
TORCH_SRC_DIR = os.environ["PYTORCH_SRC_DIR"]
35+
36+
37+
def get_baseline(baseline_log) -> float:
38+
with open(baseline_log, "r") as f:
39+
last_line = f.readlines()[-1].strip()
40+
if last_line.endswith("x"):
41+
last_line = last_line[:-1]
42+
return float(last_line)
43+
44+
45+
def get_current_value(stdout_lines) -> float:
46+
last_line = stdout_lines[-1].strip()
47+
if last_line.endswith("x"):
48+
last_line = last_line[:-1]
49+
return float(last_line)
50+
51+
52+
if __name__ == "__main__":
53+
assert REPRO_CMDLINE is not None, "REPRO_CMDLINE is not set."
54+
cmdline = REPRO_CMDLINE.split()
55+
56+
# functional regression
57+
if FUNCTIONAL:
58+
try:
59+
subprocess.check_call(cmdline, cwd=TORCH_SRC_DIR)
60+
except subprocess.CalledProcessError as e:
61+
print(f"cmd line {cmdline} failed: {e}")
62+
exit(e.returncode)
63+
exit(0)
64+
65+
assert BASELINE_LOG and os.path.exists(BASELINE_LOG), (
66+
f"BASELINE_LOG is not set or to a non-exist location: {BASELINE_LOG}."
67+
)
68+
baseline_signal = get_baseline(BASELINE_LOG)
69+
p = subprocess.Popen(cmdline, cwd=TORCH_SRC_DIR, stdout=subprocess.PIPE, stderr=None)
70+
assert p.stdout is not None
71+
stdout_lines = []
72+
for line in p.stdout:
73+
decoded_line = line.decode("utf-8").strip()
74+
print(decoded_line)
75+
stdout_lines.append(decoded_line)
76+
rc = p.wait()
77+
# if subprocess failed, exit with the return code
78+
if not rc == 0:
79+
exit(rc)
80+
# otherwise, check for the perf regression or accuracy regression
81+
current_value = get_current_value(stdout_lines)
82+
if current_value == 0 and "accuracy" in REPRO_CMDLINE:
83+
print("Accuracy test failed, exit with 1.")
84+
exit(1)
85+
smaller_value = min(baseline_signal, current_value)
86+
larger_value = max(baseline_signal, current_value)
87+
assert smaller_value > 0, "smaller_value should be positive, got zero."
88+
ratio = (larger_value - smaller_value) / smaller_value * 100
89+
if larger_value > smaller_value * (1 + REGRESSION_THRESHOLD):
90+
print(
91+
f"Regression detected: current value {current_value}, {larger_value} / {smaller_value} - 1 == {ratio}% , threshold {REGRESSION_THRESHOLD * 100}%)"
92+
)
93+
exit(1)
94+
else:
95+
print(
96+
f"No regression detected: current value {current_value}, {larger_value} / {smaller_value} - 1 == {ratio}%, threshold {REGRESSION_THRESHOLD * 100}%)"
97+
)
98+
exit(0)

‎.github/scripts/bisect/run.sh‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
6+
readonly DETECTOR="${WORKSPACE_DIR}/.ci/bisect/regression_detector.py"
7+
8+
required_envs=(
9+
WORKSPACE_DIR
10+
GOOD_COMMIT
11+
BAD_COMMIT
12+
PYTORCH_SRC_DIR
13+
PYTORCH_REPO
14+
VISION_SRC_DIR
15+
CUDA_HOME
16+
FUNCTIONAL
17+
REPRO_CMDLINE
18+
)
19+
20+
for env_name in "${required_envs[@]}"; do
21+
if [[ -z "${!env_name:-}" ]]; then
22+
echo "Missing required environment variable: ${env_name}" >&2
23+
exit 1
24+
fi
25+
done
26+
27+
checkout_pytorch_commit() {
28+
local repo_dir="$1"
29+
local commit="$2"
30+
cd "${repo_dir}"
31+
git reset --hard origin/main
32+
git checkout --detach "${commit}"
33+
git submodule sync --recursive
34+
git submodule update --init --recursive
35+
cd -
36+
}
37+
38+
readonly LOG_DIR="${WORKSPACE_DIR}/bisect_logs"
39+
40+
mkdir -p "${LOG_DIR}"
41+
42+
# step 1: setup pytorch build environment
43+
tritonparse_dir=$(dirname $(python -c "import tritonparse; print(tritonparse.__file__)"))
44+
bash ${tritonparse_dir}/bisect/scripts/prepare_build_pytorch.sh
45+
46+
# step 2: build and run the good commit (baseline)
47+
BASELINE_LOG="${LOG_DIR}/baseline.log"
48+
checkout_pytorch_commit "${PYTORCH_SRC_DIR}" "${GOOD_COMMIT}"
49+
bash ${tritonparse_dir}/bisect/scripts/build_pytorch.sh
50+
eval ${REPRO_CMDLINE} 2>&1 | tee "${BASELINE_LOG}"
51+
52+
# step 3: build and run the bad commit
53+
checkout_pytorch_commit "${PYTORCH_SRC_DIR}" "${BAD_COMMIT}"
54+
bash ${tritonparse_dir}/bisect/scripts/build_pytorch.sh
55+
# allow the regression detector to exit with error code
56+
set +e
57+
BASELINE_LOG="${BASELINE_LOG}" python ./.ci/bisect/regression_detector.py
58+
PREFLIGHT_RC=$?
59+
set -e
60+
61+
# if no regression, exit early and report error: this shouldn't happen
62+
if [ ${PREFLIGHT_RC} -eq 0 ]; then
63+
echo "ERROR: No regression detected on bad commit (${BAD_COMMIT}) relative to good commit (${GOOD_COMMIT})."
64+
echo "The regression detector exited with 0, meaning the bad commit behaves the same as the good commit."
65+
echo "Please verify that your good_commit and bad_commit are correct, or adjust the REGRESSION_THRESHOLD (currently ${REGRESSION_THRESHOLD}%)."
66+
exit 1
67+
elif [ ${PREFLIGHT_RC} -ne 1 ] && [ ${FUNCTIONAL} -ne 1 ]; then
68+
echo "WARNING: Pre-flight regression check exited with unexpected code ${PREFLIGHT_RC}."
69+
echo "This may indicate a build or environment issue. Proceeding with bisect anyway."
70+
fi
71+
72+
# kick off the bisect!
73+
BASELINE_LOG="${BASELINE_LOG}" USE_UV=0 \
74+
tritonparseoss bisect \
75+
--no-tui \
76+
--target torch \
77+
--torch-dir "${PYTORCH_SRC_DIR}" \
78+
--test-script "${DETECTOR}" \
79+
--good "${GOOD_COMMIT}" \
80+
--bad "${BAD_COMMIT}" \
81+
--log-dir "${LOG_DIR}" \
82+
--per-commit-log

‎.github/workflows/pytorch-bisect.yaml‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
name: PyTorch Bisect
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
benchmark:
7+
description: Benchmark to bisect
8+
required: true
9+
type: choice
10+
default: inductor
11+
options:
12+
- inductor
13+
type:
14+
description: Bisect type
15+
required: true
16+
type: choice
17+
default: performance
18+
options:
19+
- functional
20+
- performance
21+
runner:
22+
description: Hardware runner
23+
required: true
24+
type: choice
25+
default: linux.dgx.b200
26+
options:
27+
- linux.dgx.b200
28+
- linux.aws.h100
29+
repro_cmdline:
30+
description: The command line to reproduce
31+
required: true
32+
type: string
33+
good_commit:
34+
description: Last known good PyTorch commit
35+
required: true
36+
type: string
37+
bad_commit:
38+
description: First known bad PyTorch commit
39+
required: true
40+
type: string
41+
regression_threshold:
42+
type: number
43+
default: 10
44+
description: |
45+
Performance regression threshold in %
46+
47+
jobs:
48+
bisect:
49+
name: Run PyTorch Bisect
50+
runs-on: ${{ inputs.runner }}
51+
env:
52+
WORKSPACE_DIR: ${{ github.workspace }}
53+
GOOD_COMMIT: ${{ inputs.good_commit }}
54+
BAD_COMMIT: ${{ inputs.bad_commit }}
55+
REGRESSION_THRESHOLD: ${{ inputs.regression_threshold }}
56+
PYTORCH_REPO: pytorch/pytorch
57+
FUNCTIONAL: ${{ inputs.type == 'functional' && '1' || '0' }}
58+
REPRO_CMDLINE: ${{ inputs.command_arguments }}
59+
CUDA_HOME: ${{ github.workspace }}/torch-benchmarks/cuda
60+
permissions:
61+
id-token: write
62+
contents: read
63+
steps:
64+
- name: Checkout repository
65+
uses: actions/checkout@v4
66+
67+
- name: Checkout tritonparse
68+
uses: actions/checkout@v4
69+
with:
70+
repository: pytorch-labs/tritonparse
71+
path: torch-benchmarks/tritonparse
72+
ref: xz9/add-torch-bisect
73+
fetch-depth: 0
74+
75+
- name: Checkout PyTorch
76+
uses: actions/checkout@v4
77+
with:
78+
repository: pytorch/pytorch
79+
path: torch-benchmarks/pytorch
80+
submodules: recursive
81+
fetch-depth: 0
82+
83+
- name: Checkout torchvision
84+
uses: actions/checkout@v4
85+
with:
86+
repository: pytorch/vision
87+
path: torch-benchmarks/torchvision
88+
fetch-depth: 0
89+
90+
- name: Install uv environment
91+
uses: pytorch/test-infra/.github/actions/setup-uv@main
92+
93+
- name: Prepare bisect environment
94+
shell: bash
95+
run: |
96+
set -eux
97+
uv venv "${GITHUB_WORKSPACE}/torch-benchmarks/venv"
98+
export VIRTUAL_ENV="${GITHUB_WORKSPACE}/torch-benchmarks/venv"
99+
echo "VIRTUAL_ENV=${VIRTUAL_ENV}" >> $GITHUB_ENV
100+
echo "PYTORCH_SRC_DIR=$(realpath torch-benchmarks/pytorch)" >> $GITHUB_ENV
101+
echo "VISION_SRC_DIR=$(realpath torch-benchmarks/torchvision)" >> $GITHUB_ENV
102+
uv pip install -e torch-benchmarks/tritonparse
103+
104+
- name: Run PyTorch bisect
105+
shell: bash
106+
run: |
107+
uv run .github/scripts/bisect/run.sh
108+
109+
- uses: actions/upload-artifact@v4
110+
if: always()
111+
with:
112+
name: pytorch-bisect-logs
113+
path: ${{ github.workspace }}/bisect_logs
114+
retention-days: 30

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,4 @@ output/
22
*.log
33
*.xml
44
vllm-benchmarks/commit
5+
__pycache__/

0 commit comments

Comments
 (0)