Skip to content

Commit 3ef58c4

Browse files
huydhnhuydhn
committed
Add accuracy check
Signed-off-by: Huy Do <huydhn@gmail.com>
1 parent 5176b11 commit 3ef58c4

7 files changed

Lines changed: 91 additions & 24 deletions

File tree

‎.github/workflows/gpt-oss-benchmark.yml‎

Lines changed: 73 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ jobs:
2626
model: openai/gpt-oss-120b
2727
docker-image: rocm/vllm-dev:open-mi300-08052025
2828
# gpt-oss-20b
29-
- runner: linux.aws.h100.4
29+
- runner: linux.aws.h100
3030
model: openai/gpt-oss-20b
3131
docker-image: 'public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:6d8d0a24c02bfd84d46b3016b865a44f048ae84b'
3232
- runner: linux.dgx.b200
@@ -150,10 +150,12 @@ jobs:
150150
# vLLM-related environment variables
151151
ENGINE_VERSION: v1
152152
SAVE_TO_PYTORCH_BENCHMARK_FORMAT: 1
153+
MODEL: ${{ matrix.model }}
153154
run: |
154155
set -eux
155156
156157
if [[ "${DEVICE_TYPE}" == *B200* ]]; then
158+
# Just to unblock this change on B200
157159
aws configure set aws_access_key_id "${AWS_ACCESS_KEY_ID}"
158160
aws configure set aws_secret_access_key "${AWS_SECRET_ACCESS_KEY}"
159161
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
@@ -184,6 +186,7 @@ jobs:
184186
185187
container_name=$(docker run \
186188
${GPU_FLAG:-} \
189+
-e MODEL \
187190
-e DEVICE_NAME \
188191
-e DEVICE_TYPE \
189192
-e HF_TOKEN \
@@ -205,16 +208,76 @@ jobs:
205208
-w /tmp/workspace \
206209
"${DOCKER_IMAGE}"
207210
)
211+
212+
# Run perf tests
208213
docker exec -t "${container_name}" bash -c "
209-
cd vllm-benchmarks/vllm
214+
pushd vllm-benchmarks/vllm
210215
cp vllm/benchmarks/lib/utils.py /app/vllm-os-mini/vllm/benchmarks/utils.py || true
211216
if [[ $DEVICE_NAME != 'rocm' ]]; then
217+
pip install -U openai transformers
212218
pip install --pre vllm==0.10.1+gptoss \
213219
--extra-index-url https://wheels.vllm.ai/gpt-oss/ \
214220
--extra-index-url https://download.pytorch.org/whl/nightly/cu128
215221
fi
216-
sleep 7200
222+
pip freeze
217223
bash .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
224+
popd
225+
"
226+
227+
# Run accuracy check
228+
docker exec -t "${container_name}" bash -c "
229+
local tp
230+
if [[ $MODEL == 'openai/gpt-oss-120b' ]]; then
231+
tp=4
232+
elfi [[ $MODEL == 'openai/gpt-oss-20b' ]]; then
233+
tp=1
234+
fi
235+
236+
# Prepare the accuracy test
237+
vllm serve $MODEL --tensor_parallel_size $tp &
238+
server_pid=$!
239+
240+
wait_for_server() {
241+
timeout 1200 bash -c '
242+
until curl -X POST localhost:8000/v1/completions; do
243+
sleep 1
244+
done' && return 0 || return 1
245+
}
246+
247+
if wait_for_server; then
248+
echo 'vLLM server is up and running'
249+
else
250+
echo 'vLLM failed to start within the timeout period'
251+
fi
252+
253+
pushd vllm-benchmarks/gpt-oss
254+
mkdir -p /tmp/gpqa_openai
255+
256+
# Low
257+
OPENAI_API_KEY='' python3 -m gpt_oss.evals --base-url http://localhost:8000/v1 \
258+
--model $MODEL \
259+
--eval gpqa \
260+
--reasoning-effort low \
261+
--n-threads $(expr $(nproc) / 2)
262+
263+
# Mid
264+
OPENAI_API_KEY='' python3 -m gpt_oss.evals --base-url http://localhost:8000/v1 \
265+
--model $MODEL \
266+
--eval gpqa \
267+
--reasoning-effort medium \
268+
--n-threads $(expr $(nproc) / 2)
269+
270+
# High
271+
OPENAI_API_KEY='' python3 -m gpt_oss.evals --base-url http://localhost:8000/v1 \
272+
--model $MODEL \
273+
--eval gpqa \
274+
--reasoning-effort high \
275+
--n-threads $(expr $(nproc) / 2)
276+
277+
mv /tmp/gpqa_openai .
278+
popd
279+
280+
kill -9 $server_pid
218281
"
219282
220283
- name: Authenticate with AWS
@@ -256,3 +319,10 @@ jobs:
256319
with:
257320
name: benchmark-results--${{ env.SANITIZED_DEVICE_TYPE }}-${{ env.SANITIZED_MODEL }}
258321
path: vllm-benchmarks/vllm/benchmarks/results
322+
323+
# Keep a copy of the accuracy results on GitHub for reference
324+
- uses: actions/upload-artifact@v4
325+
with:
326+
name: accuracy-results--${{ env.SANITIZED_DEVICE_TYPE }}-${{ env.SANITIZED_MODEL }}
327+
path: |
328+
vllm-benchmarks/gpt-oss/gpqa_openai

‎vllm-benchmarks/benchmarks/cuda/latency-tests.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@
5959
"load_format": "dummy",
6060
"num_iters_warmup": 5,
6161
"num_iters": 15,
62-
"max_model_len": 4096
62+
"max_model_len": 8192
6363
}
6464
},
6565
{
@@ -70,7 +70,7 @@
7070
"load_format": "dummy",
7171
"num_iters_warmup": 5,
7272
"num_iters": 15,
73-
"max_model_len": 4096
73+
"max_model_len": 8192
7474
}
7575
},
7676
{
@@ -81,7 +81,7 @@
8181
"load_format": "dummy",
8282
"num_iters_warmup": 5,
8383
"num_iters": 15,
84-
"max_model_len": 4096
84+
"max_model_len": 8192
8585
}
8686
}
8787
]

‎vllm-benchmarks/benchmarks/cuda/serving-tests.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -422,7 +422,7 @@
422422
"disable_log_stats": "",
423423
"disable_log_requests": "",
424424
"load_format": "dummy",
425-
"max_model_len": 4096
425+
"max_model_len": 8192
426426
},
427427
"client_parameters": {
428428
"model": "",
@@ -442,7 +442,7 @@
442442
"disable_log_stats": "",
443443
"disable_log_requests": "",
444444
"load_format": "dummy",
445-
"max_model_len": 4096
445+
"max_model_len": 8192
446446
},
447447
"client_parameters": {
448448
"model": "",
@@ -462,7 +462,7 @@
462462
"disable_log_stats": "",
463463
"disable_log_requests": "",
464464
"load_format": "dummy",
465-
"max_model_len": 4096
465+
"max_model_len": 8192
466466
},
467467
"client_parameters": {
468468
"model": "",

‎vllm-benchmarks/benchmarks/cuda/throughput-tests.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@
6565
"dataset": "./ShareGPT_V3_unfiltered_cleaned_split.json",
6666
"num_prompts": 200,
6767
"backend": "vllm",
68-
"max_model_len": 4096
68+
"max_model_len": 8192
6969
}
7070
},
7171
{
@@ -77,7 +77,7 @@
7777
"dataset": "./ShareGPT_V3_unfiltered_cleaned_split.json",
7878
"num_prompts": 200,
7979
"backend": "vllm",
80-
"max_model_len": 4096
80+
"max_model_len": 8192
8181
}
8282
},
8383
{
@@ -89,7 +89,7 @@
8989
"dataset": "./ShareGPT_V3_unfiltered_cleaned_split.json",
9090
"num_prompts": 200,
9191
"backend": "vllm",
92-
"max_model_len": 4096
92+
"max_model_len": 8192
9393
}
9494
}
9595
]

‎vllm-benchmarks/benchmarks/rocm/latency-tests.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@
5959
"load_format": "dummy",
6060
"num_iters_warmup": 5,
6161
"num_iters": 15,
62-
"max_model_len": 4096
62+
"max_model_len": 8192
6363
}
6464
},
6565
{
@@ -70,7 +70,7 @@
7070
"load_format": "dummy",
7171
"num_iters_warmup": 5,
7272
"num_iters": 15,
73-
"max_model_len": 4096
73+
"max_model_len": 8192
7474
}
7575
},
7676
{
@@ -81,7 +81,7 @@
8181
"load_format": "dummy",
8282
"num_iters_warmup": 5,
8383
"num_iters": 15,
84-
"max_model_len": 4096
84+
"max_model_len": 8192
8585
}
8686
}
8787
]

‎vllm-benchmarks/benchmarks/rocm/serving-tests.json‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -421,8 +421,7 @@
421421
"disable_log_stats": "",
422422
"disable_log_requests": "",
423423
"load_format": "dummy",
424-
"max_model_len": 4096,
425-
"compilation-config": "{'full_cuda_graph': true}"
424+
"max_model_len": 8192
426425
},
427426
"client_parameters": {
428427
"model": "",
@@ -442,8 +441,7 @@
442441
"disable_log_stats": "",
443442
"disable_log_requests": "",
444443
"load_format": "dummy",
445-
"max_model_len": 4096,
446-
"compilation-config": "{'full_cuda_graph': true}"
444+
"max_model_len": 8192
447445
},
448446
"client_parameters": {
449447
"model": "",
@@ -463,8 +461,7 @@
463461
"disable_log_stats": "",
464462
"disable_log_requests": "",
465463
"load_format": "dummy",
466-
"max_model_len": 4096,
467-
"compilation-config": "{'full_cuda_graph': true}"
464+
"max_model_len": 8192
468465
},
469466
"client_parameters": {
470467
"model": "",

‎vllm-benchmarks/benchmarks/rocm/throughput-tests.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@
6565
"dataset": "./ShareGPT_V3_unfiltered_cleaned_split.json",
6666
"num_prompts": 200,
6767
"backend": "vllm",
68-
"max_model_len": 4096
68+
"max_model_len": 8192
6969
}
7070
},
7171
{
@@ -77,7 +77,7 @@
7777
"dataset": "./ShareGPT_V3_unfiltered_cleaned_split.json",
7878
"num_prompts": 200,
7979
"backend": "vllm",
80-
"max_model_len": 4096
80+
"max_model_len": 8192
8181
}
8282
},
8383
{
@@ -89,7 +89,7 @@
8989
"dataset": "./ShareGPT_V3_unfiltered_cleaned_split.json",
9090
"num_prompts": 200,
9191
"backend": "vllm",
92-
"max_model_len": 4096
92+
"max_model_len": 8192
9393
}
9494
}
9595
]

0 commit comments

Comments
 (0)