Skip to content

Commit a17fe1a

Browse files
committed
Move the logic to a script
Signed-off-by: Huy Do <huydhn@gmail.com>
1 parent 044f1c3 commit a17fe1a

3 files changed

Lines changed: 76 additions & 72 deletions

File tree

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#!/bin/bash
2+
3+
set -eux
4+
5+
tp=0
6+
if [[ "${MODEL}" == "openai/gpt-oss-120b" ]]; then
7+
tp=4
8+
elif [[ "${MODEL}" == "openai/gpt-oss-20b" ]]; then
9+
tp=1
10+
fi
11+
12+
echo $tp
13+
# Prepare the accuracy test
14+
vllm serve $MODEL --tensor_parallel_size $tp &
15+
server_pid=$!
16+
17+
wait_for_server() {
18+
timeout 1200 bash -c '
19+
until curl -X POST localhost:8000/v1/completions; do
20+
sleep 1
21+
done' && return 0 || return 1
22+
}
23+
24+
if wait_for_server; then
25+
echo "vLLM server is up and running"
26+
else
27+
echo "vLLM failed to start within the timeout period"
28+
fi
29+
30+
pushd vllm-benchmarks/gpt-oss
31+
mkdir -p /tmp/gpqa_openai
32+
33+
# Low
34+
OPENAI_API_KEY='' python3 -m gpt_oss.evals --base-url http://localhost:8000/v1 \
35+
--model $MODEL \
36+
--eval gpqa \
37+
--reasoning-effort low \
38+
--n-threads $(expr $(nproc) / 2)
39+
40+
# Mid
41+
OPENAI_API_KEY='' python3 -m gpt_oss.evals --base-url http://localhost:8000/v1 \
42+
--model $MODEL \
43+
--eval gpqa \
44+
--reasoning-effort medium \
45+
--n-threads $(expr $(nproc) / 2)
46+
47+
# High
48+
OPENAI_API_KEY='' python3 -m gpt_oss.evals --base-url http://localhost:8000/v1 \
49+
--model $MODEL \
50+
--eval gpqa \
51+
--reasoning-effort high \
52+
--n-threads $(expr $(nproc) / 2)
53+
54+
mv /tmp/gpqa_openai .
55+
popd
56+
57+
kill -9 $server_pid
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/bin/bash
2+
3+
set -eux
4+
5+
pushd vllm-benchmarks/vllm
6+
cp vllm/benchmarks/lib/utils.py /app/vllm-os-mini/vllm/benchmarks/utils.py || true
7+
8+
if [[ $DEVICE_NAME != 'rocm' ]]; then
9+
pip install -U openai transformers
10+
pip install --pre vllm==0.10.1+gptoss \
11+
--extra-index-url https://wheels.vllm.ai/gpt-oss/ \
12+
--extra-index-url https://download.pytorch.org/whl/nightly/cu128
13+
fi
14+
15+
pip freeze
16+
bash .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
17+
popd

.github/workflows/gpt-oss-benchmark.yml

Lines changed: 2 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -210,80 +210,10 @@ jobs:
210210
)
211211
212212
# Run accuracy check
213-
docker exec -t "${container_name}" bash -c "
214-
set -eux
215-
216-
echo $MODEL
217-
218-
tp=0
219-
if [[ $MODEL == 'openai/gpt-oss-120b' ]]; then
220-
tp=4
221-
elif [[ $MODEL == 'openai/gpt-oss-20b' ]]; then
222-
tp=1
223-
fi
224-
225-
echo $tp
226-
# Prepare the accuracy test
227-
vllm serve $MODEL --tensor_parallel_size $tp &
228-
server_pid=$!
229-
230-
wait_for_server() {
231-
timeout 1200 bash -c '
232-
until curl -X POST localhost:8000/v1/completions; do
233-
sleep 1
234-
done' && return 0 || return 1
235-
}
236-
237-
if wait_for_server; then
238-
echo 'vLLM server is up and running'
239-
else
240-
echo 'vLLM failed to start within the timeout period'
241-
fi
242-
243-
pushd vllm-benchmarks/gpt-oss
244-
mkdir -p /tmp/gpqa_openai
245-
246-
# Low
247-
OPENAI_API_KEY='' python3 -m gpt_oss.evals --base-url http://localhost:8000/v1 \
248-
--model $MODEL \
249-
--eval gpqa \
250-
--reasoning-effort low \
251-
--n-threads $(expr $(nproc) / 2)
252-
253-
# Mid
254-
OPENAI_API_KEY='' python3 -m gpt_oss.evals --base-url http://localhost:8000/v1 \
255-
--model $MODEL \
256-
--eval gpqa \
257-
--reasoning-effort medium \
258-
--n-threads $(expr $(nproc) / 2)
259-
260-
# High
261-
OPENAI_API_KEY='' python3 -m gpt_oss.evals --base-url http://localhost:8000/v1 \
262-
--model $MODEL \
263-
--eval gpqa \
264-
--reasoning-effort high \
265-
--n-threads $(expr $(nproc) / 2)
266-
267-
mv /tmp/gpqa_openai .
268-
popd
269-
270-
kill -9 $server_pid
271-
"
213+
docker exec -t "${container_name}" bash .github/scripts/gpt-oss/run_accuracy_checks.sh
272214
273215
# Run perf tests
274-
docker exec -t "${container_name}" bash -c "
275-
pushd vllm-benchmarks/vllm
276-
cp vllm/benchmarks/lib/utils.py /app/vllm-os-mini/vllm/benchmarks/utils.py || true
277-
if [[ $DEVICE_NAME != 'rocm' ]]; then
278-
pip install -U openai transformers
279-
pip install --pre vllm==0.10.1+gptoss \
280-
--extra-index-url https://wheels.vllm.ai/gpt-oss/ \
281-
--extra-index-url https://download.pytorch.org/whl/nightly/cu128
282-
fi
283-
pip freeze
284-
bash .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
285-
popd
286-
"
216+
docker exec -t "${container_name}" bash .github/scripts/gpt-oss/run_benchmarks.sh
287217
288218
- name: Authenticate with AWS
289219
# AWS CUDA runners already have access to the bucket via its runner IAM role

0 commit comments

Comments
 (0)