Benchmark aime25

Signed-off-by: Huy Do <huydhn@gmail.com>

Author: huydhn

.github/scripts/gpt-oss/run_accuracy_checks.sh

@@ -41,55 +41,57 @@ fi
 
 pushd vllm-benchmarks/gpt-oss
 mkdir -p /tmp/gpqa_openai
+mkdir -p /tmp/aime25_openai
 
 if [[ "${DEVICE_NAME}" == "rocm" ]]; then
   # Not sure why this is needed on ROCm
   pushd gpt_oss
   # Low
   OPENAI_API_KEY="" python3 -mevals --base-url http://localhost:8000/v1 \
     --model $MODEL \
-    --eval gpqa \
+    --eval aime25 \
     --reasoning-effort low \
     --n-threads $(expr $(nproc) / 2)
 
   # Mid
   OPENAI_API_KEY="" python3 -mevals --base-url http://localhost:8000/v1 \
     --model $MODEL \
-    --eval gpqa \
+    --eval aime25 \
     --reasoning-effort medium \
     --n-threads $(expr $(nproc) / 2)
 
   # High
   OPENAI_API_KEY="" python3 -mevals --base-url http://localhost:8000/v1 \
     --model $MODEL \
-    --eval gpqa \
+    --eval \
     --reasoning-effort high \
     --n-threads $(expr $(nproc) / 2)
   popd
 else
   # Low
   OPENAI_API_KEY="" python3 -m gpt_oss.evals --base-url http://localhost:8000/v1 \
     --model $MODEL \
-    --eval gpqa \
+    --eval aime25 \
     --reasoning-effort low \
     --n-threads $(expr $(nproc) / 2)
 
   # Mid
   OPENAI_API_KEY="" python3 -m gpt_oss.evals --base-url http://localhost:8000/v1 \
     --model $MODEL \
-    --eval gpqa \
+    --eval aime25 \
     --reasoning-effort medium \
     --n-threads $(expr $(nproc) / 2)
 
   # High
   OPENAI_API_KEY="" python3 -m gpt_oss.evals --base-url http://localhost:8000/v1 \
     --model $MODEL \
-    --eval gpqa \
+    --eval aime25 \
     --reasoning-effort high \
     --n-threads $(expr $(nproc) / 2)
 fi
 
 mv /tmp/gpqa_openai .
+mv /tmp/aime25_openai .
 popd
 
 kill -9 $server_pid

.github/scripts/gpt-oss/run_benchmarks.sh

@@ -25,5 +25,6 @@ if [[ "${DEVICE_NAME}" != "rocm" ]]; then
 fi
 
 pip freeze
-bash .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
+# Just run accuracy tests for now
+# bash .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
 popd

.github/workflows/gpt-oss-benchmark.yml

@@ -192,7 +192,7 @@ jobs:
           docker exec -t "${container_name}" bash .github/scripts/gpt-oss/run_benchmarks.sh
 
           # Run accuracy check (turning this on later if needed)
-          # docker exec -t "${container_name}" bash .github/scripts/gpt-oss/run_accuracy_checks.sh
+          docker exec -t "${container_name}" bash .github/scripts/gpt-oss/run_accuracy_checks.sh
 
       - name: Authenticate with AWS
         # AWS CUDA runners already have access to the bucket via its runner IAM role