Another round

Signed-off-by: Huy Do <huydhn@gmail.com>

Author: huydhn

.github/scripts/gpt-oss/run_accuracy_checks.sh

@@ -44,30 +44,38 @@ mkdir -p /tmp/gpqa_openai
 
 # Not sure why this is needed on ROCm image
 if [[ "${DEVICE_NAME}" == "rocm" ]]; then
-  export PYTHONPATH=$(pwd)
+  pushd gpt_oss
+  # Low
+  OPENAI_API_KEY="" python3 -mevals --base-url http://localhost:8000/v1 \
+    --model $MODEL \
+    --eval gpqa \
+    --reasoning-effort low \
+    --n-threads $(expr $(nproc) / 2)
+  popd
+else
+  sleep 7200
+  # Low
+  #OPENAI_API_KEY="" python3 -m gpt_oss.evals --base-url http://localhost:8000/v1 \
+  #  --model $MODEL \
+  #  --eval gpqa \
+  #  --reasoning-effort low \
+  #  --n-threads $(expr $(nproc) / 2)
+  #
+  # Mid
+  #OPENAI_API_KEY="" python3 -m gpt_oss.evals --base-url http://localhost:8000/v1 \
+  #  --model $MODEL \
+  #  --eval gpqa \
+  #  --reasoning-effort medium \
+  #  --n-threads $(expr $(nproc) / 2)
+  #
+  ## High
+  #OPENAI_API_KEY="" python3 -m gpt_oss.evals --base-url http://localhost:8000/v1 \
+  #  --model $MODEL \
+  #  --eval gpqa \
+  #  --reasoning-effort high \
+  #  --n-threads $(expr $(nproc) / 2)
 fi
 
-# Low
-OPENAI_API_KEY="" python3 -m gpt_oss.evals --base-url http://localhost:8000/v1 \
-  --model $MODEL \
-  --eval gpqa \
-  --reasoning-effort low \
-  --n-threads $(expr $(nproc) / 2)
-
-# Mid
-OPENAI_API_KEY="" python3 -m gpt_oss.evals --base-url http://localhost:8000/v1 \
-  --model $MODEL \
-  --eval gpqa \
-  --reasoning-effort medium \
-  --n-threads $(expr $(nproc) / 2)
-
-# High
-OPENAI_API_KEY="" python3 -m gpt_oss.evals --base-url http://localhost:8000/v1 \
-  --model $MODEL \
-  --eval gpqa \
-  --reasoning-effort high \
-  --n-threads $(expr $(nproc) / 2)
-
 mv /tmp/gpqa_openai .
 popd
 

.github/workflows/gpt-oss-benchmark.yml

@@ -162,6 +162,12 @@ jobs:
             aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
           fi
 
+          # Leaving 1GB for the runner and other things
+          TOTAL_AVAILABLE_MEMORY_IN_GB=$(awk '/MemTotal/ { printf "%.3f \n", $2/1024/1024 - 1 }' /proc/meminfo)
+          # https://docs.docker.com/engine/containers/resource_constraints/#--memory-swap-details, the 3GB swap
+          # comes from https://github.com/pytorch/test-infra/pull/6058
+          TOTAL_MEMORY_WITH_SWAP=$(("${TOTAL_AVAILABLE_MEMORY_IN_GB%.*}" + 3))
+
           container_name=$(docker run \
             ${GPU_FLAG:-} \
             -e MODEL \
@@ -170,6 +176,8 @@ jobs:
             -e HF_TOKEN \
             -e ENGINE_VERSION \
             -e SAVE_TO_PYTORCH_BENCHMARK_FORMAT \
+            --memory="${TOTAL_AVAILABLE_MEMORY_IN_GB%.*}g" \
+            --memory-swap="${TOTAL_MEMORY_WITH_SWAP}g" \
             --ipc=host \
             --tty \
             --detach \