atom-benchmark.yaml

name: ATOM Benchmark

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

on:
  schedule:
    # Nightly at 01:00 Beijing time (17:00 UTC)
    - cron: '0 17 * * *'
  workflow_dispatch:
    inputs:
      deepseek-r1-0528:
        description: "Benchmark DeepSeek-R1-0528"
        type: boolean
        default: true
      gpt-oss-120b:
        description: "Benchmark gpt-oss-120b"
        type: boolean
        default: true
      extra_args:
        description: "Extra arguments to pass to the ATOM server"
        type: string
      image:
        description: "Image to use for the benchmark"
        type: string
        default: "rocm/atom-dev:latest"
      enable_profiler:
        description: "Enable torch profiler to collect trace for performance debugging"
        type: boolean
        default: false
      param_lists:
        description: |
          "Benchmark parameter lists. 
          Input as a single or multiple sets (comma-separated, semicolon between sets), 
          format: input_length,output_length,concurrency,random_range_ratio.
          Example (single set): 1024,1024,128,0.8
          Example (multiple sets): 1024,1024,128,0.8;2048,1024,256,0.7"
        type: string
        default: "1024,1024,128,0.8"

jobs:
  parse-param-lists:
    name: Parse parameter lists
    runs-on: ubuntu-latest
    outputs:
      matrix_json: ${{ steps.parse-param-lists.outputs.matrix_json }}
    env:
      # Nightly params aligned with InferenceX dsr1-fp8 config: 1k1k, 1k8k, 8k1k with multiple concurrency levels
      NIGHTLY_PARAM_LISTS: "1024,1024,1,0.8;1024,1024,2,0.8;1024,1024,4,0.8;1024,1024,8,0.8;1024,1024,16,0.8;1024,1024,32,0.8;1024,1024,64,0.8;1024,1024,128,0.8;1024,1024,256,0.8;1024,8192,1,0.8;1024,8192,2,0.8;1024,8192,4,0.8;1024,8192,8,0.8;1024,8192,16,0.8;1024,8192,32,0.8;8192,1024,1,0.8;8192,1024,2,0.8;8192,1024,4,0.8;8192,1024,8,0.8;8192,1024,16,0.8;8192,1024,32,0.8;8192,1024,64,0.8;8192,1024,128,0.8"
    steps:
      - name: Parse parameter lists
        id: parse-param-lists
        run: |
          if [ "${{ github.event_name }}" = "schedule" ]; then
            PARAM_LISTS="${{ env.NIGHTLY_PARAM_LISTS }}"
            echo "Using nightly param lists (1k1k, 1k8k, 8k1k with multiple concurrency)"
            echo "PARAM_LISTS: ${PARAM_LISTS}"
          else
            PARAM_LISTS="${{ inputs.param_lists || '1024,1024,128,0.8' }}"
            echo "Using param_lists: ${PARAM_LISTS}"
          fi
          IFS=';' read -ra SETS <<< "${PARAM_LISTS}"
          MATRIX_JSON="["
          SEP=""
          for SET in "${SETS[@]}"; do
            IFS=',' read -ra PARAMS <<< "$SET"
            INPUT_LEN="${PARAMS[0]}"
            OUTPUT_LEN="${PARAMS[1]}"
            CONCURRENCY="${PARAMS[2]}"
            RANDOM_RANGE_RATIO="${PARAMS[3]}"
            MATRIX_JSON="${MATRIX_JSON}${SEP}{\"input_length\":${INPUT_LEN},\"output_length\":${OUTPUT_LEN},\"concurrency\":${CONCURRENCY},\"random_range_ratio\":${RANDOM_RANGE_RATIO}}"
            SEP=","
          done
          MATRIX_JSON="${MATRIX_JSON}]"
          echo "matrix_json=${MATRIX_JSON}" >> $GITHUB_OUTPUT
      
      - name: Print matrix JSON
        run: |
          echo "Matrix JSON: ${{ steps.parse-param-lists.outputs.matrix_json }}"

  deepseek-r1-0528-benchmark:
    if: ${{ github.event_name == 'schedule' || inputs.deepseek-r1-0528 }}
    name: DeepSeek-R1-0528 Benchmark (input=${{ matrix.config.input_length }}, output=${{ matrix.config.output_length }}, conc=${{ matrix.config.concurrency }}, ratio=${{ matrix.config.random_range_ratio }})
    needs: [parse-param-lists]
    strategy:
      fail-fast: false
      matrix:
        config: ${{ fromJson(needs.parse-param-lists.outputs.matrix_json) }}
    
    env:
      MODEL_PATH: "deepseek-ai/DeepSeek-R1-0528"
      ARGS: "--kv_cache_dtype fp8 -tp 8"
      ISL: ${{ matrix.config.input_length }}
      OSL: ${{ matrix.config.output_length }}
      CONC: ${{ matrix.config.concurrency }}
      RANDOM_RANGE_RATIO: ${{ matrix.config.random_range_ratio }}
      RESULT_FILENAME: deepseek-r1-0528-${{ matrix.config.input_length }}-${{ matrix.config.output_length }}-${{ matrix.config.concurrency }}-${{ matrix.config.random_range_ratio }}

    runs-on: atom-mi355-8gpu.predownload
    steps:
      - name: Kill all Docker containers
        run: |
          echo "=== Cleaning up containers on $(hostname) ==="
          containers=$(docker ps -q)
          if [ -n "$containers" ]; then
            docker kill $containers || true
          fi
          docker run --rm -v "${GITHUB_WORKSPACE:-$PWD}":/workspace -w /workspace --privileged rocm/pytorch:latest bash -lc "ls -la /workspace/ && rm -rf /workspace/*" || true

      - name: Show Docker containers
        run: docker ps -a

      - name: Show ROCm memory usage
        run: rocm-smi --showmemuse

      - name: Show ROCm GPU processes
        run: rocm-smi --showpidgpus

      - name: Checkout ATOM repo
        uses: actions/checkout@v4

      - name: Start CI container
        run: |
          echo "Clean up containers..."
          docker ps -aq -f name=atom-benchmark | xargs -r docker stop | xargs -r docker rm

          if [ -f "/etc/podinfo/gha-render-devices" ]; then
            DEVICE_FLAG=$(cat /etc/podinfo/gha-render-devices)
          else
            DEVICE_FLAG="--device /dev/dri"
          fi

          if [ -d "/models" ]; then
            MODEL_MOUNT="-v /models:/models"
          else
            echo "Warning: /models directory not found on runner; skipping /models mount and disabling model pre-download optimization."
            MODEL_MOUNT=""
          fi

          docker run -dt --device=/dev/kfd $DEVICE_FLAG \
          -v "${GITHUB_WORKSPACE:-$PWD}":/workspace \
          $MODEL_MOUNT \
          -w /workspace \
          --ipc=host --group-add video \
          --shm-size=16G \
          --privileged \
          --cap-add=SYS_PTRACE \
          -e HF_TOKEN="${HF_TOKEN:-}" \
          --security-opt seccomp=unconfined \
          --ulimit memlock=-1 \
          --pull always \
          --ulimit stack=67108864 \
          -e ISL=${{ env.ISL }} \
          -e OSL=${{ env.OSL }} \
          -e CONC=${{ env.CONC }} \
          -e RANDOM_RANGE_RATIO=${{ env.RANDOM_RANGE_RATIO }} \
          -e ENABLE_TORCH_PROFILER=${{ inputs.enable_profiler && '1' || '0' }} \
          -e ATOM_DISABLE_MMAP=true \
          -v "${{ github.workspace }}:/workspace" \
          -w /workspace \
          --name atom-benchmark \
          ${{ inputs.image || 'rocm/atom-dev:latest' }}

        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Download models
        run: |
          if [ -d "/models" ]; then
            echo "/models directory found, downloading model to /models/${{ env.MODEL_PATH }}"
            if ! docker exec -e HF_TOKEN=${{ secrets.AMD_HF_TOKEN }} atom-benchmark bash -lc "hf download ${{ env.MODEL_PATH }} --local-dir /models/${{ env.MODEL_PATH }}"; then
              echo "Model download failed for '${{ env.MODEL_PATH }}'. Aborting."
              exit 1
            fi
          else
            echo "/models directory not found, skipping model download"
          fi

      - name: Run benchmark
        timeout-minutes: 60
        run: |
          set -euo pipefail
          echo ""
          echo "========== Launching ATOM server =========="
          if [ -d "/models" ]; then
            model_path="/models/${{ env.MODEL_PATH }}"
          else
            model_path="${{ env.MODEL_PATH }}"
          fi
          # Run launch and benchmark in a single exec so the server process is not killed when the first shell exits
          docker exec atom-benchmark bash -lc "
            ENABLE_TORCH_PROFILER=${{ inputs.enable_profiler && '1' || '0' }} .github/scripts/atom_test.sh launch $model_path ${{ env.ARGS }} ${{ inputs.extra_args || '' }}
            echo '========== Running benchmark test =========='
            ENABLE_TORCH_PROFILER=${{ inputs.enable_profiler && '1' || '0' }} RESULT_FILENAME=${{ env.RESULT_FILENAME }} .github/scripts/atom_test.sh benchmark $model_path
            ls -all .
          "

      - name: Copy profiler traces from container
        if: ${{ inputs.enable_profiler }}
        run: |
          docker cp atom-benchmark:/app/trace ./profiler-traces || echo "No profiler traces found"
          ls -la ./profiler-traces/ 2>/dev/null || true

      - name: Upload profiler traces
        if: ${{ inputs.enable_profiler }}
        uses: actions/upload-artifact@v4
        with:
          name: profiler-traces-deepseek-r1-0528-${{ matrix.config.input_length }}-${{ matrix.config.output_length }}-${{ matrix.config.concurrency }}
          path: profiler-traces/

      - name: Upload benchmark result
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.RESULT_FILENAME }}
          path: ${{ env.RESULT_FILENAME }}.json

      - name: Clean Up
        if: always()
        run: |
          # TODO: run a separate container for cleanup of the workspace due to permission issue to remove some pyc files under __pycache__ whose owners are root.
          # We should use non-root user to run the test to avoid this issue.
          set -x
          echo "========== Cleaning up workspace =========="
          docker run --rm -v "${GITHUB_WORKSPACE:-$PWD}":/workspace -w /workspace --privileged ${{ inputs.image || 'rocm/atom-dev:latest' }} bash -lc "rm -rf /workspace/atom/ /workspace/aiter/ /workspace/bench_serving/" || true
          docker stop atom-benchmark || true
          docker rm atom-benchmark || true

  gpt-oss-120b-benchmark:
    if: ${{ github.event_name == 'schedule' || inputs.gpt-oss-120b }}
    name: GPT-OSS-120b Benchmark (input=${{ matrix.config.input_length }}, output=${{ matrix.config.output_length }}, conc=${{ matrix.config.concurrency }}, ratio=${{ matrix.config.random_range_ratio }})
    needs: [parse-param-lists]
    strategy:
      fail-fast: false
      matrix:
        config: ${{ fromJson(needs.parse-param-lists.outputs.matrix_json) }}
    
    env:
      MODEL_PATH: "openai/gpt-oss-120b"
      ARGS: "--kv_cache_dtype fp8"
      ISL: ${{ matrix.config.input_length }}
      OSL: ${{ matrix.config.output_length }}
      CONC: ${{ matrix.config.concurrency }}
      RANDOM_RANGE_RATIO: ${{ matrix.config.random_range_ratio }}
      RESULT_FILENAME: gpt-oss-120b-${{ matrix.config.input_length }}-${{ matrix.config.output_length }}-${{ matrix.config.concurrency }}-${{ matrix.config.random_range_ratio }}

    runs-on: atom-mi355-8gpu.predownload
    steps:
      - name: Kill all Docker containers
        run: |
          echo "=== Cleaning up containers on $(hostname) ==="
          containers=$(docker ps -q)
          if [ -n "$containers" ]; then
            docker kill $containers || true
          fi
          docker run --rm -v "${GITHUB_WORKSPACE:-$PWD}":/workspace -w /workspace --privileged rocm/pytorch:latest bash -lc "ls -la /workspace/ && rm -rf /workspace/*" || true

      - name: Show Docker containers
        run: docker ps -a

      - name: Show ROCm memory usage
        run: rocm-smi --showmemuse

      - name: Show ROCm GPU processes
        run: rocm-smi --showpidgpus

      - name: Checkout ATOM repo
        uses: actions/checkout@v4

      - name: Start CI container
        run: |
          echo "Clean up containers..."
          docker ps -aq -f name=atom-benchmark | xargs -r docker stop | xargs -r docker rm

          if [ -f "/etc/podinfo/gha-render-devices" ]; then
            DEVICE_FLAG=$(cat /etc/podinfo/gha-render-devices)
          else
            DEVICE_FLAG="--device /dev/dri"
          fi

          if [ -d "/models" ]; then
            MODEL_MOUNT="-v /models:/models"
          else
            echo "Warning: /models directory not found on runner; skipping /models mount and disabling model pre-download optimization."
            MODEL_MOUNT=""
          fi

          docker run -dt --device=/dev/kfd $DEVICE_FLAG \
          -v "${GITHUB_WORKSPACE:-$PWD}":/workspace \
          $MODEL_MOUNT \
          -w /workspace \
          --ipc=host --group-add video \
          --shm-size=16G \
          --privileged \
          --cap-add=SYS_PTRACE \
          -e HF_TOKEN="${HF_TOKEN:-}" \
          --security-opt seccomp=unconfined \
          --ulimit memlock=-1 \
          --ulimit stack=67108864 \
          -e ISL=${{ env.ISL }} \
          -e OSL=${{ env.OSL }} \
          -e CONC=${{ env.CONC }} \
          -e RANDOM_RANGE_RATIO=${{ env.RANDOM_RANGE_RATIO }} \
          -e ATOM_GPT_OSS_MODEL=1 \
          -e ENABLE_TORCH_PROFILER=${{ inputs.enable_profiler && '1' || '0' }} \
          -e ATOM_DISABLE_MMAP=true \
          -v "${{ github.workspace }}:/workspace" \
          -w /workspace \
          --name atom-benchmark \
          ${{ inputs.image || 'rocm/atom-dev:latest' }}

        env:
          GITHUB_WORKSPACE: ${{ github.workspace }}

      - name: Download models
        run: |
          if [ -d "/models" ]; then
            echo "/models directory found, downloading model to /models/${{ env.MODEL_PATH }}"
            if ! docker exec -e HF_TOKEN=${{ secrets.AMD_HF_TOKEN }} atom-benchmark bash -lc "hf download ${{ env.MODEL_PATH }} --local-dir /models/${{ env.MODEL_PATH }}"; then
              echo "Model download failed for '${{ env.MODEL_PATH }}'. Aborting."
              exit 1
            fi
          else
            echo "/models directory not found, skipping model download"
          fi

      - name: Run benchmark
        timeout-minutes: 60
        run: |
          set -euo pipefail
          echo ""
          echo "========== Launching ATOM server =========="
          if [ -d "/models" ]; then
            model_path="/models/${{ env.MODEL_PATH }}"
          else
            model_path="${{ env.MODEL_PATH }}"
          fi
          # Run launch and benchmark in a single exec so the server process is not killed when the first shell exits
          docker exec atom-benchmark bash -lc "
            ENABLE_TORCH_PROFILER=${{ inputs.enable_profiler && '1' || '0' }} .github/scripts/atom_test.sh launch $model_path ${{ env.ARGS }} ${{ inputs.extra_args || '' }}
            echo '========== Running benchmark test =========='
            ENABLE_TORCH_PROFILER=${{ inputs.enable_profiler && '1' || '0' }} RESULT_FILENAME=${{ env.RESULT_FILENAME }} .github/scripts/atom_test.sh benchmark $model_path
            ls -all .
          "

      - name: Copy profiler traces from container
        if: ${{ inputs.enable_profiler }}
        run: |
          docker cp atom-benchmark:/app/trace ./profiler-traces || echo "No profiler traces found"
          ls -la ./profiler-traces/ 2>/dev/null || true

      - name: Upload profiler traces
        if: ${{ inputs.enable_profiler }}
        uses: actions/upload-artifact@v4
        with:
          name: profiler-traces-gpt-oss-120b-${{ matrix.config.input_length }}-${{ matrix.config.output_length }}-${{ matrix.config.concurrency }}
          path: profiler-traces/

      - name: Upload benchmark result
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.RESULT_FILENAME }}
          path: ${{ env.RESULT_FILENAME }}.json

      - name: Clean Up
        if: always()
        run: |
          # TODO: run a separate container for cleanup of the workspace due to permission issue to remove some pyc files under __pycache__ whose owners are root.
          # We should use non-root user to run the test to avoid this issue.
          set -x
          echo "========== Cleaning up workspace =========="
          docker run --rm -v "${GITHUB_WORKSPACE:-$PWD}":/workspace -w /workspace --privileged ${{ inputs.image || 'rocm/atom-dev:latest' }} bash -lc "rm -rf /workspace/atom/ /workspace/aiter/ /workspace/bench_serving/" || true
          docker stop atom-benchmark || true
          docker rm atom-benchmark || true

  summarize-benchmark-result:
    if: always()
    name: Summarize benchmark result
    needs: [deepseek-r1-0528-benchmark, gpt-oss-120b-benchmark]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout ATOM repo
        uses: actions/checkout@v4
        
      - name: Download all benchmark results
        uses: actions/download-artifact@v4
        with: 
          pattern: '*'
          merge-multiple: true
          path: .

      - name: Download baseline from previous nightly run
        id: baseline
        run: |
          PREV_RUN_ID=$(gh run list \
            --workflow="ATOM Benchmark" \
            --branch=main \
            --event=schedule \
            --status=success \
            --limit=1 \
            --json databaseId \
            --jq '.[0].databaseId // empty')

          if [ -n "$PREV_RUN_ID" ] && [ "$PREV_RUN_ID" != "${{ github.run_id }}" ]; then
            echo "Downloading baseline from run #$PREV_RUN_ID"
            mkdir -p /tmp/baseline
            gh run download "$PREV_RUN_ID" --dir /tmp/baseline || echo "::warning::Failed to download baseline artifacts"
            BASELINE_COUNT=$(find /tmp/baseline -name '*.json' | wc -l)
            echo "Downloaded $BASELINE_COUNT baseline files"
            echo "baseline_dir=/tmp/baseline" >> $GITHUB_OUTPUT
          else
            echo "No previous successful nightly run found"
            echo "baseline_dir=" >> $GITHUB_OUTPUT
          fi
        env:
          GH_TOKEN: ${{ github.token }}
      
      - name: List all benchmark results
        run: |
          echo "=== Current results ==="
          ls -la *.json 2>/dev/null || echo "No JSON files in current dir"
          if [ -d "/tmp/baseline" ]; then
            echo "=== Baseline results ==="
            find /tmp/baseline -name '*.json' | head -20
          fi
      
      - name: Summarize benchmark result
        run: |
          BASELINE_ARG=""
          if [ -n "${{ steps.baseline.outputs.baseline_dir }}" ]; then
            BASELINE_ARG="--baseline-dir ${{ steps.baseline.outputs.baseline_dir }}"
          fi
          .github/scripts/summarize.py . $BASELINE_ARG >> $GITHUB_STEP_SUMMARY