vllm-profiling.yml

# TODO: Refactor the workflows to extract the common parts into a GHA reusable module
name: vLLM Profiling

on:
  schedule:
    - cron: '0 0 * * 0'
  workflow_dispatch:
    inputs:
      vllm_branch:
        description: vLLM branch (main, releases/vERSION for release validation, or refs/pull/PR_NUMBER/head for pre-merge check on pull request)
        required: true
        type: string
        default: main
      vllm_commit:
        description: vLLM commit (optional, default to the latest commit in the branch that has not yet been benchmarked)
        required: false
        type: string
      # TODO: add support for profiling on a specific model and runner
  pull_request:
    paths:
      - .github/workflows/vllm-profiling.yml

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}
  cancel-in-progress: true

jobs:
  # OSDC/ARC runners are ephemeral pods with no Docker daemon, so the vLLM CI
  # image is run via the job-level container: key (see the profiling job) and
  # must be resolved up front. docker manifest inspect needs a daemon, so do it
  # here on a GitHub-hosted runner and pass the resolved image down.
  resolve-image:
    name: Resolve vLLM CI image
    runs-on: ubuntu-latest
    if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'pytorch' }}
    outputs:
      docker-image: ${{ steps.resolve.outputs.docker-image }}
      head-sha: ${{ steps.resolve.outputs.head-sha }}
    steps:
      - name: Checkout vLLM repository
        uses: actions/checkout@v4
        with:
          repository: vllm-project/vllm
          path: vllm
          ref: ${{ inputs.vllm_branch || 'main' }}
          fetch-depth: 0

      - name: Resolve the latest available vLLM CI image
        id: resolve
        working-directory: vllm
        env:
          HEAD_BRANCH: ${{ inputs.vllm_branch || 'main' }}
          HEAD_SHA: ${{ inputs.vllm_commit || '' }}
        run: |
          set -eux

          if [[ "${HEAD_BRANCH}" == "main" ]]; then
            DOCKER_IMAGE_PREFIX=public.ecr.aws/q9t5s3a7/vllm-ci-postmerge-repo
          else
            DOCKER_IMAGE_PREFIX=public.ecr.aws/q9t5s3a7/vllm-ci-test-repo
          fi

          if [[ -z "${HEAD_SHA}" ]]; then
            for i in {0..99}; do
              HEAD_SHA=$(git rev-parse --verify HEAD~${i})
              DOCKER_IMAGE="${DOCKER_IMAGE_PREFIX}:${HEAD_SHA}"
              # Docker image available for this commit, then exit
              if docker manifest inspect "${DOCKER_IMAGE}"; then
                break
              fi
            done
          fi

          echo "docker-image=${DOCKER_IMAGE_PREFIX}:${HEAD_SHA}" >> "${GITHUB_OUTPUT}"
          echo "head-sha=${HEAD_SHA}" >> "${GITHUB_OUTPUT}"
          echo "### Run profiling on [${HEAD_SHA}](https://github.com/vllm-project/vllm/commit/${HEAD_SHA})" >> "${GITHUB_STEP_SUMMARY}"

  profiling:
    name: Run vLLM profiling
    needs: resolve-image
    if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'pytorch' }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - runs-on: mt-l-x86iavx512-11-125-a100
            device-name: cuda
    runs-on: ${{ matrix.runs-on }}
    environment: pytorch-x-vllm
    permissions:
      id-token: write
      contents: read
    # OSDC/ARC runners run the workload inside this container; the GPU is exposed
    # by the runner pod, the same pattern as pytorch/pytorch _linux-test.yml
    # (test-osdc) and pytorch/helion.
    container:
      image: ${{ needs.resolve-image.outputs.docker-image }}
      options: --gpus all
    steps:
      - name: GPU health check
        run: nvidia-smi || true

      - name: Install git
        run: |
          # The vLLM CI image may not ship git, which actions/checkout needs
          (command -v git || (apt-get update && apt-get install -y git)) || true
          git config --global --add safe.directory '*' || true

      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set GPU name and type
        shell: bash
        run: |
          set -eux
          DEVICE_TYPE=$(nvidia-smi -i 0 --query-gpu=name --format=csv,noheader | awk '{print $2}')
          echo "DEVICE_NAME=cuda" >> $GITHUB_ENV
          echo "DEVICE_TYPE=$DEVICE_TYPE" >> $GITHUB_ENV

      - name: Run vLLM profiling
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
          VLLM_USE_MODELSCOPE: false
          VLLM_TORCH_PROFILER_DIR: ${{ github.workspace }}/vllm-profiling/profiling-results
          CUDA_VISIBLE_DEVICES: 0
          VLLM_USE_V1: 1
          S3_HEAD_SHA: ${{ needs.resolve-image.outputs.head-sha }}
          S3_GITHUB_RUN_ID: ${{ github.run_id }}
          S3_GITHUB_JOB: ${{ github.job }}
        run: |
          set -eux
          cd vllm-profiling && bash ../.github/scripts/run_vllm_profiling.sh

      - name: Authenticate with AWS for the S3 upload
        # Ephemeral OSDC pods do not carry a host IAM role, so assume the upload
        # role explicitly via OIDC (id-token: write is set above).
        uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0
        with:
          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_upload-benchmark-results
          # The max duration enforced by the server side
          role-duration-seconds: 18000
          aws-region: us-east-1

      - name: Prepare S3 upload metadata
        id: prepare_s3_upload
        env:
          REPOSITORY: vllm-project/vllm
        run: |
          set -eux

          UPLOAD_DATE=$(date -u +"%Y-%m-%d")
          echo "upload-date=${UPLOAD_DATE}" >> "${GITHUB_OUTPUT}"
          echo "s3-prefix=${UPLOAD_DATE}/${REPOSITORY}" >> "${GITHUB_OUTPUT}"

      - name: Upload profiling results to S3
        uses: seemethere/upload-artifact-s3@v5
        with:
          s3-prefix: ${{ steps.prepare_s3_upload.outputs.s3-prefix }}
          retention-days: 180
          path: vllm-profiling/profiling-results
          if-no-files-found: warn

      - uses: actions/upload-artifact@v4
        with:
          name: profiling-results--${{ env.DEVICE_TYPE }}
          path: vllm-profiling/profiling-results