Run vLLM tests #2204
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run vLLM tests | |
| on: | |
| schedule: | |
| # Run every 4 hours | |
| - cron: '0 */4 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| vllm_branch: | |
| description: vLLM branch (main, releases/vERSION for release validation, or refs/pull/PR_NUMBER/head for pre-merge check on pull request) | |
| required: true | |
| type: string | |
| default: main | |
| vllm_commit: | |
| description: vLLM commit (optional, default to the latest commit in the branch that has not yet been benchmarked) | |
| required: false | |
| type: string | |
| pull_request: | |
| paths: | |
| - .github/workflows/vllm-ci-test.yml | |
| - .github/scripts/run_vllm_tests.sh | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }} | |
| cancel-in-progress: true | |
| jobs: | |
| # OSDC/ARC runners are ephemeral pods with no Docker daemon, so the vLLM CI | |
| # image is run via the job-level container: key (see the test job) and must be | |
| # resolved up front. docker manifest inspect needs a daemon, so do it here on a | |
| # GitHub-hosted runner and pass the resolved image down. | |
| resolve-image: | |
| name: Resolve vLLM CI image | |
| runs-on: ubuntu-latest | |
| if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'pytorch' }} | |
| outputs: | |
| docker-image: ${{ steps.resolve.outputs.docker-image }} | |
| steps: | |
| - name: Checkout vLLM repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: vllm-project/vllm | |
| path: vllm | |
| ref: ${{ inputs.vllm_branch || 'main' }} | |
| fetch-depth: 0 | |
| - name: Resolve the latest available vLLM CI image | |
| id: resolve | |
| working-directory: vllm | |
| env: | |
| HEAD_BRANCH: ${{ inputs.vllm_branch || 'main' }} | |
| HEAD_SHA: ${{ inputs.vllm_commit || '' }} | |
| run: | | |
| set -eux | |
| # Mimic the logic from vllm ci-infra test template | |
| if [[ "${HEAD_BRANCH}" == "main" ]]; then | |
| DOCKER_IMAGE_PREFIX=public.ecr.aws/q9t5s3a7/vllm-ci-postmerge-repo | |
| else | |
| DOCKER_IMAGE_PREFIX=public.ecr.aws/q9t5s3a7/vllm-ci-test-repo | |
| fi | |
| if [[ -z "${HEAD_SHA}" ]]; then | |
| # Looking back the latest 100 commits is enough | |
| for i in {0..99}; do | |
| # Check if the image is there, if it doesn't then check an older one | |
| # because the commit is too recent | |
| HEAD_SHA=$(git rev-parse --verify HEAD~${i}) | |
| DOCKER_IMAGE="${DOCKER_IMAGE_PREFIX}:${HEAD_SHA}" | |
| if docker manifest inspect "${DOCKER_IMAGE}"; then | |
| break | |
| fi | |
| done | |
| fi | |
| echo "docker-image=${DOCKER_IMAGE_PREFIX}:${HEAD_SHA}" >> "${GITHUB_OUTPUT}" | |
| test: | |
| name: Run vLLM tests | |
| needs: resolve-image | |
| if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'pytorch' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # TODO (huydhn): Figure out later if we need to scale this up to multiple runners | |
| - runs-on: mt-l-x86iamx-88-900-h100-4 | |
| device-name: cuda | |
| permissions: | |
| id-token: write | |
| contents: read | |
| runs-on: ${{ matrix.runs-on }} | |
| environment: pytorch-x-vllm | |
| # OSDC/ARC runners run the workload inside this container; the GPU is exposed | |
| # by the runner pod, the same pattern as pytorch/pytorch _linux-test.yml | |
| # (test-osdc) and pytorch/helion. | |
| container: | |
| image: ${{ needs.resolve-image.outputs.docker-image }} | |
| options: --gpus all | |
| steps: | |
| - name: GPU health check | |
| run: nvidia-smi || true | |
| - name: Install git | |
| run: | | |
| # The vLLM CI image may not ship git, which actions/checkout needs | |
| (command -v git || (apt-get update && apt-get install -y git)) || true | |
| git config --global --add safe.directory '*' || true | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Run vLLM tests | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| run: | | |
| set -eux | |
| bash .github/scripts/run_vllm_tests.sh |