vLLM Profiling #88
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # TODO: Refactor the workflows to extract the common parts into a GHA reusable module | |
| name: vLLM Profiling | |
| on: | |
| schedule: | |
| - cron: '0 0 * * 0' | |
| workflow_dispatch: | |
| inputs: | |
| vllm_branch: | |
| description: vLLM branch (main, releases/vERSION for release validation, or refs/pull/PR_NUMBER/head for pre-merge check on pull request) | |
| required: true | |
| type: string | |
| default: main | |
| vllm_commit: | |
| description: vLLM commit (optional, default to the latest commit in the branch that has not yet been benchmarked) | |
| required: false | |
| type: string | |
| # TODO: add support for profiling on a specific model and runner | |
| pull_request: | |
| paths: | |
| - .github/workflows/vllm-profiling.yml | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }} | |
| cancel-in-progress: true | |
| jobs: | |
| # OSDC/ARC runners are ephemeral pods with no Docker daemon, so the vLLM CI | |
| # image is run via the job-level container: key (see the profiling job) and | |
| # must be resolved up front. docker manifest inspect needs a daemon, so do it | |
| # here on a GitHub-hosted runner and pass the resolved image down. | |
| resolve-image: | |
| name: Resolve vLLM CI image | |
| runs-on: ubuntu-latest | |
| if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'pytorch' }} | |
| outputs: | |
| docker-image: ${{ steps.resolve.outputs.docker-image }} | |
| head-sha: ${{ steps.resolve.outputs.head-sha }} | |
| steps: | |
| - name: Checkout vLLM repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: vllm-project/vllm | |
| path: vllm | |
| ref: ${{ inputs.vllm_branch || 'main' }} | |
| fetch-depth: 0 | |
| - name: Resolve the latest available vLLM CI image | |
| id: resolve | |
| working-directory: vllm | |
| env: | |
| HEAD_BRANCH: ${{ inputs.vllm_branch || 'main' }} | |
| HEAD_SHA: ${{ inputs.vllm_commit || '' }} | |
| run: | | |
| set -eux | |
| if [[ "${HEAD_BRANCH}" == "main" ]]; then | |
| DOCKER_IMAGE_PREFIX=public.ecr.aws/q9t5s3a7/vllm-ci-postmerge-repo | |
| else | |
| DOCKER_IMAGE_PREFIX=public.ecr.aws/q9t5s3a7/vllm-ci-test-repo | |
| fi | |
| if [[ -z "${HEAD_SHA}" ]]; then | |
| for i in {0..99}; do | |
| HEAD_SHA=$(git rev-parse --verify HEAD~${i}) | |
| DOCKER_IMAGE="${DOCKER_IMAGE_PREFIX}:${HEAD_SHA}" | |
| # Docker image available for this commit, then exit | |
| if docker manifest inspect "${DOCKER_IMAGE}"; then | |
| break | |
| fi | |
| done | |
| fi | |
| echo "docker-image=${DOCKER_IMAGE_PREFIX}:${HEAD_SHA}" >> "${GITHUB_OUTPUT}" | |
| echo "head-sha=${HEAD_SHA}" >> "${GITHUB_OUTPUT}" | |
| echo "### Run profiling on [${HEAD_SHA}](https://github.com/vllm-project/vllm/commit/${HEAD_SHA})" >> "${GITHUB_STEP_SUMMARY}" | |
| profiling: | |
| name: Run vLLM profiling | |
| needs: resolve-image | |
| if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'pytorch' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - runs-on: mt-l-x86iavx512-11-125-a100 | |
| device-name: cuda | |
| runs-on: ${{ matrix.runs-on }} | |
| environment: pytorch-x-vllm | |
| permissions: | |
| id-token: write | |
| contents: read | |
| # OSDC/ARC runners run the workload inside this container; the GPU is exposed | |
| # by the runner pod, the same pattern as pytorch/pytorch _linux-test.yml | |
| # (test-osdc) and pytorch/helion. | |
| container: | |
| image: ${{ needs.resolve-image.outputs.docker-image }} | |
| options: --gpus all | |
| steps: | |
| - name: GPU health check | |
| run: nvidia-smi || true | |
| - name: Install git | |
| run: | | |
| # The vLLM CI image may not ship git, which actions/checkout needs | |
| (command -v git || (apt-get update && apt-get install -y git)) || true | |
| git config --global --add safe.directory '*' || true | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set GPU name and type | |
| shell: bash | |
| run: | | |
| set -eux | |
| DEVICE_TYPE=$(nvidia-smi -i 0 --query-gpu=name --format=csv,noheader | awk '{print $2}') | |
| echo "DEVICE_NAME=cuda" >> $GITHUB_ENV | |
| echo "DEVICE_TYPE=$DEVICE_TYPE" >> $GITHUB_ENV | |
| - name: Run vLLM profiling | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| VLLM_USE_MODELSCOPE: false | |
| VLLM_TORCH_PROFILER_DIR: ${{ github.workspace }}/vllm-profiling/profiling-results | |
| CUDA_VISIBLE_DEVICES: 0 | |
| VLLM_USE_V1: 1 | |
| S3_HEAD_SHA: ${{ needs.resolve-image.outputs.head-sha }} | |
| S3_GITHUB_RUN_ID: ${{ github.run_id }} | |
| S3_GITHUB_JOB: ${{ github.job }} | |
| run: | | |
| set -eux | |
| cd vllm-profiling && bash ../.github/scripts/run_vllm_profiling.sh | |
| - name: Authenticate with AWS for the S3 upload | |
| # Ephemeral OSDC pods do not carry a host IAM role, so assume the upload | |
| # role explicitly via OIDC (id-token: write is set above). | |
| uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0 | |
| with: | |
| role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_upload-benchmark-results | |
| # The max duration enforced by the server side | |
| role-duration-seconds: 18000 | |
| aws-region: us-east-1 | |
| - name: Prepare S3 upload metadata | |
| id: prepare_s3_upload | |
| env: | |
| REPOSITORY: vllm-project/vllm | |
| run: | | |
| set -eux | |
| UPLOAD_DATE=$(date -u +"%Y-%m-%d") | |
| echo "upload-date=${UPLOAD_DATE}" >> "${GITHUB_OUTPUT}" | |
| echo "s3-prefix=${UPLOAD_DATE}/${REPOSITORY}" >> "${GITHUB_OUTPUT}" | |
| - name: Upload profiling results to S3 | |
| uses: seemethere/upload-artifact-s3@v5 | |
| with: | |
| s3-prefix: ${{ steps.prepare_s3_upload.outputs.s3-prefix }} | |
| retention-days: 180 | |
| path: vllm-profiling/profiling-results | |
| if-no-files-found: warn | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: profiling-results--${{ env.DEVICE_TYPE }} | |
| path: vllm-profiling/profiling-results |