ci-pr-checks.yaml

name: CI - Test

on:
  push:
    branches:
      - main
      - 'release-*'
  pull_request:
    branches:
      - main

permissions:
  contents: read
  actions: read

concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  check-changes:
    runs-on: ubuntu-latest
    outputs:
      src: ${{ steps.filter.outputs.src }}
      helm: ${{ steps.filter.outputs.helm }}
    steps:
      - name: Checkout source
        uses: actions/checkout@v6
      - uses: dorny/paths-filter@v4
        id: filter
        with:
          filters: |
            src:
              - '**/*.go'
              - Dockerfile.*
              - Makefile*
              - go.mod
              - go.sum
              - scripts/**
              - hack/**
              - test/**
              - .github/actions/docker-build-and-push/action.yml
              - .github/actions/e2e-runner-setup/action.yml
              - .github/workflows/ci-pr-checks.yaml
            helm:
              - 'config/charts/**'
              - 'hack/verify-helm.sh'
              - '.github/workflows/ci-pr-checks.yaml'

  verify-helm-charts:
    needs: check-changes
    if: ${{ needs.check-changes.outputs.helm == 'true' && (github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout source
        uses: actions/checkout@v6

      - name: Set up Go
        uses: actions/setup-go@v6
        with:
          go-version-file: go.mod

      - name: Run make verify-helm-charts
        run: make verify-helm-charts

  test:
    needs: check-changes
    if: ${{ needs.check-changes.outputs.src == 'true' }}
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - name: Checkout source
        uses: actions/checkout@v6

      - name: Create Go cache dirs
        id: go-cache
        run: |
          mkdir -p "$HOME/.cache/llm-d-gomodcache" "$HOME/.cache/llm-d-gobuildcache"
          echo "mod=$HOME/.cache/llm-d-gomodcache" >> "$GITHUB_OUTPUT"
          echo "build=$HOME/.cache/llm-d-gobuildcache" >> "$GITHUB_OUTPUT"

      - name: Cache Go modules and build cache
        uses: actions/cache@v5
        with:
          path: |
            ${{ steps.go-cache.outputs.mod }}
            ${{ steps.go-cache.outputs.build }}
          key: go-cache-${{ runner.os }}-${{ hashFiles('go.sum') }}
          restore-keys: |
            go-cache-${{ runner.os }}-

      # Missing cache is not an error; compare-coverage.sh reports all
      # components as new and exits 0.
      - name: Restore main branch coverage baseline
        if: github.event_name == 'pull_request'
        uses: actions/cache/restore@v5
        with:
          path: coverage/baseline
          key: coverage-main

      - name: Run unit tests
        shell: bash
        env:
          GO_MOD_CACHE_VOL: ${{ steps.go-cache.outputs.mod }}
          GO_BUILD_CACHE_VOL: ${{ steps.go-cache.outputs.build }}
        run: make test-unit

      - name: Run hermetic integration tests
        shell: bash
        env:
          GO_MOD_CACHE_VOL: ${{ steps.go-cache.outputs.mod }}
          GO_BUILD_CACHE_VOL: ${{ steps.go-cache.outputs.build }}
        run: make test-integration-hermetic

      - name: Compare coverage against main baseline
        id: coverage-gate
        if: github.event_name == 'pull_request'
        continue-on-error: true
        shell: bash
        run: make coverage-compare BASELINE_DIR=coverage/baseline

      - name: Find latest release branch
        if: github.event_name == 'pull_request'
        id: find-releases
        shell: bash
        run: |
          gh api "repos/${{ github.repository }}/branches" \
            --paginate \
            --jq '[.[] | select(.name | test("^release-[0-9]")) | .name][]' \
            | sort -V | tail -1 > /tmp/release_branches.txt
          echo "Release branches found:"
          cat /tmp/release_branches.txt
        env:
          GH_TOKEN: ${{ github.token }}

      - name: Download and compare release coverage baselines
        if: github.event_name == 'pull_request'
        shell: bash
        run: |
          if [[ ! -s /tmp/release_branches.txt ]]; then
            echo "No release branches found, skipping."
            exit 0
          fi
          while IFS= read -r branch; do
            [[ -z "$branch" ]] && continue
            echo "Looking up latest successful run for branch: $branch"
            run_id=$(gh run list \
              --repo "${{ github.repository }}" \
              --branch "$branch" \
              --workflow "ci-pr-checks.yaml" \
              --status success \
              --limit 1 \
              --json databaseId \
              --jq '.[0].databaseId // empty')
            if [[ -z "$run_id" ]]; then
              echo "No successful run found for $branch, skipping."
              continue
            fi
            dest="coverage/release-baselines/$branch"
            mkdir -p "$dest"
            if gh run download "$run_id" \
                --repo "${{ github.repository }}" \
                --name "coverage-baseline-$branch" \
                --dir "$dest"; then
              make coverage-compare BASELINE_DIR="$dest" COVERAGE_LABEL="$branch"
            else
              echo "No coverage artifact for $branch in run $run_id, skipping."
            fi
          done < /tmp/release_branches.txt
        env:
          GH_TOKEN: ${{ github.token }}

      - name: Stage coverage baseline for caching
        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
        shell: bash
        run: |
          mkdir -p coverage/baseline
          cp coverage/*.out coverage/baseline/

      - name: Save coverage baseline (main branch only)
        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
        uses: actions/cache/save@v5
        with:
          path: coverage/baseline
          key: coverage-main

      - name: Upload coverage artifact (release branches)
        if: >-
          github.event_name == 'push' &&
          startsWith(github.ref, 'refs/heads/release-')
        uses: actions/upload-artifact@v7
        with:
          name: coverage-baseline-${{ github.ref_name }}
          path: coverage/*.out
          retention-days: 400
          overwrite: true

      - name: Enforce coverage gate
        if: github.event_name == 'pull_request' && steps.coverage-gate.outcome == 'failure'
        run: |
          echo "::error::Coverage regression detected -- see the coverage report in the job summary above."
          exit 1

  e2e-image-common:
    needs: check-changes
    if: ${{ needs.check-changes.outputs.src == 'true' }}
    runs-on: ubuntu-latest
    timeout-minutes: 20
    strategy:
      fail-fast: false
      matrix:
        image:
          - name: builder
            source: build
            docker-file: Dockerfile.builder
            image-name: llm-d-builder
            archive: llm-d-builder
          - name: epp
            source: build
            docker-file: Dockerfile.epp
            image-name: llm-d-router-endpoint-picker
            archive: llm-d-router-endpoint-picker
          - name: simulator
            source: pull
            image: ghcr.io/llm-d/llm-d-inference-sim:v0.9.2
            archive: llm-d-inference-sim
    name: e2e-image-common (${{ matrix.image.name }})
    steps:
      - name: Checkout source
        if: ${{ matrix.image.source == 'build' }}
        uses: actions/checkout@v6

      - name: Build ${{ matrix.image.name }} image
        if: ${{ matrix.image.source == 'build' }}
        uses: ./.github/actions/docker-build-and-push
        with:
          docker-file: ${{ matrix.image.docker-file }}
          image-name: ${{ matrix.image.image-name }}
          tag: dev
          registry: ghcr.io/llm-d
          push: 'false'
          buildx-outputs: type=docker,dest=${{ runner.temp }}/${{ matrix.image.archive }}.tar
          commit-sha: ${{ github.sha }}

      - name: Pull ${{ matrix.image.name }} image
        if: ${{ matrix.image.source == 'pull' }}
        run: |
          docker pull --platform linux/amd64 "${{ matrix.image.image }}"
          docker save "${{ matrix.image.image }}" -o "${{ runner.temp }}/${{ matrix.image.archive }}.tar"

      - name: Upload ${{ matrix.image.name }} image
        uses: actions/upload-artifact@v7
        with:
          name: e2e-image-${{ matrix.image.name }}
          path: ${{ runner.temp }}/${{ matrix.image.archive }}.tar
          retention-days: 1
          compression-level: 0

  e2e-image-router:
    needs: check-changes
    if: ${{ needs.check-changes.outputs.src == 'true' }}
    runs-on: ubuntu-latest
    timeout-minutes: 20
    strategy:
      fail-fast: false
      matrix:
        image:
          - name: sidecar
            source: build
            docker-file: Dockerfile.sidecar
            image-name: llm-d-router-disagg-sidecar
            archive: llm-d-router-disagg-sidecar
          - name: renderer
            source: pull
            image: vllm/vllm-openai-cpu:v0.21.0
            archive: vllm-openai-cpu
    name: e2e-image-router (${{ matrix.image.name }})
    steps:
      - name: Checkout source
        if: ${{ matrix.image.source == 'build' }}
        uses: actions/checkout@v6

      - name: Build ${{ matrix.image.name }} image
        if: ${{ matrix.image.source == 'build' }}
        uses: ./.github/actions/docker-build-and-push
        with:
          docker-file: ${{ matrix.image.docker-file }}
          image-name: ${{ matrix.image.image-name }}
          tag: dev
          registry: ghcr.io/llm-d
          push: 'false'
          buildx-outputs: type=docker,dest=${{ runner.temp }}/${{ matrix.image.archive }}.tar
          commit-sha: ${{ github.sha }}

      - name: Pull ${{ matrix.image.name }} image
        if: ${{ matrix.image.source == 'pull' }}
        run: |
          docker pull --platform linux/amd64 "${{ matrix.image.image }}"
          docker save "${{ matrix.image.image }}" -o "${{ runner.temp }}/${{ matrix.image.archive }}.tar"

      - name: Upload ${{ matrix.image.name }} image
        uses: actions/upload-artifact@v7
        with:
          name: e2e-image-${{ matrix.image.name }}
          path: ${{ runner.temp }}/${{ matrix.image.archive }}.tar
          retention-days: 1
          compression-level: 0

  e2e-gaie:
    needs:
      - check-changes
      - e2e-image-common
    if: ${{ needs.check-changes.outputs.src == 'true' }}
    runs-on: ubuntu-latest
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        suite:
          - name: traffic
            label-filter: "!GAIEMetrics && !GAIELeaderElection"
          - name: metrics
            label-filter: "GAIEMetrics"
    name: e2e-gaie (${{ matrix.suite.name }})
    steps:
      - name: Checkout source
        uses: actions/checkout@v6

      - name: Set up e2e runner
        id: e2e-setup
        uses: ./.github/actions/e2e-runner-setup

      - name: Download builder image
        uses: actions/download-artifact@v8
        with:
          name: e2e-image-builder
          path: ${{ runner.temp }}/e2e-images

      - name: Download EPP image
        uses: actions/download-artifact@v8
        with:
          name: e2e-image-epp
          path: ${{ runner.temp }}/e2e-images

      - name: Download simulator image
        uses: actions/download-artifact@v8
        with:
          name: e2e-image-simulator
          path: ${{ runner.temp }}/e2e-images

      - name: Load e2e images
        shell: bash
        run: |
          for image in "${RUNNER_TEMP}"/e2e-images/*.tar; do
            docker load --input "$image"
          done

      - name: Run e2e-gaie (${{ matrix.suite.name }})
        shell: bash
        env:
          GO_MOD_CACHE_VOL: ${{ steps.e2e-setup.outputs.go-mod-cache }}
          GO_BUILD_CACHE_VOL: ${{ steps.e2e-setup.outputs.go-build-cache }}
          E2E_LABEL_FILTER: ${{ matrix.suite.label-filter }}
          PULL_SIDECAR_IMAGE: "false"
          PULL_VLLM_RENDER_IMAGE: "false"
        run: make test-e2e-gaie-run

  e2e-router:
    needs:
      - check-changes
      - e2e-image-common
      - e2e-image-router
    if: ${{ needs.check-changes.outputs.src == 'true' }}
    runs-on: ubuntu-latest
    timeout-minutes: 45
    strategy:
      fail-fast: false
      matrix:
        suite:
          - name: pd
            label-filter: "!Disruptive && !Extended && !SharedStorage && !Metrics"
            needs-renderer: "false"
          - name: pd-shared-storage-deprecated
            label-filter: "SharedStorage && DeprecatedPD"
            needs-renderer: "false"
          - name: pd-shared-storage-disagg
            label-filter: "SharedStorage && Disagg"
            needs-renderer: "false"
          - name: pd-metrics
            label-filter: "Metrics"
            needs-renderer: "false"
          - name: extended
            label-filter: "Extended"
            needs-renderer: "true"
          - name: disruption
            label-filter: "Disruptive"
            needs-renderer: "false"
    name: e2e-router (${{ matrix.suite.name }})
    steps:
      - name: Checkout source
        uses: actions/checkout@v6

      - name: Set up e2e runner
        id: e2e-setup
        uses: ./.github/actions/e2e-runner-setup

      - name: Download builder image
        uses: actions/download-artifact@v8
        with:
          name: e2e-image-builder
          path: ${{ runner.temp }}/e2e-images

      - name: Download EPP image
        uses: actions/download-artifact@v8
        with:
          name: e2e-image-epp
          path: ${{ runner.temp }}/e2e-images

      - name: Download sidecar image
        uses: actions/download-artifact@v8
        with:
          name: e2e-image-sidecar
          path: ${{ runner.temp }}/e2e-images

      - name: Download simulator image
        uses: actions/download-artifact@v8
        with:
          name: e2e-image-simulator
          path: ${{ runner.temp }}/e2e-images

      - name: Download renderer image
        if: ${{ matrix.suite.needs-renderer == 'true' }}
        uses: actions/download-artifact@v8
        with:
          name: e2e-image-renderer
          path: ${{ runner.temp }}/e2e-images

      - name: Load e2e images
        shell: bash
        run: |
          for image in "${RUNNER_TEMP}"/e2e-images/*.tar; do
            docker load --input "$image"
          done

      - name: Run e2e-router (${{ matrix.suite.name }})
        shell: bash
        env:
          GO_MOD_CACHE_VOL: ${{ steps.e2e-setup.outputs.go-mod-cache }}
          GO_BUILD_CACHE_VOL: ${{ steps.e2e-setup.outputs.go-build-cache }}
          E2E_LABEL_FILTER: ${{ matrix.suite.label-filter }}
          LOAD_VLLM_RENDER_IMAGE: ${{ matrix.suite.needs-renderer }}
          PULL_VLLM_RENDER_IMAGE: "false"
        run: make test-e2e-scheduler-run