ci.yml

name: CI

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  # The merge queue creates a temporary branch and fires `merge_group`; without
  # this trigger the required checks never start and queued PRs stall forever.
  merge_group:
    branches: [main]
    types: [checks_requested]
  # Manual trigger for on-demand E2E runs on the self-hosted GPU runners, without
  # waiting on a push/PR or the full gate. Dispatch against any ref
  # (`gh workflow run ci.yml --ref <branch> -f platform=... -f tier=...`) — GitHub
  # runs THAT ref's copy of this file, so the E2E jobs it selects live on the ref.
  # A dispatch skips build-and-test (see its `if:`) for a fast loop. This trigger
  # must exist on the default branch for the workflow to be dispatchable at all.
  workflow_dispatch:
    inputs:
      platform:
        description: Which runner(s) to target
        type: choice
        default: all
        options: [all, mock, app-dev-gpu, strix-ubuntu, strix-windows]
      tier:
        description: Which scenario tier(s) to run
        type: choice
        default: both
        options: [both, expect-pass, known-bugs]

concurrency:
  # Manual dispatches get a UNIQUE group (run_id) so a run that gets stuck — e.g.
  # a job queued on a temporarily offline self-hosted runner, which GitHub cannot
  # cancel — never holds the shared group and blocks later dispatches. push / PR /
  # merge_group keep the shared per-ref group so a new commit still supersedes the
  # previous in-flight run.
  group: >-
    ${{ github.workflow }}-${{ github.ref }}-${{
      github.event_name == 'workflow_dispatch' && github.run_id || 'shared' }}
  cancel-in-progress: true

permissions:
  contents: read

env:
  CARGO_TERM_COLOR: always

jobs:
  # Decide which file categories a change touches so unaffected jobs can skip
  # their expensive work. Filtering runs ONLY on pull_request; on push and in the
  # merge queue every category is forced true, so post-merge `main` and queued
  # PRs always run the full suite. That keeps the skip from ever starving a
  # required check in the merge queue (a never-produced required check stalls the
  # queue forever — see the `commit-signatures` note below). Each output folds in
  # `.github/workflows/**`, so any change to CI itself triggers a full run.
  changes:
    runs-on: ubuntu-latest
    # paths-filter reads the PR's changed-file list from the API on
    # pull_request events; that needs pull-requests:read on top of the
    # workflow-wide contents:read (job-level permissions replace the default
    # set, so contents:read is repeated here for checkout).
    permissions:
      contents: read
      pull-requests: read
    outputs:
      # `<filter> || <fallback>`: the filter result on PRs (empty string when the
      # filter step is skipped off-PR), otherwise the forced 'true'.
      rust: ${{ steps.filter.outputs.rust || steps.all.outputs.forced }}
      heavy: ${{ steps.filter.outputs.heavy || steps.all.outputs.forced }}
      lint: ${{ steps.filter.outputs.lint || steps.all.outputs.forced }}
      tpn: ${{ steps.filter.outputs.tpn || steps.all.outputs.forced }}
    steps:
      - uses: actions/checkout@v6

      - name: Filter changed paths (pull requests only)
        id: filter
        if: github.event_name == 'pull_request'
        uses: dorny/paths-filter@d1c1ffe0248fe513906c8e24db8ea791d46f8590 # v3.0.2
        with:
          filters: |
            # Rust-only checks (fmt, clippy, coverage).
            rust:
              - '**/*.rs'
              - '**/Cargo.toml'
              - 'Cargo.lock'
              - 'rust-toolchain*'
              # clippy runs `cargo xtask manifest --check` against this file.
              - 'MANIFEST.md'
              - '.github/workflows/**'
            # build-and-test runs cargo AND the python/shell smoke + acceptance
            # steps, so it depends on the Rust set plus those scripts.
            heavy:
              - '**/*.rs'
              - '**/Cargo.toml'
              - 'Cargo.lock'
              - 'rust-toolchain*'
              - 'scripts/**'
              - 'engines/**'
              - '**/*.py'
              - '**/*.sh'
              - '**/*.ps1'
              - '**/*.feature'
              - 'install*'
              - '.github/workflows/**'
            # THIRD_PARTY_NOTICES.txt staleness gate: anything that changes the
            # dependency tree, the cargo-about config/template, the generator, or
            # the generated file itself.
            tpn:
              - 'Cargo.lock'
              - '**/Cargo.toml'
              - 'about.toml'
              - 'about.hbs'
              - 'THIRD_PARTY_NOTICES.txt'
              - 'xtask/**'
              - '.github/workflows/**'
            # Python / Shell / PowerShell lint job.
            lint:
              - 'scripts/**'
              - 'engines/**'
              - '**/*.py'
              - '**/*.sh'
              - '**/*.ps1'
              - 'ruff.toml'
              - 'PSScriptAnalyzerSettings.psd1'
              - '.github/workflows/**'

      - name: Force full run off pull requests
        id: all
        if: github.event_name != 'pull_request'
        run: echo "forced=true" >> "$GITHUB_OUTPUT"

  build-and-test:
    runs-on: ubuntu-latest
    # Don't spend per-OS build/test cycles unless the cheap lint gate is green.
    needs: [changes, clippy, prek]
    # A manual E2E dispatch skips this heavy job for a fast loop; the E2E jobs
    # tolerate a skipped build-and-test in their own `if:` guards.
    if: github.event_name != 'workflow_dispatch'
    steps:
      - uses: actions/checkout@v6

      - name: Install native build deps
        if: needs.changes.outputs.heavy == 'true'
        run: sudo apt-get update && sudo apt-get install -y pkg-config libcap-dev

      - uses: actions-rust-lang/setup-rust-toolchain@v1
        if: needs.changes.outputs.heavy == 'true'

      # Build the runtime binaries the smoke + acceptance steps below need (every
      # engine binary; smoke hard-fails on a missing one). Unit/integration tests
      # run in the dedicated `test` job on only the crates a change can affect;
      # `clippy --all-targets` is the compile-check for the non-binary targets, so
      # dropping `--all-targets` here loses no coverage.
      - name: Build
        if: needs.changes.outputs.heavy == 'true'
        run: cargo build --workspace

      - name: Local no-fallback smoke
        if: needs.changes.outputs.heavy == 'true'
        run: python scripts/smoke_local.py --skip-build

      - name: Acceptance harness self-tests
        if: needs.changes.outputs.heavy == 'true'
        run: |
          python scripts/comfyui_therock_gpu_test.py --self-test
          python scripts/local_assistant_therock_gpu_test.py --self-test
          python scripts/vllm_therock_gpu_test.py --self-test
          python scripts/wsl_preflight.py --self-test

      - name: Portable WSL build deps self-test
        if: needs.changes.outputs.heavy == 'true'
        run: bash scripts/setup-wsl-portable-build-deps.sh --self-test

      - name: Release readiness self-test
        if: needs.changes.outputs.heavy == 'true'
        run: python scripts/release_readiness.py --self-test

      - name: Acceptance install lifecycle
        if: needs.changes.outputs.heavy == 'true'
        run: ./scripts/acceptance-install-upgrade-tui-uninstall.sh

  # Fine-grained test selection (the second half of smarter CI; the path-based
  # job skip is the `changes` job above). Runs cargo's unit + integration tests
  # for only the crates a change can reach — the changed crates plus their
  # transitive dependents, computed by `cargo xtask affected` from the workspace
  # dependency graph — instead of the whole workspace.
  #
  # Merge-queue safety: the narrowing happens ONLY on pull_request. On push and
  # in the merge queue the selection is forced to `--workspace`, so post-merge
  # `main` and every queued PR run the full suite — a required check is never
  # starved and the queue cannot stall. The full Windows suite
  # (windows-build-and-test) also runs the whole workspace on every change, so it
  # backstops any crate a graph-based selection could miss (e.g. an integration
  # test that drives another crate only at runtime).
  test:
    name: Test (affected crates)
    runs-on: ubuntu-latest
    # Same lint gate as the OS build jobs: don't spend a test cycle until the
    # cheap checks are green.
    needs: [changes, clippy, prek]
    # A manual E2E dispatch only exercises the E2E jobs; skip the rest.
    if: github.event_name != 'workflow_dispatch'
    steps:
      - uses: actions/checkout@v6
        with:
          # `cargo xtask affected` diffs origin/<base>...HEAD, so it needs the
          # base history and the real PR head (not the synthetic merge ref).
          fetch-depth: 0
          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || '' }}

      - name: Install native build deps
        if: needs.changes.outputs.rust == 'true'
        run: sudo apt-get update && sudo apt-get install -y pkg-config libcap-dev

      - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0
        if: needs.changes.outputs.rust == 'true'
        with:
          # Keep the PR (subset) and push/merge (`--workspace`) builds in separate
          # cache lanes. A subset build resolves Cargo features differently than
          # the whole workspace, so a shared cache would make the two evict and
          # recompile each other's artifacts on every alternation.
          cache-key: ${{ github.event_name == 'pull_request' && 'affected-pr' || 'full' }}

      - name: Install cargo-nextest
        if: needs.changes.outputs.rust == 'true'
        uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6
        with:
          tool: nextest

      - name: Select affected crates
        id: sel
        if: needs.changes.outputs.rust == 'true'
        run: |
          # The `cargo xtask` alias builds xtask in release (target/release), so
          # computing the selection doesn't perturb the debug feature resolution
          # the nextest build below uses.
          if [ "${{ github.event_name }}" = "pull_request" ]; then
            sel=$(cargo xtask affected --base "origin/${{ github.base_ref }}")
          else
            sel="--workspace"
          fi
          echo "affected selection: ${sel:-<no rust crates changed>}"
          echo "sel=$sel" >> "$GITHUB_OUTPUT"

      - name: Test (nextest)
        # Skip cleanly when nothing Rust-relevant resolved to a crate, so the
        # required check still reports success.
        if: needs.changes.outputs.rust == 'true' && steps.sel.outputs.sel != ''
        # `sel` is intentionally unquoted: it expands to either `--workspace` or
        # several `-p <crate>` flags that must word-split into separate argv
        # entries. The values are crate names from our own tool, not user input.
        run: cargo nextest run ${{ steps.sel.outputs.sel }}

  windows-build-and-test:
    runs-on: windows-latest
    env:
      CARGO_INCREMENTAL: "0"
    # Don't spend per-OS build/test cycles unless the cheap lint gate is green.
    needs: [changes, clippy, prek]
    # A manual E2E dispatch only exercises the E2E jobs; skip the rest.
    if: github.event_name != 'workflow_dispatch'
    steps:
      - uses: actions/checkout@v6

      - uses: actions-rust-lang/setup-rust-toolchain@v1
        if: needs.changes.outputs.heavy == 'true'
        with:
          cache-on-failure: true

      - name: Build
        if: needs.changes.outputs.heavy == 'true'
        shell: pwsh
        run: |
          cargo build --workspace --all-targets
          cargo build --release -p rocm -p rocmd -p rocm-engine-lemonade -p rocm-engine-vllm -p xtask

      - name: Test
        if: needs.changes.outputs.heavy == 'true'
        shell: pwsh
        run: cargo test --workspace --all-targets

      - name: Local no-fallback smoke
        if: needs.changes.outputs.heavy == 'true'
        shell: pwsh
        run: python .\scripts\smoke_local.py --skip-build

      - name: Acceptance harness self-tests
        if: needs.changes.outputs.heavy == 'true'
        shell: pwsh
        run: |
          python .\scripts\comfyui_therock_gpu_test.py --self-test
          python .\scripts\local_assistant_therock_gpu_test.py --self-test
          python .\scripts\vllm_therock_gpu_test.py --self-test
          python .\scripts\wsl_preflight.py --self-test

      - name: Release readiness self-test
        if: needs.changes.outputs.heavy == 'true'
        shell: pwsh
        run: python .\scripts\release_readiness.py --self-test

      - name: Check PowerShell installer syntax
        if: needs.changes.outputs.heavy == 'true'
        shell: pwsh
        run: |
          $null = [scriptblock]::Create((Get-Content .\install.ps1 -Raw))
          $null = [scriptblock]::Create((Get-Content .\scripts\package-windows-release.ps1 -Raw))

      - name: Acceptance install lifecycle
        if: needs.changes.outputs.heavy == 'true'
        shell: pwsh
        run: .\scripts\acceptance-install-upgrade-tui-uninstall.ps1

  prek:
    # The fast lint gate: every toolchain-light lint/hygiene check, run from the
    # single source of truth in .pre-commit-config.yaml — hygiene + Python (ruff)
    # + Shell (shellcheck) + cargo fmt (rustfmt only, no compile).
    # `--no-group local-tools` skips the hooks that need a full Rust compile,
    # hawkeye, or PowerShell — those have their own jobs (clippy, build-and-test,
    # license-headers, powershell-lint).
    name: prek (lint / hygiene)
    runs-on: ubuntu-latest
    # Always-on: prek is the cheapest gate (no compile) and its hygiene hooks run
    # on every file type, so it is relevant to any change — including docs-only
    # PRs that match no `changes` category. Like license-headers and
    # commit-signatures, it is not gated.
    # Exception: a manual E2E dispatch only exercises the E2E jobs; skip the rest.
    if: github.event_name != 'workflow_dispatch'
    steps:
      - uses: actions/checkout@v6

      # cargo-fmt hook needs rustfmt (no build).
      - uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          components: rustfmt
          cache: false

      # Supply-chain hardening: the action is pinned to a commit SHA and prek to
      # an exact version. The action ships a hardcoded SHA-256 checksum map
      # (src/known-checksums.ts) and verifies the downloaded prek archive against
      # it, so a tampered release can't be installed. This commit is past the
      # v2.0.4 tag because it is the one that added prek 0.4.5 to that map — pin
      # an earlier commit and the action would not know 0.4.5 and skip the check.
      - uses: j178/prek-action@dec4adcd2fa1db9d50a39668b22112a37235e730 # prek 0.4.5 checksums
        with:
          prek-version: 0.4.5
          extra-args: --all-files --no-group local-tools

  clippy:
    runs-on: ubuntu-latest
    needs: changes
    # A manual E2E dispatch only exercises the E2E jobs; skip the rest.
    if: github.event_name != 'workflow_dispatch'
    steps:
      - uses: actions/checkout@v6

      - uses: actions-rust-lang/setup-rust-toolchain@v1
        if: needs.changes.outputs.rust == 'true'

      - name: Clippy
        if: needs.changes.outputs.rust == 'true'
        run: cargo clippy --locked --workspace --all-targets -- -D warnings

      - name: MANIFEST.md dependency table is current
        if: needs.changes.outputs.rust == 'true'
        run: cargo xtask manifest --check

  third-party-notices:
    name: Third-party notices current
    runs-on: ubuntu-latest
    needs: changes
    # A manual E2E dispatch only exercises the E2E jobs; skip the rest.
    if: github.event_name != 'workflow_dispatch'
    steps:
      - uses: actions/checkout@v6

      - uses: actions-rust-lang/setup-rust-toolchain@v1
        if: needs.changes.outputs.tpn == 'true'

      # Pin the cargo-about version so the check compares against the exact same
      # generator contributors use; a different version can format the notices
      # differently and fail the byte-for-byte comparison. This version has no
      # prebuilt release binary and its `cargo install` needs the `cli` feature,
      # so build from source (cached by version); `--locked` uses cargo-about's
      # own lockfile for a reproducible generator.
      - name: Cache cargo-about
        if: needs.changes.outputs.tpn == 'true'
        id: cache-cargo-about
        uses: actions/cache@v5
        with:
          path: ~/.cargo/bin/cargo-about
          key: ${{ runner.os }}-cargo-about-0.9.1

      - name: Install cargo-about
        if: needs.changes.outputs.tpn == 'true' && steps.cache-cargo-about.outputs.cache-hit != 'true'
        run: cargo install cargo-about@0.9.1 --locked --features cli

      - name: THIRD_PARTY_NOTICES.txt is current
        if: needs.changes.outputs.tpn == 'true'
        run: cargo xtask tpn --check

  coverage:
    name: Coverage (rocm-dash crates, ratcheted)
    runs-on: ubuntu-latest
    needs: changes
    # A manual E2E dispatch only exercises the E2E jobs; skip the rest.
    if: github.event_name != 'workflow_dispatch'
    steps:
      - uses: actions/checkout@v6

      - name: Install native build deps
        if: needs.changes.outputs.rust == 'true'
        run: sudo apt-get update && sudo apt-get install -y pkg-config libcap-dev

      - uses: dtolnay/rust-toolchain@1.96.0
        if: needs.changes.outputs.rust == 'true'
        with:
          components: llvm-tools-preview

      - name: Install cargo-llvm-cov
        if: needs.changes.outputs.rust == 'true'
        uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6
        with:
          tool: cargo-llvm-cov

      - name: Cache cargo registry and build
        if: needs.changes.outputs.rust == 'true'
        uses: actions/cache@v5
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            target
          key: ${{ runner.os }}-cargo-cov-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-cargo-cov-

      # Ratcheted, fail-on-regression floor scoped to the transplanted rocm-dash
      # crates (the clean anchor) — deliberately NOT the whole workspace, so the
      # gate does not over-claim coverage of the large untyped rocm-cli core
      # Baseline measured 2026-06-11: 73.7% lines across the four
      # crates. Floor set just below measured; ratchet upward as the larger TUI
      # tab files (bench/overview/modal) gain tests.
      - name: Coverage gate (rocm-dash crates, >= 70% lines)
        if: needs.changes.outputs.rust == 'true'
        run: |
          cargo llvm-cov --no-cfg-coverage \
            -p rocm-dash-core -p rocm-dash-collectors \
            -p rocm-dash-daemon -p rocm-dash-tui \
            --fail-under-lines 70

  powershell-lint:
    # Runs the prek `powershell-script-analyzer` hook's logic (cargo xtask
    # powershell-lint) under BOTH PowerShell editions. windows-latest is the only
    # runner with Windows PowerShell 5.1 (`powershell`); it also ships PowerShell
    # 7 (`pwsh`). Kept separate from windows-build-and-test so it runs in parallel
    # rather than behind that slow job.
    name: Lint (PowerShell)
    runs-on: windows-latest
    needs: changes
    # A manual E2E dispatch only exercises the E2E jobs; skip the rest.
    if: github.event_name != 'workflow_dispatch'
    steps:
      - uses: actions/checkout@v6

      - uses: actions-rust-lang/setup-rust-toolchain@v1
        if: needs.changes.outputs.lint == 'true'

      - name: Install PSScriptAnalyzer (PowerShell 7)
        if: needs.changes.outputs.lint == 'true'
        shell: pwsh
        run: Install-Module PSScriptAnalyzer -Scope CurrentUser -Force -RequiredVersion 1.25.0

      - name: Install PSScriptAnalyzer (Windows PowerShell 5.1)
        if: needs.changes.outputs.lint == 'true'
        shell: powershell
        run: |
          [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
          Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force | Out-Null
          Install-Module PSScriptAnalyzer -Scope CurrentUser -Force -RequiredVersion 1.25.0

      - name: PSScriptAnalyzer (PowerShell 7)
        if: needs.changes.outputs.lint == 'true'
        run: cargo xtask powershell-lint --shell pwsh

      - name: PSScriptAnalyzer (Windows PowerShell 5.1)
        if: needs.changes.outputs.lint == 'true'
        run: cargo xtask powershell-lint --shell powershell

  license-headers:
    name: License header check (hawkeye)
    runs-on: ubuntu-latest
    env:
      # Install a pinned prebuilt hawkeye binary instead of pulling the
      # korandoru/hawkeye Docker image on every run, and verify it against a
      # sha256 recorded here. The hash lives in-repo, out of band from the
      # release: GitHub release assets are mutable, so a republished/tampered
      # artifact (and its co-published .sha256) would pass an upstream-only
      # check — pinning the hash here makes that fail instead. To upgrade, bump
      # the version and replace the hash with the new release's published
      # .sha256 (confirm the bytes first).
      HAWKEYE_VERSION: v6.5.1
      HAWKEYE_SHA256: d6eb0505a45a15244f4f789158aafe5e3f1a7dc86c9dc1d7651f3cb1e1b321e0
      HAWKEYE_INSTALL_DIR: ${{ github.workspace }}/.hawkeye-bin
    # A manual E2E dispatch only exercises the E2E jobs; skip the rest.
    if: github.event_name != 'workflow_dispatch'
    steps:
      - uses: actions/checkout@v6

      - name: Cache hawkeye binary
        id: cache-hawkeye
        uses: actions/cache@v5
        with:
          path: ${{ env.HAWKEYE_INSTALL_DIR }}
          key: ${{ runner.os }}-hawkeye-${{ env.HAWKEYE_VERSION }}-${{ env.HAWKEYE_SHA256 }}

      - name: Install hawkeye
        if: steps.cache-hawkeye.outputs.cache-hit != 'true'
        run: |
          tarball="hawkeye-x86_64-unknown-linux-gnu.tar.xz"
          curl --proto '=https' --tlsv1.2 -LsSf \
            "https://github.com/korandoru/hawkeye/releases/download/${HAWKEYE_VERSION}/${tarball}" \
            -o "${tarball}"
          echo "${HAWKEYE_SHA256}  ${tarball}" | sha256sum -c -
          mkdir -p "${HAWKEYE_INSTALL_DIR}"
          tar -xJf "${tarball}" -C "${HAWKEYE_INSTALL_DIR}" --strip-components=1
          rm -f "${tarball}"

      - name: Check license headers
        run: |
          "${HAWKEYE_INSTALL_DIR}/hawkeye" check

  commit-signatures:
    name: Commit signatures + sign-off
    runs-on: ubuntu-latest
    # Runs on pull requests and in the merge queue. Both define a base that
    # bounds the commit range. Handling `merge_group` matters once this is a
    # *required* check: the queue fires `merge_group` (not `pull_request`), so a
    # job gated to PRs only would never produce the required check and the queue
    # entry would stall forever.
    if: github.event_name == 'pull_request' || github.event_name == 'merge_group'
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          # PR: check out the PR head, not the synthetic refs/pull/N/merge commit
          # (which is unsigned and lacks a sign-off, and would otherwise appear
          # in origin/<base>..HEAD and fail the gate on every PR).
          # merge_group: check out the queue head — the rebased/squashed commits
          # GitHub built for the group. Merge commits are disabled on this repo,
          # so these are normal commits that preserve their Signed-off-by trailer
          # and are GitHub "Verified" (web-flow signed).
          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event.merge_group.head_sha }}

      - uses: actions-rust-lang/setup-rust-toolchain@v1

      - name: Verify commits are signed and signed-off
        # `--require-verified` shells out to the `gh` CLI; pass the job token so
        # `gh api` is authenticated (avoids the unauthenticated rate limit).
        # Base: the PR's base branch, or — in the queue — the exact commit the
        # merge group was built on (`base_sha`), so the range is just the queued
        # commits and not whatever else has since landed on the base branch.
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          BASE: ${{ github.event_name == 'pull_request' && format('origin/{0}', github.base_ref) || github.event.merge_group.base_sha }}
        run: cargo xtask verify-commits --base "$BASE" --require-verified

  # Hardware (real AMD GPU / WSL) smoke tests on dedicated self-hosted runners
  # are intentionally not part of this workflow yet. See
  # docs/ci-hardware-testing.md for the planned design; they will land in a
  # follow-up PR.

  # ── E2E tests (cucumber-rs) ───────────────────────────────────────────────
  #
  # BDD scenarios in Gherkin (.feature files) backed by Rust step functions.
  # Three selections: blocking mock tier, non-blocking known bugs, non-blocking
  # GPU. Cucumber tag filters are exact-match, so known-bug scenarios carry a
  # bare `@expected-failure` (plus `@expected-failure-EAI-NNNN` for traceability)
  # and hardware-dependent scenarios carry `@gpu`.

  # Blocking mock-tier job: must stay green. Known-bug scenarios and
  # hardware-dependent scenarios are excluded here and run in the non-blocking
  # jobs below.
  e2e:
    name: E2E tests
    timeout-minutes: 15
    runs-on: ubuntu-latest
    needs: [changes, build-and-test]
    # On push/PR/merge_group: unchanged — run when build-and-test succeeded and
    # the change is heavy. On workflow_dispatch: build-and-test is skipped, so
    # tolerate that and gate on the platform/tier inputs instead (mock tier here).
    if: >-
      always()
      && needs.changes.result == 'success'
      && (
        (github.event_name != 'workflow_dispatch'
          && needs.build-and-test.result == 'success'
          && needs.changes.outputs.heavy == 'true')
        || (github.event_name == 'workflow_dispatch'
          && (inputs.platform == 'all' || inputs.platform == 'mock')
          && (inputs.tier == 'both' || inputs.tier == 'expect-pass'))
      )
    steps:
      - uses: actions/checkout@v6

      - name: Install native build deps
        run: sudo apt-get update && sudo apt-get install -y pkg-config libcap-dev

      - uses: actions-rust-lang/setup-rust-toolchain@v1

      # The workspace test job selects only affected crates and runs the cucumber
      # harness, not the library's own unit tests — run the report generator's
      # unit tests (stats counting, scenario status, date formatting) here.
      - name: Unit tests (e2e-cucumber lib)
        run: cargo test -p e2e-cucumber --lib

      - name: Run E2E tests (mock tier, excluding known bugs)
        run: cargo xtask e2e -- -t "not @gpu and not @expected-failure"

      - name: Upload E2E report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: e2e-report
          path: tests/e2e-cucumber/results/

  # Runs ONLY the known-bug scenarios under xfail inversion (`--expect-failures`):
  # a tagged scenario failing is the expected, green outcome. The job goes red
  # only when a known bug is silently fixed (XPASS — remove its @expected-failure
  # tag so it joins the blocking e2e job above), an untagged scenario fails, or a
  # parse/hook error occurs — all actionable, so this gates rather than being an
  # always-red check reviewers learn to ignore.
  e2e-known-bugs:
    name: E2E tests (known bugs)
    timeout-minutes: 15
    runs-on: ubuntu-latest
    needs: [changes, build-and-test]
    # See `e2e`: dispatch tolerates skipped build-and-test; mock tier, known-bugs.
    if: >-
      always()
      && needs.changes.result == 'success'
      && (
        (github.event_name != 'workflow_dispatch'
          && needs.build-and-test.result == 'success'
          && needs.changes.outputs.heavy == 'true')
        || (github.event_name == 'workflow_dispatch'
          && (inputs.platform == 'all' || inputs.platform == 'mock')
          && (inputs.tier == 'both' || inputs.tier == 'known-bugs'))
      )
    steps:
      - uses: actions/checkout@v6

      - name: Install native build deps
        run: sudo apt-get update && sudo apt-get install -y pkg-config libcap-dev

      - uses: actions-rust-lang/setup-rust-toolchain@v1

      - name: Run known-bug E2E scenarios (mock tier)
        run: cargo xtask e2e --expect-failures -- -t "@expected-failure and not @gpu"

      - name: Upload E2E report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: e2e-known-bugs-report
          path: tests/e2e-cucumber/results/

  # GPU hardware scenarios that are expected to PASS. Mirrors the mock `e2e` job
  # but for `@gpu` hardware-dependent scenarios; known-bug `@gpu` scenarios are
  # split into e2e-gpu-known-bugs below so a red here always means a real GPU
  # regression, never a known bug still reproducing.
  e2e-gpu:
    name: E2E tests (GPU)
    timeout-minutes: 15
    runs-on: [self-hosted, linux, amd-gpu]
    needs: [changes, build-and-test]
    # See `e2e`: dispatch tolerates skipped build-and-test; app-dev-gpu, expect-pass.
    if: >-
      always()
      && needs.changes.result == 'success'
      && (
        (github.event_name != 'workflow_dispatch'
          && needs.build-and-test.result == 'success'
          && needs.changes.outputs.heavy == 'true')
        || (github.event_name == 'workflow_dispatch'
          && (inputs.platform == 'all' || inputs.platform == 'app-dev-gpu')
          && (inputs.tier == 'both' || inputs.tier == 'expect-pass'))
      )
    continue-on-error: true
    steps:
      - uses: actions/checkout@v6

      # cache: false — on this self-hosted runner we persist the build cache
      # ourselves via CARGO_TARGET_DIR (below). The action's built-in
      # Swatinem/rust-cache otherwise tries to SAVE the large target dir to
      # GitHub's cache service in a post-step (slow/hangs) and its cleanup wipes
      # the local target.
      - uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          cache: false

      # `actions/checkout` runs `git clean -ffdx`, which deletes the gitignored
      # `target/` inside the repo every job → a full ~15min rebuild each run.
      # Point CARGO_TARGET_DIR at a sibling of the checkout ($RUNNER_WORKSPACE is
      # the checkout's parent — untouched by git clean and persistent between jobs
      # on a self-hosted runner), so cargo rebuilds incrementally: unchanged →
      # reused, changed → only the diff.
      - name: Run E2E tests on GPU hardware (excluding known bugs)
        run: |
          export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target"
          # Share heavy immutable artifacts (TheRock runtimes ~3.3GB, HF weights,
          # vLLM venv) across scenarios so they download once per runner, not per
          # scenario. Persistent path; service state stays isolated per scenario.
          export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared"
          cargo xtask e2e -- -t "@gpu and not @expected-failure"

      - name: Upload E2E report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: e2e-gpu-report
          path: tests/e2e-cucumber/results/

  # GPU counterpart of e2e-known-bugs: runs ONLY the `@gpu @expected-failure`
  # scenarios under xfail inversion (`--expect-failures`). Green when each known
  # GPU bug still fails as expected; red only when one is silently fixed (XPASS —
  # remove its @expected-failure tag so it joins e2e-gpu above), an untagged
  # scenario fails, or a parse/hook error occurs.
  e2e-gpu-known-bugs:
    name: E2E tests (GPU, known bugs)
    timeout-minutes: 15
    runs-on: [self-hosted, linux, amd-gpu]
    needs: [changes, build-and-test]
    # See `e2e`: dispatch tolerates skipped build-and-test; app-dev-gpu, known-bugs.
    if: >-
      always()
      && needs.changes.result == 'success'
      && (
        (github.event_name != 'workflow_dispatch'
          && needs.build-and-test.result == 'success'
          && needs.changes.outputs.heavy == 'true')
        || (github.event_name == 'workflow_dispatch'
          && (inputs.platform == 'all' || inputs.platform == 'app-dev-gpu')
          && (inputs.tier == 'both' || inputs.tier == 'known-bugs'))
      )
    continue-on-error: true
    env:
      # Known-bug scenarios are expected to fail, so don't grant a serve the full
      # cold-start window — fail fast instead of waiting the default 600s when a
      # bug manifests as a serve that never becomes ready.
      E2E_SERVE_TIMEOUT_SECS: "90"
    steps:
      - uses: actions/checkout@v6

      # See e2e-gpu: cache: false because we persist the cache ourselves via
      # CARGO_TARGET_DIR; the action's rust-cache save/cleanup is slow and harmful
      # on this self-hosted runner.
      - uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          cache: false

      # See e2e-gpu: keep the build cache out of the checkout (git clean would
      # wipe it) for incremental rebuilds across jobs.
      - name: Run known-bug E2E scenarios on GPU hardware
        run: |
          export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target"
          # See e2e-gpu: share heavy immutable artifacts across scenarios.
          export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared"
          cargo xtask e2e --expect-failures -- -t "@gpu and @expected-failure"

      - name: Upload E2E report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: e2e-gpu-known-bugs-report
          path: tests/e2e-cucumber/results/

  # Second AMD GPU architecture: the Strix Halo (gfx1151) Ubuntu runner. Same
  # expect-pass / known-bugs split as app-dev-gpu's e2e-gpu jobs, targeted by the
  # `strix-halo` label so it lands only on that runner (app-dev-gpu carries
  # `amd-gpu`, not `strix-halo`). Non-blocking while this hardware is proven out.
  e2e-gpu-strix-ubuntu:
    name: E2E tests (Strix Halo, Ubuntu)
    timeout-minutes: 15
    runs-on: [self-hosted, linux, strix-halo]
    needs: [changes, build-and-test]
    # See `e2e`: dispatch tolerates skipped build-and-test; strix-ubuntu, expect-pass.
    if: >-
      always()
      && needs.changes.result == 'success'
      && (
        (github.event_name != 'workflow_dispatch'
          && needs.build-and-test.result == 'success'
          && needs.changes.outputs.heavy == 'true')
        || (github.event_name == 'workflow_dispatch'
          && (inputs.platform == 'all' || inputs.platform == 'strix-ubuntu')
          && (inputs.tier == 'both' || inputs.tier == 'expect-pass'))
      )
    continue-on-error: true
    # On this runner `/`, `/home/ubuntu`, and `/tmp` are ALL on a full root
    # partition; only /home/ubuntu/actions-runner (a 1.7T nvme) has space. So
    # EVERYTHING the job writes must land on the nvme. Point HOME there (catches
    # ~/.cache/pip, ~/.config, and any other $HOME writer — pip's cache under the
    # real /home/ubuntu is what previously failed `install sdk` with ENOSPC),
    # plus the toolchain, temp, and pip cache. The rustup bootstrap still uses
    # --no-modify-path so it doesn't touch $HOME/.profile. (The HOME override
    # brings back a cosmetic rustup euid/HOME warning — accepted: not filling the
    # root disk matters more.)
    env:
      HOME: /home/ubuntu/actions-runner/e2e-home
      CARGO_HOME: /home/ubuntu/actions-runner/.cargo
      RUSTUP_HOME: /home/ubuntu/actions-runner/.rustup
      TMPDIR: /home/ubuntu/actions-runner/tmp
      PIP_CACHE_DIR: /home/ubuntu/actions-runner/pip-cache
    steps:
      - uses: actions/checkout@v6

      - name: Prepare writable dirs on the nvme
        run: mkdir -p /home/ubuntu/actions-runner/e2e-home /home/ubuntu/actions-runner/tmp /home/ubuntu/actions-runner/pip-cache

      # Bootstrap rustup ourselves with --no-modify-path so it never writes to
      # $HOME/.profile (setup-rust-toolchain doesn't expose that flag).
      # rust-toolchain.toml pins the exact toolchain, installed on first cargo
      # use. Idempotent.
      - name: Ensure Rust toolchain
        run: |
          if ! command -v cargo >/dev/null 2>&1 && [ ! -x "$CARGO_HOME/bin/cargo" ]; then
            curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \
              | sh -s -- -y --no-modify-path --default-toolchain none
          fi
          echo "$CARGO_HOME/bin" >> "$GITHUB_PATH"

      - name: Run E2E tests on Strix Halo (excluding known bugs)
        run: |
          export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target"
          export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared"
          cargo xtask e2e -- -t "@gpu and not @expected-failure"

      - name: Upload E2E report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: e2e-gpu-strix-ubuntu-report
          path: tests/e2e-cucumber/results/

  e2e-gpu-strix-ubuntu-known-bugs:
    name: E2E tests (Strix Halo, Ubuntu, known bugs)
    timeout-minutes: 15
    runs-on: [self-hosted, linux, strix-halo]
    needs: [changes, build-and-test]
    # See `e2e`: dispatch tolerates skipped build-and-test; strix-ubuntu, known-bugs.
    if: >-
      always()
      && needs.changes.result == 'success'
      && (
        (github.event_name != 'workflow_dispatch'
          && needs.build-and-test.result == 'success'
          && needs.changes.outputs.heavy == 'true')
        || (github.event_name == 'workflow_dispatch'
          && (inputs.platform == 'all' || inputs.platform == 'strix-ubuntu')
          && (inputs.tier == 'both' || inputs.tier == 'known-bugs'))
      )
    continue-on-error: true
    # See e2e-gpu-strix-ubuntu: /, /home/ubuntu, and /tmp are all on a full root
    # partition; only the nvme has space. Point HOME + all caches at the nvme so
    # nothing writes to root (pip's cache under the real $HOME is what failed
    # `install sdk` with ENOSPC).
    env:
      HOME: /home/ubuntu/actions-runner/e2e-home
      CARGO_HOME: /home/ubuntu/actions-runner/.cargo
      RUSTUP_HOME: /home/ubuntu/actions-runner/.rustup
      TMPDIR: /home/ubuntu/actions-runner/tmp
      PIP_CACHE_DIR: /home/ubuntu/actions-runner/pip-cache
      # Known-bug tier: fail fast rather than wait the default 600s on a serve
      # that a bug prevents from becoming ready.
      E2E_SERVE_TIMEOUT_SECS: "90"
    steps:
      - uses: actions/checkout@v6

      - name: Prepare writable dirs on the nvme
        run: mkdir -p /home/ubuntu/actions-runner/e2e-home /home/ubuntu/actions-runner/tmp /home/ubuntu/actions-runner/pip-cache

      # See e2e-gpu-strix-ubuntu: bootstrap rustup with --no-modify-path so it
      # never writes $HOME/.profile on the full root disk.
      - name: Ensure Rust toolchain
        run: |
          if ! command -v cargo >/dev/null 2>&1 && [ ! -x "$CARGO_HOME/bin/cargo" ]; then
            curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \
              | sh -s -- -y --no-modify-path --default-toolchain none
          fi
          echo "$CARGO_HOME/bin" >> "$GITHUB_PATH"

      - name: Run known-bug E2E scenarios on Strix Halo
        run: |
          export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target"
          export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared"
          cargo xtask e2e --expect-failures -- -t "@gpu and @expected-failure"

      - name: Upload E2E report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: e2e-gpu-strix-ubuntu-known-bugs-report
          path: tests/e2e-cucumber/results/

  # First real Windows GPU coverage: the Strix Halo Windows 11 runner. The
  # existing windows-build-and-test uses GitHub-hosted windows-latest, which has
  # no GPU, so `@gpu` scenarios have never run on Windows hardware. Expect
  # first-run breakage to triage; non-blocking so it never gates the PR.
  e2e-gpu-strix-windows:
    name: E2E tests (Strix Halo, Windows)
    timeout-minutes: 15
    runs-on: [self-hosted, windows, strix-halo]
    needs: [changes, build-and-test]
    # See `e2e`: dispatch tolerates skipped build-and-test; strix-windows, expect-pass.
    if: >-
      always()
      && needs.changes.result == 'success'
      && (
        (github.event_name != 'workflow_dispatch'
          && needs.build-and-test.result == 'success'
          && needs.changes.outputs.heavy == 'true')
        || (github.event_name == 'workflow_dispatch'
          && (inputs.platform == 'all' || inputs.platform == 'strix-windows')
          && (inputs.tier == 'both' || inputs.tier == 'expect-pass'))
      )
    continue-on-error: true
    steps:
      - uses: actions/checkout@v6

      # setup-rust-toolchain runs an internal bash script, which this Windows
      # runner lacks (bash: command not found). Bootstrap rustup with the
      # PowerShell-native installer instead; idempotent, so it only downloads on
      # a runner that doesn't already have the toolchain. Use `powershell`
      # (Windows PowerShell 5.1, always present) rather than `pwsh` (PowerShell 7),
      # which this self-hosted runner does not have installed.
      - name: Ensure Rust toolchain (PowerShell)
        shell: powershell
        run: |
          if (-not (Get-Command cargo -ErrorAction SilentlyContinue)) {
            Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile $env:TEMP\rustup-init.exe
            # --default-toolchain none: rust-toolchain.toml pins the exact
            # version (1.96.0 + components), auto-installed on first cargo use.
            & $env:TEMP\rustup-init.exe -y --default-toolchain none
            "$env:USERPROFILE\.cargo\bin" | Out-File -FilePath $env:GITHUB_PATH -Append
          }

      - name: Run E2E tests on Strix Halo Windows (excluding known bugs)
        shell: powershell
        run: cargo xtask e2e -- -t "@gpu and not @expected-failure"

      - name: Upload E2E report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: e2e-gpu-strix-windows-report
          path: tests/e2e-cucumber/results/

  e2e-gpu-strix-windows-known-bugs:
    name: E2E tests (Strix Halo, Windows, known bugs)
    timeout-minutes: 15
    runs-on: [self-hosted, windows, strix-halo]
    needs: [changes, build-and-test]
    # See `e2e`: dispatch tolerates skipped build-and-test; strix-windows, known-bugs.
    if: >-
      always()
      && needs.changes.result == 'success'
      && (
        (github.event_name != 'workflow_dispatch'
          && needs.build-and-test.result == 'success'
          && needs.changes.outputs.heavy == 'true')
        || (github.event_name == 'workflow_dispatch'
          && (inputs.platform == 'all' || inputs.platform == 'strix-windows')
          && (inputs.tier == 'both' || inputs.tier == 'known-bugs'))
      )
    continue-on-error: true
    # Known-bug tier: fail fast rather than wait the default 600s on a serve that
    # a bug prevents from becoming ready.
    env:
      E2E_SERVE_TIMEOUT_SECS: "90"
    steps:
      - uses: actions/checkout@v6

      # See e2e-gpu-strix-windows: bootstrap rustup via Windows PowerShell 5.1
      # (`powershell`, not `pwsh`) because this runner has neither bash nor
      # PowerShell 7 for setup-rust-toolchain's internal script.
      - name: Ensure Rust toolchain (PowerShell)
        shell: powershell
        run: |
          if (-not (Get-Command cargo -ErrorAction SilentlyContinue)) {
            Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile $env:TEMP\rustup-init.exe
            # --default-toolchain none: rust-toolchain.toml pins the exact
            # version (1.96.0 + components), auto-installed on first cargo use.
            & $env:TEMP\rustup-init.exe -y --default-toolchain none
            "$env:USERPROFILE\.cargo\bin" | Out-File -FilePath $env:GITHUB_PATH -Append
          }

      - name: Run known-bug E2E scenarios on Strix Halo Windows
        shell: powershell
        run: cargo xtask e2e --expect-failures -- -t "@gpu and @expected-failure"

      - name: Upload E2E report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: e2e-gpu-strix-windows-known-bugs-report
          path: tests/e2e-cucumber/results/

  # Consolidate every platform/tier's report into ONE cross-platform report:
  # a platform × tier matrix in the run Summary plus a single merged HTML
  # artifact. `if: always()` so a failing/non-blocking platform still appears;
  # runs on GitHub-hosted ubuntu (no GPU needed — it only parses report.json).
  #
  # Extensibility: the report CONTENT auto-includes any new platform via the
  # `*-report` artifact glob below — no change needed here. The one manual step
  # when adding a platform is appending its two job names to `needs:` (for run
  # ORDERING only; GitHub Actions has no wildcard `needs`). A future matrix
  # refactor of the 8 e2e jobs would remove even that.
  e2e-report:
    name: E2E consolidated report
    runs-on: ubuntu-latest
    needs:
      - changes
      - e2e
      - e2e-known-bugs
      - e2e-gpu
      - e2e-gpu-known-bugs
      - e2e-gpu-strix-ubuntu
      - e2e-gpu-strix-ubuntu-known-bugs
      - e2e-gpu-strix-windows
      - e2e-gpu-strix-windows-known-bugs
    # Consolidate whatever ran. On dispatch `heavy` is unset, so also run when the
    # trigger was manual; `always()` still lets it collect partial/failed tiers.
    if: >-
      always()
      && (needs.changes.outputs.heavy == 'true'
        || github.event_name == 'workflow_dispatch')
    steps:
      - uses: actions/checkout@v6

      - uses: actions-rust-lang/setup-rust-toolchain@v1

      # Pull every e2e artifact. Each extracts to e2e-artifacts/<artifact-name>/,
      # which `xtask e2e-report` turns into one labeled platform. The `*-report`
      # glob is what makes new platforms appear automatically.
      - name: Download all E2E reports
        uses: actions/download-artifact@v4
        with:
          pattern: '*-report'
          path: e2e-artifacts

      - name: Build consolidated report + step summary
        run: |
          mkdir -p consolidated
          cargo xtask e2e-report \
            --artifacts-dir e2e-artifacts \
            --html-out consolidated/index.html >> "$GITHUB_STEP_SUMMARY"

      - name: Upload consolidated report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: e2e-consolidated-report
          path: consolidated/