CI #530

Workflow file for this run

	name: CI

	on:
	push:
	branches: [main]
	pull_request:
	branches: [main]
	# The merge queue creates a temporary branch and fires `merge_group`; without
	# this trigger the required checks never start and queued PRs stall forever.
	merge_group:
	branches: [main]
	types: [checks_requested]
	# Manual trigger for on-demand E2E runs on the self-hosted GPU runners, without
	# waiting on a push/PR or the full gate. Dispatch against any ref
	# (`gh workflow run ci.yml --ref <branch> -f platform=... -f tier=...`) — GitHub
	# runs THAT ref's copy of this file, so the E2E jobs it selects live on the ref.
	# A dispatch skips build-and-test (see its `if:`) for a fast loop. This trigger
	# must exist on the default branch for the workflow to be dispatchable at all.
	workflow_dispatch:
	inputs:
	platform:
	description: Which runner(s) to target
	type: choice
	default: all
	options: [all, mock, app-dev-gpu, strix-ubuntu, strix-windows]
	tier:
	description: Which scenario tier(s) to run
	type: choice
	default: both
	options: [both, expect-pass, known-bugs]

	concurrency:
	# Manual dispatches get a UNIQUE group (run_id) so a run that gets stuck — e.g.
	# a job queued on a temporarily offline self-hosted runner, which GitHub cannot
	# cancel — never holds the shared group and blocks later dispatches. push / PR /
	# merge_group keep the shared per-ref group so a new commit still supersedes the
	# previous in-flight run.
	group: >-
	${{ github.workflow }}-${{ github.ref }}-${{
	github.event_name == 'workflow_dispatch' && github.run_id \|\| 'shared' }}
	cancel-in-progress: true

	permissions:
	contents: read

	env:
	CARGO_TERM_COLOR: always

	jobs:
	# Decide which file categories a change touches so unaffected jobs can skip
	# their expensive work. Filtering runs ONLY on pull_request; on push and in the
	# merge queue every category is forced true, so post-merge `main` and queued
	# PRs always run the full suite. That keeps the skip from ever starving a
	# required check in the merge queue (a never-produced required check stalls the
	# queue forever — see the `commit-signatures` note below). Each output folds in
	# `.github/workflows/**`, so any change to CI itself triggers a full run.
	changes:
	runs-on: ubuntu-latest
	# paths-filter reads the PR's changed-file list from the API on
	# pull_request events; that needs pull-requests:read on top of the
	# workflow-wide contents:read (job-level permissions replace the default
	# set, so contents:read is repeated here for checkout).
	permissions:
	contents: read
	pull-requests: read
	outputs:
	# `<filter> \|\| <fallback>`: the filter result on PRs (empty string when the
	# filter step is skipped off-PR), otherwise the forced 'true'.
	rust: ${{ steps.filter.outputs.rust \|\| steps.all.outputs.forced }}
	heavy: ${{ steps.filter.outputs.heavy \|\| steps.all.outputs.forced }}
	lint: ${{ steps.filter.outputs.lint \|\| steps.all.outputs.forced }}
	tpn: ${{ steps.filter.outputs.tpn \|\| steps.all.outputs.forced }}
	steps:
	- uses: actions/checkout@v6

	- name: Filter changed paths (pull requests only)
	id: filter
	if: github.event_name == 'pull_request'
	uses: dorny/paths-filter@d1c1ffe0248fe513906c8e24db8ea791d46f8590 # v3.0.2
	with:
	filters: \|
	# Rust-only checks (fmt, clippy, coverage).
	rust:
	- '*/.rs'
	- '**/Cargo.toml'
	- 'Cargo.lock'
	- 'rust-toolchain*'
	# clippy runs `cargo xtask manifest --check` against this file.
	- 'MANIFEST.md'
	- '.github/workflows/**'
	# build-and-test runs cargo AND the python/shell smoke + acceptance
	# steps, so it depends on the Rust set plus those scripts.
	heavy:
	- '*/.rs'
	- '**/Cargo.toml'
	- 'Cargo.lock'
	- 'rust-toolchain*'
	- 'scripts/**'
	- 'engines/**'
	- '*/.py'
	- '*/.sh'
	- '*/.ps1'
	- '*/.feature'
	- 'install*'
	- '.github/workflows/**'
	# THIRD_PARTY_NOTICES.txt staleness gate: anything that changes the
	# dependency tree, the cargo-about config/template, the generator, or
	# the generated file itself.
	tpn:
	- 'Cargo.lock'
	- '**/Cargo.toml'
	- 'about.toml'
	- 'about.hbs'
	- 'THIRD_PARTY_NOTICES.txt'
	- 'xtask/**'
	- '.github/workflows/**'
	# Python / Shell / PowerShell lint job.
	lint:
	- 'scripts/**'
	- 'engines/**'
	- '*/.py'
	- '*/.sh'
	- '*/.ps1'
	- 'ruff.toml'
	- 'PSScriptAnalyzerSettings.psd1'
	- '.github/workflows/**'

	- name: Force full run off pull requests
	id: all
	if: github.event_name != 'pull_request'
	run: echo "forced=true" >> "$GITHUB_OUTPUT"

	build-and-test:
	runs-on: ubuntu-latest
	# Don't spend per-OS build/test cycles unless the cheap lint gate is green.
	needs: [changes, clippy, prek]
	# A manual E2E dispatch skips this heavy job for a fast loop; the E2E jobs
	# tolerate a skipped build-and-test in their own `if:` guards.
	if: github.event_name != 'workflow_dispatch'
	steps:
	- uses: actions/checkout@v6

	- name: Install native build deps
	if: needs.changes.outputs.heavy == 'true'
	run: sudo apt-get update && sudo apt-get install -y pkg-config libcap-dev

	- uses: actions-rust-lang/setup-rust-toolchain@v1
	if: needs.changes.outputs.heavy == 'true'

	# Build the runtime binaries the smoke + acceptance steps below need (every
	# engine binary; smoke hard-fails on a missing one). Unit/integration tests
	# run in the dedicated `test` job on only the crates a change can affect;
	# `clippy --all-targets` is the compile-check for the non-binary targets, so
	# dropping `--all-targets` here loses no coverage.
	- name: Build
	if: needs.changes.outputs.heavy == 'true'
	run: cargo build --workspace

	- name: Local no-fallback smoke
	if: needs.changes.outputs.heavy == 'true'
	run: python scripts/smoke_local.py --skip-build

	- name: Acceptance harness self-tests
	if: needs.changes.outputs.heavy == 'true'
	run: \|
	python scripts/comfyui_therock_gpu_test.py --self-test
	python scripts/local_assistant_therock_gpu_test.py --self-test
	python scripts/vllm_therock_gpu_test.py --self-test
	python scripts/wsl_preflight.py --self-test

	- name: Portable WSL build deps self-test
	if: needs.changes.outputs.heavy == 'true'
	run: bash scripts/setup-wsl-portable-build-deps.sh --self-test

	- name: Release readiness self-test
	if: needs.changes.outputs.heavy == 'true'
	run: python scripts/release_readiness.py --self-test

	- name: Acceptance install lifecycle
	if: needs.changes.outputs.heavy == 'true'
	run: ./scripts/acceptance-install-upgrade-tui-uninstall.sh

	# Fine-grained test selection (the second half of smarter CI; the path-based
	# job skip is the `changes` job above). Runs cargo's unit + integration tests
	# for only the crates a change can reach — the changed crates plus their
	# transitive dependents, computed by `cargo xtask affected` from the workspace
	# dependency graph — instead of the whole workspace.
	#
	# Merge-queue safety: the narrowing happens ONLY on pull_request. On push and
	# in the merge queue the selection is forced to `--workspace`, so post-merge
	# `main` and every queued PR run the full suite — a required check is never
	# starved and the queue cannot stall. The full Windows suite
	# (windows-build-and-test) also runs the whole workspace on every change, so it
	# backstops any crate a graph-based selection could miss (e.g. an integration
	# test that drives another crate only at runtime).
	test:
	name: Test (affected crates)
	runs-on: ubuntu-latest
	# Same lint gate as the OS build jobs: don't spend a test cycle until the
	# cheap checks are green.
	needs: [changes, clippy, prek]
	# A manual E2E dispatch only exercises the E2E jobs; skip the rest.
	if: github.event_name != 'workflow_dispatch'
	steps:
	- uses: actions/checkout@v6
	with:
	# `cargo xtask affected` diffs origin/<base>...HEAD, so it needs the
	# base history and the real PR head (not the synthetic merge ref).
	fetch-depth: 0
	ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha \|\| '' }}

	- name: Install native build deps
	if: needs.changes.outputs.rust == 'true'
	run: sudo apt-get update && sudo apt-get install -y pkg-config libcap-dev

	- uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0
	if: needs.changes.outputs.rust == 'true'
	with:
	# Keep the PR (subset) and push/merge (`--workspace`) builds in separate
	# cache lanes. A subset build resolves Cargo features differently than
	# the whole workspace, so a shared cache would make the two evict and
	# recompile each other's artifacts on every alternation.
	cache-key: ${{ github.event_name == 'pull_request' && 'affected-pr' \|\| 'full' }}

	- name: Install cargo-nextest
	if: needs.changes.outputs.rust == 'true'
	uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6
	with:
	tool: nextest

	- name: Select affected crates
	id: sel
	if: needs.changes.outputs.rust == 'true'
	run: \|
	# The `cargo xtask` alias builds xtask in release (target/release), so
	# computing the selection doesn't perturb the debug feature resolution
	# the nextest build below uses.
	if [ "${{ github.event_name }}" = "pull_request" ]; then
	sel=$(cargo xtask affected --base "origin/${{ github.base_ref }}")
	else
	sel="--workspace"
	fi
	echo "affected selection: ${sel:-<no rust crates changed>}"
	echo "sel=$sel" >> "$GITHUB_OUTPUT"

	- name: Test (nextest)
	# Skip cleanly when nothing Rust-relevant resolved to a crate, so the
	# required check still reports success.
	if: needs.changes.outputs.rust == 'true' && steps.sel.outputs.sel != ''
	# `sel` is intentionally unquoted: it expands to either `--workspace` or
	# several `-p <crate>` flags that must word-split into separate argv
	# entries. The values are crate names from our own tool, not user input.
	run: cargo nextest run ${{ steps.sel.outputs.sel }}

	windows-build-and-test:
	runs-on: windows-latest
	env:
	CARGO_INCREMENTAL: "0"
	# Don't spend per-OS build/test cycles unless the cheap lint gate is green.
	needs: [changes, clippy, prek]
	# A manual E2E dispatch only exercises the E2E jobs; skip the rest.
	if: github.event_name != 'workflow_dispatch'
	steps:
	- uses: actions/checkout@v6

	- uses: actions-rust-lang/setup-rust-toolchain@v1
	if: needs.changes.outputs.heavy == 'true'
	with:
	cache-on-failure: true

	- name: Build
	if: needs.changes.outputs.heavy == 'true'
	shell: pwsh
	run: \|
	cargo build --workspace --all-targets
	cargo build --release -p rocm -p rocmd -p rocm-engine-lemonade -p rocm-engine-vllm -p xtask

	- name: Test
	if: needs.changes.outputs.heavy == 'true'
	shell: pwsh
	run: cargo test --workspace --all-targets

	- name: Local no-fallback smoke
	if: needs.changes.outputs.heavy == 'true'
	shell: pwsh
	run: python .\scripts\smoke_local.py --skip-build

	- name: Acceptance harness self-tests
	if: needs.changes.outputs.heavy == 'true'
	shell: pwsh
	run: \|
	python .\scripts\comfyui_therock_gpu_test.py --self-test
	python .\scripts\local_assistant_therock_gpu_test.py --self-test
	python .\scripts\vllm_therock_gpu_test.py --self-test
	python .\scripts\wsl_preflight.py --self-test

	- name: Release readiness self-test
	if: needs.changes.outputs.heavy == 'true'
	shell: pwsh
	run: python .\scripts\release_readiness.py --self-test

	- name: Check PowerShell installer syntax
	if: needs.changes.outputs.heavy == 'true'
	shell: pwsh
	run: \|
	$null = [scriptblock]::Create((Get-Content .\install.ps1 -Raw))
	$null = [scriptblock]::Create((Get-Content .\scripts\package-windows-release.ps1 -Raw))

	- name: Acceptance install lifecycle
	if: needs.changes.outputs.heavy == 'true'
	shell: pwsh
	run: .\scripts\acceptance-install-upgrade-tui-uninstall.ps1

	prek:
	# The fast lint gate: every toolchain-light lint/hygiene check, run from the
	# single source of truth in .pre-commit-config.yaml — hygiene + Python (ruff)
	# + Shell (shellcheck) + cargo fmt (rustfmt only, no compile).
	# `--no-group local-tools` skips the hooks that need a full Rust compile,
	# hawkeye, or PowerShell — those have their own jobs (clippy, build-and-test,
	# license-headers, powershell-lint).
	name: prek (lint / hygiene)
	runs-on: ubuntu-latest
	# Always-on: prek is the cheapest gate (no compile) and its hygiene hooks run
	# on every file type, so it is relevant to any change — including docs-only
	# PRs that match no `changes` category. Like license-headers and
	# commit-signatures, it is not gated.
	# Exception: a manual E2E dispatch only exercises the E2E jobs; skip the rest.
	if: github.event_name != 'workflow_dispatch'
	steps:
	- uses: actions/checkout@v6

	# cargo-fmt hook needs rustfmt (no build).
	- uses: actions-rust-lang/setup-rust-toolchain@v1
	with:
	components: rustfmt
	cache: false

	# Supply-chain hardening: the action is pinned to a commit SHA and prek to
	# an exact version. The action ships a hardcoded SHA-256 checksum map
	# (src/known-checksums.ts) and verifies the downloaded prek archive against
	# it, so a tampered release can't be installed. This commit is past the
	# v2.0.4 tag because it is the one that added prek 0.4.5 to that map — pin
	# an earlier commit and the action would not know 0.4.5 and skip the check.
	- uses: j178/prek-action@dec4adcd2fa1db9d50a39668b22112a37235e730 # prek 0.4.5 checksums
	with:
	prek-version: 0.4.5
	extra-args: --all-files --no-group local-tools

	clippy:
	runs-on: ubuntu-latest
	needs: changes
	# A manual E2E dispatch only exercises the E2E jobs; skip the rest.
	if: github.event_name != 'workflow_dispatch'
	steps:
	- uses: actions/checkout@v6

	- uses: actions-rust-lang/setup-rust-toolchain@v1
	if: needs.changes.outputs.rust == 'true'

	- name: Clippy
	if: needs.changes.outputs.rust == 'true'
	run: cargo clippy --locked --workspace --all-targets -- -D warnings

	- name: MANIFEST.md dependency table is current
	if: needs.changes.outputs.rust == 'true'
	run: cargo xtask manifest --check

	third-party-notices:
	name: Third-party notices current
	runs-on: ubuntu-latest
	needs: changes
	# A manual E2E dispatch only exercises the E2E jobs; skip the rest.
	if: github.event_name != 'workflow_dispatch'
	steps:
	- uses: actions/checkout@v6

	- uses: actions-rust-lang/setup-rust-toolchain@v1
	if: needs.changes.outputs.tpn == 'true'

	# Pin the cargo-about version so the check compares against the exact same
	# generator contributors use; a different version can format the notices
	# differently and fail the byte-for-byte comparison. This version has no
	# prebuilt release binary and its `cargo install` needs the `cli` feature,
	# so build from source (cached by version); `--locked` uses cargo-about's
	# own lockfile for a reproducible generator.
	- name: Cache cargo-about
	if: needs.changes.outputs.tpn == 'true'
	id: cache-cargo-about
	uses: actions/cache@v5
	with:
	path: ~/.cargo/bin/cargo-about
	key: ${{ runner.os }}-cargo-about-0.9.1

	- name: Install cargo-about
	if: needs.changes.outputs.tpn == 'true' && steps.cache-cargo-about.outputs.cache-hit != 'true'
	run: cargo install cargo-about@0.9.1 --locked --features cli

	- name: THIRD_PARTY_NOTICES.txt is current
	if: needs.changes.outputs.tpn == 'true'
	run: cargo xtask tpn --check

	coverage:
	name: Coverage (rocm-dash crates, ratcheted)
	runs-on: ubuntu-latest
	needs: changes
	# A manual E2E dispatch only exercises the E2E jobs; skip the rest.
	if: github.event_name != 'workflow_dispatch'
	steps:
	- uses: actions/checkout@v6

	- name: Install native build deps
	if: needs.changes.outputs.rust == 'true'
	run: sudo apt-get update && sudo apt-get install -y pkg-config libcap-dev

	- uses: dtolnay/rust-toolchain@1.96.0
	if: needs.changes.outputs.rust == 'true'
	with:
	components: llvm-tools-preview

	- name: Install cargo-llvm-cov
	if: needs.changes.outputs.rust == 'true'
	uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6
	with:
	tool: cargo-llvm-cov

	- name: Cache cargo registry and build
	if: needs.changes.outputs.rust == 'true'
	uses: actions/cache@v5
	with:
	path: \|
	~/.cargo/registry
	~/.cargo/git
	target
	key: ${{ runner.os }}-cargo-cov-${{ hashFiles('**/Cargo.lock') }}
	restore-keys: \|
	${{ runner.os }}-cargo-cov-

	# Ratcheted, fail-on-regression floor scoped to the transplanted rocm-dash
	# crates (the clean anchor) — deliberately NOT the whole workspace, so the
	# gate does not over-claim coverage of the large untyped rocm-cli core
	# Baseline measured 2026-06-11: 73.7% lines across the four
	# crates. Floor set just below measured; ratchet upward as the larger TUI
	# tab files (bench/overview/modal) gain tests.
	- name: Coverage gate (rocm-dash crates, >= 70% lines)
	if: needs.changes.outputs.rust == 'true'
	run: \|
	cargo llvm-cov --no-cfg-coverage \
	-p rocm-dash-core -p rocm-dash-collectors \
	-p rocm-dash-daemon -p rocm-dash-tui \
	--fail-under-lines 70

	powershell-lint:
	# Runs the prek `powershell-script-analyzer` hook's logic (cargo xtask
	# powershell-lint) under BOTH PowerShell editions. windows-latest is the only
	# runner with Windows PowerShell 5.1 (`powershell`); it also ships PowerShell
	# 7 (`pwsh`). Kept separate from windows-build-and-test so it runs in parallel
	# rather than behind that slow job.
	name: Lint (PowerShell)
	runs-on: windows-latest
	needs: changes
	# A manual E2E dispatch only exercises the E2E jobs; skip the rest.
	if: github.event_name != 'workflow_dispatch'
	steps:
	- uses: actions/checkout@v6

	- uses: actions-rust-lang/setup-rust-toolchain@v1
	if: needs.changes.outputs.lint == 'true'

	- name: Install PSScriptAnalyzer (PowerShell 7)
	if: needs.changes.outputs.lint == 'true'
	shell: pwsh
	run: Install-Module PSScriptAnalyzer -Scope CurrentUser -Force -RequiredVersion 1.25.0

	- name: Install PSScriptAnalyzer (Windows PowerShell 5.1)
	if: needs.changes.outputs.lint == 'true'
	shell: powershell
	run: \|
	[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
	Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force \| Out-Null
	Install-Module PSScriptAnalyzer -Scope CurrentUser -Force -RequiredVersion 1.25.0

	- name: PSScriptAnalyzer (PowerShell 7)
	if: needs.changes.outputs.lint == 'true'
	run: cargo xtask powershell-lint --shell pwsh

	- name: PSScriptAnalyzer (Windows PowerShell 5.1)
	if: needs.changes.outputs.lint == 'true'
	run: cargo xtask powershell-lint --shell powershell

	license-headers:
	name: License header check (hawkeye)
	runs-on: ubuntu-latest
	env:
	# Install a pinned prebuilt hawkeye binary instead of pulling the
	# korandoru/hawkeye Docker image on every run, and verify it against a
	# sha256 recorded here. The hash lives in-repo, out of band from the
	# release: GitHub release assets are mutable, so a republished/tampered
	# artifact (and its co-published .sha256) would pass an upstream-only
	# check — pinning the hash here makes that fail instead. To upgrade, bump
	# the version and replace the hash with the new release's published
	# .sha256 (confirm the bytes first).
	HAWKEYE_VERSION: v6.5.1
	HAWKEYE_SHA256: d6eb0505a45a15244f4f789158aafe5e3f1a7dc86c9dc1d7651f3cb1e1b321e0
	HAWKEYE_INSTALL_DIR: ${{ github.workspace }}/.hawkeye-bin
	# A manual E2E dispatch only exercises the E2E jobs; skip the rest.
	if: github.event_name != 'workflow_dispatch'
	steps:
	- uses: actions/checkout@v6

	- name: Cache hawkeye binary
	id: cache-hawkeye
	uses: actions/cache@v5
	with:
	path: ${{ env.HAWKEYE_INSTALL_DIR }}
	key: ${{ runner.os }}-hawkeye-${{ env.HAWKEYE_VERSION }}-${{ env.HAWKEYE_SHA256 }}

	- name: Install hawkeye
	if: steps.cache-hawkeye.outputs.cache-hit != 'true'
	run: \|
	tarball="hawkeye-x86_64-unknown-linux-gnu.tar.xz"
	curl --proto '=https' --tlsv1.2 -LsSf \
	"https://github.com/korandoru/hawkeye/releases/download/${HAWKEYE_VERSION}/${tarball}" \
	-o "${tarball}"
	echo "${HAWKEYE_SHA256} ${tarball}" \| sha256sum -c -
	mkdir -p "${HAWKEYE_INSTALL_DIR}"
	tar -xJf "${tarball}" -C "${HAWKEYE_INSTALL_DIR}" --strip-components=1
	rm -f "${tarball}"

	- name: Check license headers
	run: \|
	"${HAWKEYE_INSTALL_DIR}/hawkeye" check

	commit-signatures:
	name: Commit signatures + sign-off
	runs-on: ubuntu-latest
	# Runs on pull requests and in the merge queue. Both define a base that
	# bounds the commit range. Handling `merge_group` matters once this is a
	# required check: the queue fires `merge_group` (not `pull_request`), so a
	# job gated to PRs only would never produce the required check and the queue
	# entry would stall forever.
	if: github.event_name == 'pull_request' \|\| github.event_name == 'merge_group'
	steps:
	- uses: actions/checkout@v6
	with:
	fetch-depth: 0
	# PR: check out the PR head, not the synthetic refs/pull/N/merge commit
	# (which is unsigned and lacks a sign-off, and would otherwise appear
	# in origin/<base>..HEAD and fail the gate on every PR).
	# merge_group: check out the queue head — the rebased/squashed commits
	# GitHub built for the group. Merge commits are disabled on this repo,
	# so these are normal commits that preserve their Signed-off-by trailer
	# and are GitHub "Verified" (web-flow signed).
	ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha \|\| github.event.merge_group.head_sha }}

	- uses: actions-rust-lang/setup-rust-toolchain@v1

	- name: Verify commits are signed and signed-off
	# `--require-verified` shells out to the `gh` CLI; pass the job token so
	# `gh api` is authenticated (avoids the unauthenticated rate limit).
	# Base: the PR's base branch, or — in the queue — the exact commit the
	# merge group was built on (`base_sha`), so the range is just the queued
	# commits and not whatever else has since landed on the base branch.
	env:
	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	BASE: ${{ github.event_name == 'pull_request' && format('origin/{0}', github.base_ref) \|\| github.event.merge_group.base_sha }}
	run: cargo xtask verify-commits --base "$BASE" --require-verified

	# Hardware (real AMD GPU / WSL) smoke tests on dedicated self-hosted runners
	# are intentionally not part of this workflow yet. See
	# docs/ci-hardware-testing.md for the planned design; they will land in a
	# follow-up PR.

	# ── E2E tests (cucumber-rs) ───────────────────────────────────────────────
	#
	# BDD scenarios in Gherkin (.feature files) backed by Rust step functions.
	# Three selections: blocking mock tier, non-blocking known bugs, non-blocking
	# GPU. Cucumber tag filters are exact-match, so known-bug scenarios carry a
	# bare `@expected-failure` (plus `@expected-failure-EAI-NNNN` for traceability)
	# and hardware-dependent scenarios carry `@gpu`.

	# Blocking mock-tier job: must stay green. Known-bug scenarios and
	# hardware-dependent scenarios are excluded here and run in the non-blocking
	# jobs below.
	e2e:
	name: E2E tests
	timeout-minutes: 15
	runs-on: ubuntu-latest
	needs: [changes, build-and-test]
	# On push/PR/merge_group: unchanged — run when build-and-test succeeded and
	# the change is heavy. On workflow_dispatch: build-and-test is skipped, so
	# tolerate that and gate on the platform/tier inputs instead (mock tier here).
	if: >-
	always()
	&& needs.changes.result == 'success'
	&& (
	(github.event_name != 'workflow_dispatch'
	&& needs.build-and-test.result == 'success'
	&& needs.changes.outputs.heavy == 'true')
	\|\| (github.event_name == 'workflow_dispatch'
	&& (inputs.platform == 'all' \|\| inputs.platform == 'mock')
	&& (inputs.tier == 'both' \|\| inputs.tier == 'expect-pass'))
	)
	steps:
	- uses: actions/checkout@v6

	- name: Install native build deps
	run: sudo apt-get update && sudo apt-get install -y pkg-config libcap-dev

	- uses: actions-rust-lang/setup-rust-toolchain@v1

	# The workspace test job selects only affected crates and runs the cucumber
	# harness, not the library's own unit tests — run the report generator's
	# unit tests (stats counting, scenario status, date formatting) here.
	- name: Unit tests (e2e-cucumber lib)
	run: cargo test -p e2e-cucumber --lib

	- name: Run E2E tests (mock tier, excluding known bugs)
	run: cargo xtask e2e -- -t "not @gpu and not @expected-failure"

	- name: Upload E2E report
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: e2e-report
	path: tests/e2e-cucumber/results/

	# Runs ONLY the known-bug scenarios under xfail inversion (`--expect-failures`):
	# a tagged scenario failing is the expected, green outcome. The job goes red
	# only when a known bug is silently fixed (XPASS — remove its @expected-failure
	# tag so it joins the blocking e2e job above), an untagged scenario fails, or a
	# parse/hook error occurs — all actionable, so this gates rather than being an
	# always-red check reviewers learn to ignore.
	e2e-known-bugs:
	name: E2E tests (known bugs)
	timeout-minutes: 15
	runs-on: ubuntu-latest
	needs: [changes, build-and-test]
	# See `e2e`: dispatch tolerates skipped build-and-test; mock tier, known-bugs.
	if: >-
	always()
	&& needs.changes.result == 'success'
	&& (
	(github.event_name != 'workflow_dispatch'
	&& needs.build-and-test.result == 'success'
	&& needs.changes.outputs.heavy == 'true')
	\|\| (github.event_name == 'workflow_dispatch'
	&& (inputs.platform == 'all' \|\| inputs.platform == 'mock')
	&& (inputs.tier == 'both' \|\| inputs.tier == 'known-bugs'))
	)
	steps:
	- uses: actions/checkout@v6

	- name: Install native build deps
	run: sudo apt-get update && sudo apt-get install -y pkg-config libcap-dev

	- uses: actions-rust-lang/setup-rust-toolchain@v1

	- name: Run known-bug E2E scenarios (mock tier)
	run: cargo xtask e2e --expect-failures -- -t "@expected-failure and not @gpu"

	- name: Upload E2E report
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: e2e-known-bugs-report
	path: tests/e2e-cucumber/results/

	# GPU hardware scenarios that are expected to PASS. Mirrors the mock `e2e` job
	# but for `@gpu` hardware-dependent scenarios; known-bug `@gpu` scenarios are
	# split into e2e-gpu-known-bugs below so a red here always means a real GPU
	# regression, never a known bug still reproducing.
	e2e-gpu:
	name: E2E tests (GPU)
	timeout-minutes: 15
	runs-on: [self-hosted, linux, amd-gpu]
	needs: [changes, build-and-test]
	# See `e2e`: dispatch tolerates skipped build-and-test; app-dev-gpu, expect-pass.
	if: >-
	always()
	&& needs.changes.result == 'success'
	&& (
	(github.event_name != 'workflow_dispatch'
	&& needs.build-and-test.result == 'success'
	&& needs.changes.outputs.heavy == 'true')
	\|\| (github.event_name == 'workflow_dispatch'
	&& (inputs.platform == 'all' \|\| inputs.platform == 'app-dev-gpu')
	&& (inputs.tier == 'both' \|\| inputs.tier == 'expect-pass'))
	)
	continue-on-error: true
	steps:
	- uses: actions/checkout@v6

	# cache: false — on this self-hosted runner we persist the build cache
	# ourselves via CARGO_TARGET_DIR (below). The action's built-in
	# Swatinem/rust-cache otherwise tries to SAVE the large target dir to
	# GitHub's cache service in a post-step (slow/hangs) and its cleanup wipes
	# the local target.
	- uses: actions-rust-lang/setup-rust-toolchain@v1
	with:
	cache: false

	# `actions/checkout` runs `git clean -ffdx`, which deletes the gitignored
	# `target/` inside the repo every job → a full ~15min rebuild each run.
	# Point CARGO_TARGET_DIR at a sibling of the checkout ($RUNNER_WORKSPACE is
	# the checkout's parent — untouched by git clean and persistent between jobs
	# on a self-hosted runner), so cargo rebuilds incrementally: unchanged →
	# reused, changed → only the diff.
	- name: Run E2E tests on GPU hardware (excluding known bugs)
	run: \|
	export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target"
	# Share heavy immutable artifacts (TheRock runtimes ~3.3GB, HF weights,
	# vLLM venv) across scenarios so they download once per runner, not per
	# scenario. Persistent path; service state stays isolated per scenario.
	export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared"
	cargo xtask e2e -- -t "@gpu and not @expected-failure"

	- name: Upload E2E report
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: e2e-gpu-report
	path: tests/e2e-cucumber/results/

	# GPU counterpart of e2e-known-bugs: runs ONLY the `@gpu @expected-failure`
	# scenarios under xfail inversion (`--expect-failures`). Green when each known
	# GPU bug still fails as expected; red only when one is silently fixed (XPASS —
	# remove its @expected-failure tag so it joins e2e-gpu above), an untagged
	# scenario fails, or a parse/hook error occurs.
	e2e-gpu-known-bugs:
	name: E2E tests (GPU, known bugs)
	timeout-minutes: 15
	runs-on: [self-hosted, linux, amd-gpu]
	needs: [changes, build-and-test]
	# See `e2e`: dispatch tolerates skipped build-and-test; app-dev-gpu, known-bugs.
	if: >-
	always()
	&& needs.changes.result == 'success'
	&& (
	(github.event_name != 'workflow_dispatch'
	&& needs.build-and-test.result == 'success'
	&& needs.changes.outputs.heavy == 'true')
	\|\| (github.event_name == 'workflow_dispatch'
	&& (inputs.platform == 'all' \|\| inputs.platform == 'app-dev-gpu')
	&& (inputs.tier == 'both' \|\| inputs.tier == 'known-bugs'))
	)
	continue-on-error: true
	env:
	# Known-bug scenarios are expected to fail, so don't grant a serve the full
	# cold-start window — fail fast instead of waiting the default 600s when a
	# bug manifests as a serve that never becomes ready.
	E2E_SERVE_TIMEOUT_SECS: "90"
	steps:
	- uses: actions/checkout@v6

	# See e2e-gpu: cache: false because we persist the cache ourselves via
	# CARGO_TARGET_DIR; the action's rust-cache save/cleanup is slow and harmful
	# on this self-hosted runner.
	- uses: actions-rust-lang/setup-rust-toolchain@v1
	with:
	cache: false

	# See e2e-gpu: keep the build cache out of the checkout (git clean would
	# wipe it) for incremental rebuilds across jobs.
	- name: Run known-bug E2E scenarios on GPU hardware
	run: \|
	export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target"
	# See e2e-gpu: share heavy immutable artifacts across scenarios.
	export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared"
	cargo xtask e2e --expect-failures -- -t "@gpu and @expected-failure"

	- name: Upload E2E report
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: e2e-gpu-known-bugs-report
	path: tests/e2e-cucumber/results/

	# Second AMD GPU architecture: the Strix Halo (gfx1151) Ubuntu runner. Same
	# expect-pass / known-bugs split as app-dev-gpu's e2e-gpu jobs, targeted by the
	# `strix-halo` label so it lands only on that runner (app-dev-gpu carries
	# `amd-gpu`, not `strix-halo`). Non-blocking while this hardware is proven out.
	e2e-gpu-strix-ubuntu:
	name: E2E tests (Strix Halo, Ubuntu)
	timeout-minutes: 15
	runs-on: [self-hosted, linux, strix-halo]
	needs: [changes, build-and-test]
	# See `e2e`: dispatch tolerates skipped build-and-test; strix-ubuntu, expect-pass.
	if: >-
	always()
	&& needs.changes.result == 'success'
	&& (
	(github.event_name != 'workflow_dispatch'
	&& needs.build-and-test.result == 'success'
	&& needs.changes.outputs.heavy == 'true')
	\|\| (github.event_name == 'workflow_dispatch'
	&& (inputs.platform == 'all' \|\| inputs.platform == 'strix-ubuntu')
	&& (inputs.tier == 'both' \|\| inputs.tier == 'expect-pass'))
	)
	continue-on-error: true
	# On this runner `/`, `/home/ubuntu`, and `/tmp` are ALL on a full root
	# partition; only /home/ubuntu/actions-runner (a 1.7T nvme) has space. So
	# EVERYTHING the job writes must land on the nvme. Point HOME there (catches
	# ~/.cache/pip, ~/.config, and any other $HOME writer — pip's cache under the
	# real /home/ubuntu is what previously failed `install sdk` with ENOSPC),
	# plus the toolchain, temp, and pip cache. The rustup bootstrap still uses
	# --no-modify-path so it doesn't touch $HOME/.profile. (The HOME override
	# brings back a cosmetic rustup euid/HOME warning — accepted: not filling the
	# root disk matters more.)
	env:
	HOME: /home/ubuntu/actions-runner/e2e-home
	CARGO_HOME: /home/ubuntu/actions-runner/.cargo
	RUSTUP_HOME: /home/ubuntu/actions-runner/.rustup
	TMPDIR: /home/ubuntu/actions-runner/tmp
	PIP_CACHE_DIR: /home/ubuntu/actions-runner/pip-cache
	steps:
	- uses: actions/checkout@v6

	- name: Prepare writable dirs on the nvme
	run: mkdir -p /home/ubuntu/actions-runner/e2e-home /home/ubuntu/actions-runner/tmp /home/ubuntu/actions-runner/pip-cache

	# Bootstrap rustup ourselves with --no-modify-path so it never writes to
	# $HOME/.profile (setup-rust-toolchain doesn't expose that flag).
	# rust-toolchain.toml pins the exact toolchain, installed on first cargo
	# use. Idempotent.
	- name: Ensure Rust toolchain
	run: \|
	if ! command -v cargo >/dev/null 2>&1 && [ ! -x "$CARGO_HOME/bin/cargo" ]; then
	curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \
	\| sh -s -- -y --no-modify-path --default-toolchain none
	fi
	echo "$CARGO_HOME/bin" >> "$GITHUB_PATH"

	- name: Run E2E tests on Strix Halo (excluding known bugs)
	run: \|
	export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target"
	export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared"
	cargo xtask e2e -- -t "@gpu and not @expected-failure"

	- name: Upload E2E report
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: e2e-gpu-strix-ubuntu-report
	path: tests/e2e-cucumber/results/

	e2e-gpu-strix-ubuntu-known-bugs:
	name: E2E tests (Strix Halo, Ubuntu, known bugs)
	timeout-minutes: 15
	runs-on: [self-hosted, linux, strix-halo]
	needs: [changes, build-and-test]
	# See `e2e`: dispatch tolerates skipped build-and-test; strix-ubuntu, known-bugs.
	if: >-
	always()
	&& needs.changes.result == 'success'
	&& (
	(github.event_name != 'workflow_dispatch'
	&& needs.build-and-test.result == 'success'
	&& needs.changes.outputs.heavy == 'true')
	\|\| (github.event_name == 'workflow_dispatch'
	&& (inputs.platform == 'all' \|\| inputs.platform == 'strix-ubuntu')
	&& (inputs.tier == 'both' \|\| inputs.tier == 'known-bugs'))
	)
	continue-on-error: true
	# See e2e-gpu-strix-ubuntu: /, /home/ubuntu, and /tmp are all on a full root
	# partition; only the nvme has space. Point HOME + all caches at the nvme so
	# nothing writes to root (pip's cache under the real $HOME is what failed
	# `install sdk` with ENOSPC).
	env:
	HOME: /home/ubuntu/actions-runner/e2e-home
	CARGO_HOME: /home/ubuntu/actions-runner/.cargo
	RUSTUP_HOME: /home/ubuntu/actions-runner/.rustup
	TMPDIR: /home/ubuntu/actions-runner/tmp
	PIP_CACHE_DIR: /home/ubuntu/actions-runner/pip-cache
	# Known-bug tier: fail fast rather than wait the default 600s on a serve
	# that a bug prevents from becoming ready.
	E2E_SERVE_TIMEOUT_SECS: "90"
	steps:
	- uses: actions/checkout@v6

	- name: Prepare writable dirs on the nvme
	run: mkdir -p /home/ubuntu/actions-runner/e2e-home /home/ubuntu/actions-runner/tmp /home/ubuntu/actions-runner/pip-cache

	# See e2e-gpu-strix-ubuntu: bootstrap rustup with --no-modify-path so it
	# never writes $HOME/.profile on the full root disk.
	- name: Ensure Rust toolchain
	run: \|
	if ! command -v cargo >/dev/null 2>&1 && [ ! -x "$CARGO_HOME/bin/cargo" ]; then
	curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \
	\| sh -s -- -y --no-modify-path --default-toolchain none
	fi
	echo "$CARGO_HOME/bin" >> "$GITHUB_PATH"

	- name: Run known-bug E2E scenarios on Strix Halo
	run: \|
	export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target"
	export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared"
	cargo xtask e2e --expect-failures -- -t "@gpu and @expected-failure"

	- name: Upload E2E report
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: e2e-gpu-strix-ubuntu-known-bugs-report
	path: tests/e2e-cucumber/results/

	# First real Windows GPU coverage: the Strix Halo Windows 11 runner. The
	# existing windows-build-and-test uses GitHub-hosted windows-latest, which has
	# no GPU, so `@gpu` scenarios have never run on Windows hardware. Expect
	# first-run breakage to triage; non-blocking so it never gates the PR.
	e2e-gpu-strix-windows:
	name: E2E tests (Strix Halo, Windows)
	timeout-minutes: 15
	runs-on: [self-hosted, windows, strix-halo]
	needs: [changes, build-and-test]
	# See `e2e`: dispatch tolerates skipped build-and-test; strix-windows, expect-pass.
	if: >-
	always()
	&& needs.changes.result == 'success'
	&& (
	(github.event_name != 'workflow_dispatch'
	&& needs.build-and-test.result == 'success'
	&& needs.changes.outputs.heavy == 'true')
	\|\| (github.event_name == 'workflow_dispatch'
	&& (inputs.platform == 'all' \|\| inputs.platform == 'strix-windows')
	&& (inputs.tier == 'both' \|\| inputs.tier == 'expect-pass'))
	)
	continue-on-error: true
	steps:
	- uses: actions/checkout@v6

	# setup-rust-toolchain runs an internal bash script, which this Windows
	# runner lacks (bash: command not found). Bootstrap rustup with the
	# PowerShell-native installer instead; idempotent, so it only downloads on
	# a runner that doesn't already have the toolchain. Use `powershell`
	# (Windows PowerShell 5.1, always present) rather than `pwsh` (PowerShell 7),
	# which this self-hosted runner does not have installed.
	- name: Ensure Rust toolchain (PowerShell)
	shell: powershell
	run: \|
	if (-not (Get-Command cargo -ErrorAction SilentlyContinue)) {
	Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile $env:TEMP\rustup-init.exe
	# --default-toolchain none: rust-toolchain.toml pins the exact
	# version (1.96.0 + components), auto-installed on first cargo use.
	& $env:TEMP\rustup-init.exe -y --default-toolchain none
	"$env:USERPROFILE\.cargo\bin" \| Out-File -FilePath $env:GITHUB_PATH -Append
	}

	- name: Run E2E tests on Strix Halo Windows (excluding known bugs)
	shell: powershell
	run: cargo xtask e2e -- -t "@gpu and not @expected-failure"

	- name: Upload E2E report
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: e2e-gpu-strix-windows-report
	path: tests/e2e-cucumber/results/

	e2e-gpu-strix-windows-known-bugs:
	name: E2E tests (Strix Halo, Windows, known bugs)
	timeout-minutes: 15
	runs-on: [self-hosted, windows, strix-halo]
	needs: [changes, build-and-test]
	# See `e2e`: dispatch tolerates skipped build-and-test; strix-windows, known-bugs.
	if: >-
	always()
	&& needs.changes.result == 'success'
	&& (
	(github.event_name != 'workflow_dispatch'
	&& needs.build-and-test.result == 'success'
	&& needs.changes.outputs.heavy == 'true')
	\|\| (github.event_name == 'workflow_dispatch'
	&& (inputs.platform == 'all' \|\| inputs.platform == 'strix-windows')
	&& (inputs.tier == 'both' \|\| inputs.tier == 'known-bugs'))
	)
	continue-on-error: true
	# Known-bug tier: fail fast rather than wait the default 600s on a serve that
	# a bug prevents from becoming ready.
	env:
	E2E_SERVE_TIMEOUT_SECS: "90"
	steps:
	- uses: actions/checkout@v6

	# See e2e-gpu-strix-windows: bootstrap rustup via Windows PowerShell 5.1
	# (`powershell`, not `pwsh`) because this runner has neither bash nor
	# PowerShell 7 for setup-rust-toolchain's internal script.
	- name: Ensure Rust toolchain (PowerShell)
	shell: powershell
	run: \|
	if (-not (Get-Command cargo -ErrorAction SilentlyContinue)) {
	Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile $env:TEMP\rustup-init.exe
	# --default-toolchain none: rust-toolchain.toml pins the exact
	# version (1.96.0 + components), auto-installed on first cargo use.
	& $env:TEMP\rustup-init.exe -y --default-toolchain none
	"$env:USERPROFILE\.cargo\bin" \| Out-File -FilePath $env:GITHUB_PATH -Append
	}

	- name: Run known-bug E2E scenarios on Strix Halo Windows
	shell: powershell
	run: cargo xtask e2e --expect-failures -- -t "@gpu and @expected-failure"

	- name: Upload E2E report
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: e2e-gpu-strix-windows-known-bugs-report
	path: tests/e2e-cucumber/results/

	# Consolidate every platform/tier's report into ONE cross-platform report:
	# a platform × tier matrix in the run Summary plus a single merged HTML
	# artifact. `if: always()` so a failing/non-blocking platform still appears;
	# runs on GitHub-hosted ubuntu (no GPU needed — it only parses report.json).
	#
	# Extensibility: the report CONTENT auto-includes any new platform via the
	# `*-report` artifact glob below — no change needed here. The one manual step
	# when adding a platform is appending its two job names to `needs:` (for run
	# ORDERING only; GitHub Actions has no wildcard `needs`). A future matrix
	# refactor of the 8 e2e jobs would remove even that.
	e2e-report:
	name: E2E consolidated report
	runs-on: ubuntu-latest
	needs:
	- changes
	- e2e
	- e2e-known-bugs
	- e2e-gpu
	- e2e-gpu-known-bugs
	- e2e-gpu-strix-ubuntu
	- e2e-gpu-strix-ubuntu-known-bugs
	- e2e-gpu-strix-windows
	- e2e-gpu-strix-windows-known-bugs
	# Consolidate whatever ran. On dispatch `heavy` is unset, so also run when the
	# trigger was manual; `always()` still lets it collect partial/failed tiers.
	if: >-
	always()
	&& (needs.changes.outputs.heavy == 'true'
	\|\| github.event_name == 'workflow_dispatch')
	steps:
	- uses: actions/checkout@v6

	- uses: actions-rust-lang/setup-rust-toolchain@v1

	# Pull every e2e artifact. Each extracts to e2e-artifacts/<artifact-name>/,
	# which `xtask e2e-report` turns into one labeled platform. The `*-report`
	# glob is what makes new platforms appear automatically.
	- name: Download all E2E reports
	uses: actions/download-artifact@v4
	with:
	pattern: '*-report'
	path: e2e-artifacts

	- name: Build consolidated report + step summary
	run: \|
	mkdir -p consolidated
	cargo xtask e2e-report \
	--artifacts-dir e2e-artifacts \
	--html-out consolidated/index.html >> "$GITHUB_STEP_SUMMARY"

	- name: Upload consolidated report
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: e2e-consolidated-report
	path: consolidated/

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

CI #530

Workflow file

CI #530

Uh oh!

Workflow file for this run