Nightly #85
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly | |
| on: | |
| schedule: | |
| - cron: "0 6 * * *" | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| env: | |
| CARGO_TERM_COLOR: always | |
| # Production trust gate — dormant until the owner sets the repository variable | |
| # ROCM_CLI_REQUIRE_PRODUCTION_TRUST (e.g. "1") and provides the trust inputs | |
| # below. While the variable is unset/falsy, release_readiness.py skips | |
| # validate_production_trust() and behavior is unchanged. The values are public | |
| # keys and paths (not the private signing key, which stays step-scoped). When | |
| # enabling the gate, the recipe-index inputs also require a step that | |
| # materializes the hosted index + its .sig into the workspace. | |
| ROCM_CLI_REQUIRE_PRODUCTION_TRUST: ${{ vars.ROCM_CLI_REQUIRE_PRODUCTION_TRUST }} | |
| ROCM_CLI_SIGNING_PUBLIC_KEY_PEM: ${{ secrets.ROCM_CLI_SIGNING_PUBLIC_KEY_PEM }} | |
| ROCM_CLI_METADATA_PUBLIC_KEY_PEM: ${{ secrets.ROCM_CLI_METADATA_PUBLIC_KEY_PEM }} | |
| ROCM_CLI_MODEL_RECIPE_INDEX_PATH: ${{ vars.ROCM_CLI_MODEL_RECIPE_INDEX_PATH }} | |
| ROCM_CLI_MODEL_RECIPE_INDEX_PUBLIC_KEY_PATH: ${{ vars.ROCM_CLI_MODEL_RECIPE_INDEX_PUBLIC_KEY_PATH }} | |
| jobs: | |
| nightly: | |
| name: Build Linux nightly assets | |
| runs-on: ubuntu-latest | |
| outputs: | |
| skip: ${{ steps.check.outputs.skip }} | |
| date: ${{ steps.meta.outputs.date }} | |
| short_sha: ${{ steps.meta.outputs.short_sha }} | |
| staging_tag: ${{ steps.staging.outputs.tag }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 | |
| - name: Verify pinned keys match the signing key | |
| run: cargo xtask verify-pinned-keys | |
| - name: Check for changes since last nightly | |
| id: check | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| LAST=$(gh release view nightly --json createdAt --jq .createdAt 2>/dev/null || echo "1970-01-01T00:00:00Z") | |
| COMMITS=$(git log --since="${LAST}" --oneline | wc -l) | |
| echo "commits=${COMMITS}" >> "$GITHUB_OUTPUT" | |
| if [ "${COMMITS}" -eq 0 ] && [ "${{ github.event_name }}" != "workflow_dispatch" ]; then | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| echo "No new commits since last nightly — skipping" | |
| else | |
| echo "skip=false" >> "$GITHUB_OUTPUT" | |
| echo "${COMMITS} new commit(s) since last nightly" | |
| fi | |
| - name: Cache Cargo home and build artifacts | |
| if: steps.check.outputs.skip != 'true' | |
| uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| path: | | |
| ~/.cargo/bin | |
| ~/.cargo/registry/cache | |
| ~/.cargo/registry/index | |
| ~/.cargo/registry/src | |
| ~/.cargo/rustup | |
| target | |
| key: Linux-manylinux-release-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }} | |
| restore-keys: | | |
| Linux-manylinux-release- | |
| - name: Build release binaries | |
| if: steps.check.outputs.skip != 'true' | |
| run: | | |
| # The bind-mount source must exist and be owned by the runner user; | |
| # Docker would otherwise create it as root, leaving the non-root | |
| # container user unable to write into CARGO_HOME/RUSTUP_HOME. | |
| mkdir -p "$HOME/.cargo" | |
| # Install system build deps as root: yum needs to write to | |
| # /var/cache/yum and the RPM db, which a non-root container user | |
| # cannot do. These land in the container's ephemeral layer (not the | |
| # bind mount), so this runs every time; only the Cargo/rustup state | |
| # below is cached. | |
| docker run --rm \ | |
| -v "$PWD":/work \ | |
| -w /work \ | |
| quay.io/pypa/manylinux2014_x86_64 \ | |
| /bin/bash -lc ' | |
| set -euo pipefail | |
| yum install -y pkgconfig libcap-devel | |
| ' | |
| # Build as the runner user so cached Cargo/rustup files are owned by | |
| # the runner (the host-side cache save step must be able to read them). | |
| docker run --rm \ | |
| -u "$(id -u):$(id -g)" \ | |
| -e CARGO_HOME=/host-cargo \ | |
| -e RUSTUP_HOME=/host-cargo/rustup \ | |
| -v "$HOME/.cargo":/host-cargo \ | |
| -v "$PWD":/work \ | |
| -w /work \ | |
| quay.io/pypa/manylinux2014_x86_64 \ | |
| /bin/bash -lc ' | |
| set -euo pipefail | |
| if [ ! -f /host-cargo/bin/cargo ]; then | |
| curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain none --no-modify-path | |
| fi | |
| export PATH="/host-cargo/bin:$PATH" | |
| cargo build --release -p rocm -p rocmd -p xtask | |
| ' | |
| - name: Set nightly metadata | |
| id: meta | |
| if: steps.check.outputs.skip != 'true' | |
| run: | | |
| echo "date=$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT" | |
| echo "short_sha=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" | |
| - name: Package nightly bundle | |
| if: steps.check.outputs.skip != 'true' | |
| env: | |
| ROCM_CLI_SIGNING_PRIVATE_KEY_PEM: ${{ secrets.ROCM_CLI_SIGNING_PRIVATE_KEY_PEM }} | |
| ROCM_CLI_REQUIRE_SIGNATURE: "1" | |
| run: | | |
| DATE="${{ steps.meta.outputs.date }}" | |
| SHORT_SHA="${{ steps.meta.outputs.short_sha }}" | |
| DIST="rocm-cli-nightly-${DATE}-${SHORT_SHA}-linux-amd64" | |
| chmod +x install.sh | |
| cargo xtask package "${DIST}" | |
| cp "dist/${DIST}.tar.gz" dist/rocm-cli-nightly-linux-amd64.tar.gz | |
| (cd dist && sha256sum rocm-cli-nightly-linux-amd64.tar.gz > rocm-cli-nightly-linux-amd64.tar.gz.sha256) | |
| if [ -f "dist/${DIST}.tar.gz.sig" ]; then | |
| cp "dist/${DIST}.tar.gz.sig" dist/rocm-cli-nightly-linux-amd64.tar.gz.sig | |
| else | |
| echo "nightly signature was not produced" >&2 | |
| exit 1 | |
| fi | |
| python scripts/release_readiness.py \ | |
| --dist dist \ | |
| --require-signatures \ | |
| --require-rocm-asset-names \ | |
| --require-exact-assets \ | |
| --asset "${DIST}.tar.gz" \ | |
| --asset rocm-cli-nightly-linux-amd64.tar.gz | |
| echo "DIST=${DIST}" >> "$GITHUB_ENV" | |
| echo "DATE=${DATE}" >> "$GITHUB_ENV" | |
| echo "SHORT_SHA=${SHORT_SHA}" >> "$GITHUB_ENV" | |
| - name: Create staging nightly release | |
| id: staging | |
| if: steps.check.outputs.skip != 'true' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| STAGING_TAG="nightly-staging-${DATE}-${SHORT_SHA}" | |
| gh release delete "${STAGING_TAG}" --yes --cleanup-tag 2>/dev/null || true | |
| gh release create "${STAGING_TAG}" \ | |
| --draft \ | |
| --title "rocm-cli nightly (${DATE} · ${SHORT_SHA})" \ | |
| --notes "$(cat <<EOF | |
| Automated nightly build from \`main\` at $(date -u +"%Y-%m-%d %H:%M UTC"). | |
| Commit: [\`${SHORT_SHA}\`](https://github.com/${{ github.repository }}/commit/$(git rev-parse HEAD)) | |
| ${{ steps.check.outputs.commits }} commit(s) since the previous nightly. | |
| Install the latest nightly on Linux x86_64: | |
| \`\`\`bash | |
| curl -fsSL https://raw.githubusercontent.com/${{ github.repository }}/main/install.sh | sh -s -- nightly | |
| \`\`\` | |
| Install the latest nightly on Windows x86_64 from PowerShell: | |
| \`\`\`powershell | |
| \$script = "\$env:TEMP\install-rocm-cli.ps1"; irm https://raw.githubusercontent.com/${{ github.repository }}/main/install.ps1 -OutFile \$script; powershell -ExecutionPolicy Bypass -File \$script nightly | |
| \`\`\` | |
| EOF | |
| )" \ | |
| --prerelease \ | |
| dist/*.tar.gz dist/*.tar.gz.sha256 dist/*.sig | |
| echo "tag=${STAGING_TAG}" >> "$GITHUB_OUTPUT" | |
| windows-nightly: | |
| name: Build and upload Windows nightly | |
| runs-on: windows-latest | |
| needs: nightly | |
| if: needs.nightly.outputs.skip != 'true' | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 | |
| - name: Verify pinned keys match the signing key | |
| shell: pwsh | |
| run: cargo xtask verify-pinned-keys | |
| - name: Build release binaries | |
| shell: pwsh | |
| run: cargo build --release -p rocm -p rocmd -p xtask | |
| - name: Package Windows bundle | |
| shell: pwsh | |
| env: | |
| ROCM_CLI_SIGNING_PRIVATE_KEY_PEM: ${{ secrets.ROCM_CLI_SIGNING_PRIVATE_KEY_PEM }} | |
| ROCM_CLI_REQUIRE_SIGNATURE: "1" | |
| run: | | |
| $date = "${{ needs.nightly.outputs.date }}" | |
| $shortSha = "${{ needs.nightly.outputs.short_sha }}" | |
| $dist = "rocm-cli-nightly-$date-$shortSha-windows-amd64" | |
| cargo xtask package $dist | |
| Copy-Item "dist\$dist.zip" "dist\rocm-cli-nightly-windows-amd64.zip" | |
| $hash = (Get-FileHash -Algorithm SHA256 -LiteralPath "dist\rocm-cli-nightly-windows-amd64.zip").Hash.ToLowerInvariant() | |
| Set-Content -LiteralPath "dist\rocm-cli-nightly-windows-amd64.zip.sha256" -Value "$hash rocm-cli-nightly-windows-amd64.zip" -Encoding ascii | |
| if (Test-Path "dist\$dist.zip.sig") { | |
| Copy-Item "dist\$dist.zip.sig" "dist\rocm-cli-nightly-windows-amd64.zip.sig" | |
| } else { | |
| throw "nightly signature was not produced" | |
| } | |
| python .\scripts\release_readiness.py ` | |
| --dist dist ` | |
| --require-signatures ` | |
| --require-rocm-asset-names ` | |
| --require-exact-assets ` | |
| --asset "$dist.zip" ` | |
| --asset rocm-cli-nightly-windows-amd64.zip | |
| - name: Upload Windows nightly assets | |
| shell: pwsh | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| $stagingTag = "${{ needs.nightly.outputs.staging_tag }}" | |
| if (-not $stagingTag) { | |
| throw "nightly staging release tag was not produced" | |
| } | |
| if (-not (Get-ChildItem dist -File -Include *.sig -Recurse)) { | |
| throw "nightly signatures are required" | |
| } | |
| $assets = Get-ChildItem dist -File -Include *.zip,*.sha256,*.sig -Recurse | ForEach-Object { $_.FullName } | |
| gh release upload $stagingTag @assets --clobber | |
| publish-nightly: | |
| name: Publish nightly | |
| runs-on: ubuntu-latest | |
| needs: | |
| - nightly | |
| - windows-nightly | |
| if: needs.nightly.outputs.skip != 'true' | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Publish staged nightly | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GH_REPO: ${{ github.repository }} | |
| run: | | |
| STAGING_TAG="${{ needs.nightly.outputs.staging_tag }}" | |
| DATE="${{ needs.nightly.outputs.date }}" | |
| SHORT_SHA="${{ needs.nightly.outputs.short_sha }}" | |
| ASSET_DIR="$(mktemp -d)" | |
| if [ -z "${STAGING_TAG}" ]; then | |
| echo "nightly staging release tag was not produced" >&2 | |
| exit 1 | |
| fi | |
| gh release download "${STAGING_TAG}" --dir "${ASSET_DIR}" | |
| python scripts/release_readiness.py \ | |
| --dist "${ASSET_DIR}" \ | |
| --require-signatures \ | |
| --require-rocm-asset-names \ | |
| --require-exact-assets \ | |
| --asset "rocm-cli-nightly-${DATE}-${SHORT_SHA}-linux-amd64.tar.gz" \ | |
| --asset rocm-cli-nightly-linux-amd64.tar.gz \ | |
| --asset "rocm-cli-nightly-${DATE}-${SHORT_SHA}-windows-amd64.zip" \ | |
| --asset rocm-cli-nightly-windows-amd64.zip | |
| TITLE="$(gh release view "${STAGING_TAG}" --json name -q .name)" | |
| NOTES="$(gh release view "${STAGING_TAG}" --json body -q .body)" | |
| gh release delete nightly --yes --cleanup-tag 2>/dev/null || true | |
| gh release create nightly \ | |
| --title "${TITLE}" \ | |
| --notes "${NOTES}" \ | |
| --prerelease \ | |
| "${ASSET_DIR}"/* | |
| # Convenience assets for quick curl-and-run, attached after the | |
| # readiness gate (NOT part of the signed exact-asset set): | |
| # rocm / rocm.exe - standalone CLI binaries | |
| # linux-binaries.tar.gz - byte-identical alias of the signed | |
| # rocm-cli-nightly-linux-amd64.tar.gz bundle | |
| # windows-binaries.zip - byte-identical alias of the signed | |
| # rocm-cli-nightly-windows-amd64.zip bundle | |
| # Integrity-conscious users should prefer the signed *-amd64 archives. | |
| EXTRA="$(mktemp -d)" | |
| mkdir -p "${EXTRA}/lin" "${EXTRA}/win" | |
| tar -xzf "${ASSET_DIR}/rocm-cli-nightly-linux-amd64.tar.gz" -C "${EXTRA}/lin" | |
| unzip -o -q "${ASSET_DIR}/rocm-cli-nightly-windows-amd64.zip" -d "${EXTRA}/win" | |
| cp "$(find "${EXTRA}/lin" -type f -name rocm)" "${EXTRA}/rocm" | |
| cp "$(find "${EXTRA}/win" -type f -name rocm.exe)" "${EXTRA}/rocm.exe" | |
| cp "${ASSET_DIR}/rocm-cli-nightly-linux-amd64.tar.gz" "${EXTRA}/linux-binaries.tar.gz" | |
| cp "${ASSET_DIR}/rocm-cli-nightly-windows-amd64.zip" "${EXTRA}/windows-binaries.zip" | |
| gh release upload nightly \ | |
| "${EXTRA}/rocm" "${EXTRA}/rocm.exe" \ | |
| "${EXTRA}/linux-binaries.tar.gz" "${EXTRA}/windows-binaries.zip" --clobber | |
| # The staging release is a draft, which never creates a git tag, so | |
| # --cleanup-tag would 422 on a nonexistent ref. Just delete the draft. | |
| gh release delete "${STAGING_TAG}" --yes 2>/dev/null || true | |
| cleanup-staging-nightly: | |
| name: Clean failed nightly staging release | |
| runs-on: ubuntu-latest | |
| needs: | |
| - nightly | |
| - windows-nightly | |
| if: always() && needs.nightly.outputs.skip != 'true' && needs.windows-nightly.result != 'success' && needs.nightly.outputs.staging_tag != '' | |
| steps: | |
| - name: Delete staging release | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GH_REPO: ${{ github.repository }} | |
| run: gh release delete "${{ needs.nightly.outputs.staging_tag }}" --yes --cleanup-tag 2>/dev/null || true | |
| # Full E2E on app-dev MI300X INCLUDING the expensive `@nightly` scenarios (the | |
| # large-model serve), which the per-PR ci.yml `e2e-gpu` job skips to stay fast. | |
| # Runs unconditionally on the schedule/dispatch (independent of the nightly | |
| # BUILD skip above — a regression run is worth doing even with no new commits). | |
| # Non-blocking, mirrors ci.yml's e2e-gpu job but sets E2E_INCLUDE_NIGHTLY. | |
| e2e-gpu-nightly: | |
| name: E2E tests (GPU, incl. nightly-only) | |
| timeout-minutes: 90 | |
| runs-on: [self-hosted, linux, amd-gpu] | |
| continue-on-error: true | |
| env: | |
| # Include @nightly scenarios (the large-model serve). See src/expectation.rs. | |
| E2E_INCLUDE_NIGHTLY: "1" | |
| # Serve-readiness floor; a `@serve-timeout` tag lengthens it for the large | |
| # model (its cold ~54 GiB load exceeds this default). | |
| E2E_SERVE_TIMEOUT_SECS: "300" | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| # Reclaim the GPU before running: kill only e2e leftovers scoped to | |
| # /tmp/rocm-e2e-* and the e2e-target/e2e-shared trees — never the runner or | |
| # any /workload manual-testing processes (see ci.yml e2e-gpu for the same). | |
| - name: Reclaim GPU from stray E2E processes | |
| run: | | |
| pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true | |
| pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true | |
| pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true | |
| pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true | |
| pkill -f 'e2e-target/release/rocm daemon' 2>/dev/null || true | |
| rm -rf /tmp/rocm-e2e-* 2>/dev/null || true | |
| echo "reclaimed" | |
| # cache: false — this self-hosted runner persists the build cache itself via | |
| # CARGO_TARGET_DIR (below), so the action's rust-cache is pure overhead. | |
| - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 | |
| with: | |
| cache: false | |
| - name: Run full E2E on GPU hardware (incl. nightly-only) | |
| run: | | |
| export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" | |
| export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" | |
| # Share uv's wheel cache off the near-full PVC, on the roomy `/` overlay | |
| # (see ci.yml e2e-gpu): warm `rocm install sdk` ~34s vs cold ~160s. | |
| export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache" | |
| # Share ONE installed managed runtime across serve/chat scenarios so | |
| # install sdk runs once per runner, not per scenario (see ci.yml e2e-gpu). | |
| # The shared dir IS the pre-warm's own data/runtimes; NEVER move it — a | |
| # post-install mv invalidates the absolute paths install sdk bakes into | |
| # the runtime manifest and every serve fails instantly (run 29320025393). | |
| prewarm="$RUNNER_WORKSPACE/e2e-prewarm" | |
| export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" | |
| # Build both binaries once; reuse them for pre-warm + suite. | |
| # See the e2e-gpu lane in e2e-selfhosted.yml for why the | |
| # e2e-test-hooks feature must match what `cargo xtask e2e` would build. | |
| cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks | |
| export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" | |
| export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" | |
| # Pre-warm the shared runtime once/serially before the suite, refreshing | |
| # it when the channel index has a newer version. Install in place (no mv) | |
| # so the manifest's absolute paths stay valid. See the e2e-selfhosted.yml | |
| # e2e-gpu lane and `xtask e2e-prewarm` for the full rationale (EAI-8057). | |
| HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ | |
| UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ | |
| cargo xtask e2e-prewarm --channel release --prewarm-dir "$prewarm" | |
| cargo xtask e2e | |
| # Canonical artifact name, matching the per-PR workflow rather than a | |
| # `-nightly-` variant. `parse_descriptor` in the e2e-report crate maps | |
| # these names to Platform/OS, and an unrecognised one falls back to a | |
| # titlecased platform with the OS hardcoded to Linux — which would have | |
| # labelled the Windows lane below as Linux. Artifacts are scoped to their | |
| # run, so sharing the name with the per-PR workflow collides with nothing. | |
| - name: Upload E2E report | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: e2e-gpu-report | |
| path: tests/e2e-cucumber/results/ | |
| # rad3 R9700 counterpart to e2e-gpu-nightly. Mirrors the per-PR e2e-gpu-rad3 | |
| # lane in e2e-selfhosted.yml (scale-to-zero runner, r9700 label, own uv-cache | |
| # PVC, GPU preflight), with @nightly scenarios enabled. Keep the two in step: | |
| # the report crate asserts the nightly lanes publish the same platforms as the | |
| # per-PR lanes. | |
| e2e-gpu-nightly-rad3: | |
| name: E2E tests (rad3 R9700, incl. nightly-only) | |
| timeout-minutes: 90 | |
| runs-on: [self-hosted, linux, r9700] | |
| continue-on-error: true | |
| env: | |
| E2E_INCLUDE_NIGHTLY: "1" | |
| E2E_SERVE_TIMEOUT_SECS: "300" | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Reclaim GPU from stray E2E processes | |
| run: | | |
| pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true | |
| pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true | |
| pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true | |
| pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true | |
| pkill -f 'e2e-target/release/rocm daemon' 2>/dev/null || true | |
| rm -rf /tmp/rocm-e2e-* 2>/dev/null || true | |
| echo "reclaimed" | |
| # Bounded wait for a free GPU. Floor is half of the R9700S's 32GB — high | |
| # enough to catch a leftover serve, low enough that a clean pod passes on | |
| # the first poll. Same as the per-PR rad3 lane. | |
| - name: GPU preflight (bounded wait for an available GPU) | |
| run: | | |
| MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-16}" | |
| CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" | |
| min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) | |
| deadline=$(( SECONDS + CEILING_SECS )) | |
| reason="rocm-smi never returned within its timeout (driver wedged or GPU absent)" | |
| while [ "$SECONDS" -lt "$deadline" ]; do | |
| out=$(timeout 15 rocm-smi --showmeminfo vram 2>/dev/null) || { sleep 5; continue; } | |
| total=$(printf '%s\n' "$out" | grep -i 'VRAM Total Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) | |
| used=$(printf '%s\n' "$out" | grep -i 'VRAM Total Used Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) | |
| if [ -z "$total" ] || [ -z "$used" ]; then | |
| reason="rocm-smi returned no VRAM figures (no AMD GPU detected)" | |
| sleep 5; continue | |
| fi | |
| free=$(( total - used )) | |
| if [ "$free" -ge "$min_free" ]; then | |
| echo "GPU ready: $(( free / 1024 / 1024 / 1024 )) GiB free (>= ${MIN_FREE_GIB} GiB)." | |
| exit 0 | |
| fi | |
| reason="VRAM never dropped below the floor: only $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB) — a serve is likely still holding the GPU" | |
| echo "waiting: $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB)…" | |
| sleep 5 | |
| done | |
| echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" | |
| exit 1 | |
| - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 | |
| with: | |
| cache: false | |
| - name: Run full E2E on rad3 (incl. nightly-only) | |
| run: | | |
| # $RUNNER_WORKSPACE is on the runner-work PVC, which survives the pod | |
| # being scaled away, so the cargo target dir and shared caches stay warm. | |
| export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" | |
| export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" | |
| # Separate PVC mounted at exactly this path by the runner's cluster | |
| # overlay; if you change this path, change the overlay that mounts it too. | |
| export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache" | |
| prewarm="$RUNNER_WORKSPACE/e2e-prewarm" | |
| export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" | |
| # See the e2e-gpu lane in e2e-selfhosted.yml for why the | |
| # e2e-test-hooks feature must match what `cargo xtask e2e` would build. | |
| cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks | |
| export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" | |
| export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" | |
| # Pre-warm once, serially, in place (no mv/symlink), and refresh it when | |
| # the channel index has published a newer runtime — the tree is a cache, | |
| # not a one-shot. See `xtask e2e-prewarm` (EAI-8057). | |
| HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ | |
| UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ | |
| cargo xtask e2e-prewarm --channel release --prewarm-dir "$prewarm" | |
| cargo xtask e2e | |
| - name: Upload E2E report | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: e2e-gpu-rad3-report | |
| path: tests/e2e-cucumber/results/ | |
| # Strix Halo counterpart to e2e-gpu-nightly. The same @nightly large-model | |
| # scenario serves the platform-specific Lemonade GGUF on this host. Keep the | |
| # runner's storage and runtime setup aligned with ci.yml's proven Strix Ubuntu job. | |
| e2e-gpu-nightly-strix: | |
| name: E2E tests (Strix Halo, incl. nightly-only) | |
| timeout-minutes: 90 | |
| # `native` disambiguates the two Linux Strix runners: the WSL host also | |
| # carries `strix-halo`, but the paths below exist only on the native one. | |
| runs-on: [self-hosted, linux, strix-halo, native] | |
| continue-on-error: true | |
| env: | |
| HOME: /home/ubuntu/actions-runner/e2e-home | |
| CARGO_HOME: /home/ubuntu/actions-runner/.cargo | |
| RUSTUP_HOME: /home/ubuntu/actions-runner/.rustup | |
| TMPDIR: /home/ubuntu/actions-runner/tmp | |
| PIP_CACHE_DIR: /home/ubuntu/actions-runner/pip-cache | |
| E2E_INCLUDE_NIGHTLY: "1" | |
| E2E_SERVE_TIMEOUT_SECS: "300" | |
| E2E_TUI_TIMEOUT_SECS: "90" | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Prepare writable dirs on the nvme + reclaim GPU from stray E2E procs | |
| run: | | |
| mkdir -p "$HOME" "$TMPDIR" "$PIP_CACHE_DIR" | |
| pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true | |
| pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true | |
| pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true | |
| pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true | |
| rm -rf /tmp/rocm-e2e-* 2>/dev/null || true | |
| echo "prepared + reclaimed" | |
| - name: GPU preflight (bounded wait for an available GPU) | |
| run: | | |
| MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-8}" | |
| CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" | |
| min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) | |
| deadline=$(( SECONDS + CEILING_SECS )) | |
| reason="rocm-smi never returned within its timeout (driver wedged or GPU absent)" | |
| while [ "$SECONDS" -lt "$deadline" ]; do | |
| out=$(timeout 15 rocm-smi --showmeminfo vram 2>/dev/null) || { sleep 5; continue; } | |
| total=$(printf '%s\n' "$out" | grep -i 'VRAM Total Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) | |
| used=$(printf '%s\n' "$out" | grep -i 'VRAM Total Used Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) | |
| if [ -z "$total" ] || [ -z "$used" ]; then | |
| reason="rocm-smi returned no VRAM figures (no AMD GPU detected)" | |
| sleep 5; continue | |
| fi | |
| free=$(( total - used )) | |
| if [ "$free" -ge "$min_free" ]; then | |
| echo "GPU ready: $(( free / 1024 / 1024 / 1024 )) GiB free (>= ${MIN_FREE_GIB} GiB)." | |
| exit 0 | |
| fi | |
| reason="VRAM never dropped below the floor: only $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB) — a serve is likely still holding the GPU" | |
| echo "waiting: $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB)…" | |
| sleep 5 | |
| done | |
| echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" | |
| exit 1 | |
| - name: Ensure Rust toolchain | |
| run: | | |
| if ! command -v cargo >/dev/null 2>&1 && [ ! -x "$CARGO_HOME/bin/cargo" ]; then | |
| curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \ | |
| | sh -s -- -y --no-modify-path --default-toolchain none | |
| fi | |
| echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" | |
| - name: Run E2E tests on Strix Halo (incl. nightly-only) | |
| run: | | |
| export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" | |
| export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" | |
| prewarm="$RUNNER_WORKSPACE/e2e-prewarm" | |
| export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" | |
| # See the e2e-gpu lane in e2e-selfhosted.yml for why the | |
| # e2e-test-hooks feature must match what `cargo xtask e2e` would build. | |
| cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks | |
| export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" | |
| export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" | |
| # Cache, not a one-shot: refreshed when the channel index has a newer | |
| # version. See `xtask e2e-prewarm` (EAI-8057). | |
| HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ | |
| cargo xtask e2e-prewarm --channel release --prewarm-dir "$prewarm" | |
| cargo xtask e2e | |
| - name: Upload E2E report | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: e2e-gpu-strix-ubuntu-report | |
| path: tests/e2e-cucumber/results/ | |
| # Windows counterpart to e2e-gpu-nightly-strix. Keep the PowerShell-native | |
| # setup aligned with ci.yml's proven Strix Windows job while opting into the | |
| # same platform-adaptive @nightly large-model scenario as the Linux runner. | |
| e2e-gpu-nightly-strix-windows: | |
| name: E2E tests (Strix Halo, Windows, incl. nightly-only) | |
| timeout-minutes: 90 | |
| runs-on: [self-hosted, windows, strix-halo, native] | |
| continue-on-error: true | |
| env: | |
| E2E_INCLUDE_NIGHTLY: "1" | |
| E2E_SERVE_TIMEOUT_SECS: "300" | |
| E2E_TUI_TIMEOUT_SECS: "90" | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Reclaim GPU from stray E2E processes | |
| shell: powershell | |
| run: | | |
| Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | | |
| Where-Object { $_.CommandLine -match 'rocm-e2e|__engine-serve-http|e2e-target|e2e-shared' -and $_.CommandLine -match 'llama-server|vllm|rocm ' } | | |
| ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue } | |
| Write-Host "reclaimed" | |
| # Best-effort when rocm-smi is absent; when present, fail within the bounded | |
| # ceiling instead of letting a held or wedged GPU consume the full job cap. | |
| - name: GPU preflight (bounded wait for an available GPU) | |
| shell: powershell | |
| run: | | |
| $minFreeGiB = if ($env:GPU_PREFLIGHT_MIN_FREE_GIB) { [int]$env:GPU_PREFLIGHT_MIN_FREE_GIB } else { 8 } | |
| $ceilingSecs = if ($env:GPU_PREFLIGHT_CEILING_SECS) { [int]$env:GPU_PREFLIGHT_CEILING_SECS } else { 90 } | |
| if (-not (Get-Command rocm-smi -ErrorAction SilentlyContinue)) { | |
| Write-Host "rocm-smi not found on this Windows runner; skipping GPU preflight (best-effort)." | |
| exit 0 | |
| } | |
| $minFree = [int64]$minFreeGiB * 1GB | |
| $deadline = (Get-Date).AddSeconds($ceilingSecs) | |
| $reason = "rocm-smi never returned usable VRAM figures" | |
| while ((Get-Date) -lt $deadline) { | |
| $out = (rocm-smi --showmeminfo vram 2>$null | Out-String) | |
| $total = ([regex]::Matches($out, 'VRAM Total Memory \(B\):\s*(\d+)') | Select-Object -First 1).Groups[1].Value | |
| $used = ([regex]::Matches($out, 'VRAM Total Used Memory \(B\):\s*(\d+)') | Select-Object -First 1).Groups[1].Value | |
| if (-not $total -or -not $used) { | |
| $reason = "rocm-smi returned no VRAM figures (no AMD GPU detected)" | |
| Start-Sleep -Seconds 5; continue | |
| } | |
| $free = [int64]$total - [int64]$used | |
| if ($free -ge $minFree) { | |
| Write-Host "GPU ready: $([math]::Floor($free/1GB)) GiB free (>= $minFreeGiB GiB)." | |
| exit 0 | |
| } | |
| $reason = "VRAM never dropped below the floor: only $([math]::Floor($free/1GB)) GiB free (< $minFreeGiB GiB) - a serve is likely still holding the GPU" | |
| Write-Host "waiting: $([math]::Floor($free/1GB)) GiB free (< $minFreeGiB GiB)..." | |
| Start-Sleep -Seconds 5 | |
| } | |
| Write-Host "::error::GPU preflight failed after ${ceilingSecs}s: $reason" | |
| exit 1 | |
| # setup-rust-toolchain needs bash, which is unavailable on this runner. | |
| - name: Ensure Rust toolchain (PowerShell) | |
| shell: powershell | |
| run: | | |
| if (-not (Get-Command cargo -ErrorAction SilentlyContinue)) { | |
| Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile $env:TEMP\rustup-init.exe | |
| & $env:TEMP\rustup-init.exe -y --default-toolchain none | |
| "$env:USERPROFILE\.cargo\bin" | Out-File -FilePath $env:GITHUB_PATH -Append | |
| } | |
| - name: Run E2E tests on Strix Halo Windows (incl. nightly-only) | |
| shell: powershell | |
| run: | | |
| # Install the shared runtime in place because its manifest contains | |
| # absolute paths. The persistent runner workspace keeps those paths valid | |
| # across scenarios and subsequent runs. | |
| $prewarm = "$env:RUNNER_WORKSPACE\e2e-prewarm" | |
| $env:E2E_SHARED_RUNTIMES_DIR = "$prewarm\data\runtimes" | |
| # See the e2e-gpu lane in e2e-selfhosted.yml for why the | |
| # e2e-test-hooks feature must match what `cargo xtask e2e` would build. | |
| cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks | |
| if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } | |
| $targetDir = if ($env:CARGO_TARGET_DIR) { $env:CARGO_TARGET_DIR } else { "target" } | |
| $env:ROCM_CLI_BINARY = "$targetDir\release\rocm.exe" | |
| $env:ROCM_CLI_ROCMD_BINARY = "$targetDir\release\rocmd.exe" | |
| # Cache, not a one-shot: refreshed when the channel index has a newer | |
| # version. The decision lives in `xtask e2e-prewarm` so this lane cannot | |
| # drift from the Linux ones (EAI-8057). | |
| cargo xtask e2e-prewarm --channel release --prewarm-dir "$prewarm" | |
| if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } | |
| cargo xtask e2e | |
| - name: Upload E2E report | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: e2e-gpu-strix-windows-report | |
| path: tests/e2e-cucumber/results/ | |
| # WSL2 counterpart to e2e-gpu-nightly-strix: the same Strix Halo box again, but | |
| # reached through an Ubuntu distro under WSL2, so this lane exercises the WSL | |
| # host boundary AND whatever GPU access WSL exposes. Mirrors the per-PR | |
| # `e2e-wsl` lane in e2e-selfhosted.yml, opting into the same @nightly | |
| # large-model scenario as the other nightly lanes. `wsl` in `runs-on` | |
| # disambiguates it from the `native` Strix Linux runner; the native lane's | |
| # /home/ubuntu/actions-runner paths deliberately are NOT reused, since they do | |
| # not exist in the WSL distro. | |
| e2e-wsl-nightly: | |
| name: E2E tests (Strix Halo, WSL2, incl. nightly-only) | |
| timeout-minutes: 90 | |
| runs-on: [self-hosted, linux, strix-halo, wsl] | |
| continue-on-error: true | |
| env: | |
| E2E_INCLUDE_NIGHTLY: "1" | |
| E2E_SERVE_TIMEOUT_SECS: "300" | |
| # The Strix lanes share one physical machine, so a TUI frame that renders | |
| # well inside the 30s default on an idle runner can miss it while a sibling | |
| # lane loads a model. Raise the wait budget rather than let that read as a | |
| # product failure; a genuine hang still fails, just later. | |
| E2E_TUI_TIMEOUT_SECS: "90" | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| # Fail fast and loudly if this job ever lands on a native Linux runner: | |
| # every WSL-tagged scenario would silently resolve to skip instead. | |
| - name: Verify the host really is WSL2 | |
| run: | | |
| proc_version=$(cat /proc/version 2>/dev/null || true) | |
| if [ ! -e /dev/dxg ] \ | |
| && [ -z "${WSL_DISTRO_NAME+x}" ] \ | |
| && ! printf '%s\n' "$proc_version" | grep -qiE 'microsoft|wsl'; then | |
| echo "::error::runner is not a WSL host: $(uname -r)" | |
| exit 1 | |
| fi | |
| printf 'WSL kernel: %s\n' "$(uname -r)" | |
| # The native Strix runners come pre-provisioned; a WSL distro often does | |
| # not. Install the native build deps only when something is missing, and | |
| # only if passwordless sudo is available — otherwise say what is missing | |
| # instead of hanging on a password prompt. | |
| - name: Ensure native build deps | |
| run: | | |
| missing="" | |
| command -v pkg-config >/dev/null 2>&1 || missing="$missing pkg-config" | |
| command -v cc >/dev/null 2>&1 || missing="$missing build-essential" | |
| pkg-config --exists libcap 2>/dev/null || missing="$missing libcap-dev" | |
| if [ -z "$missing" ]; then | |
| echo "native build deps present" | |
| elif sudo -n true 2>/dev/null; then | |
| echo "installing:$missing" | |
| sudo -n apt-get update | |
| # shellcheck disable=SC2086 | |
| sudo -n apt-get install -y $missing | |
| else | |
| echo "::error::missing native build deps:$missing (no passwordless sudo to install them)" | |
| exit 1 | |
| fi | |
| - name: Reclaim GPU from stray E2E processes | |
| run: | | |
| pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true | |
| pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true | |
| pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true | |
| pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true | |
| rm -rf /tmp/rocm-e2e-* 2>/dev/null || true | |
| echo "reclaimed" | |
| # GPU preflight, bounded like the native lanes — but ADVISORY here. GPU | |
| # access under WSL is exactly what this lane is proving out, so an absent | |
| # rocm-smi is a reported condition, not a job failure: the capability probe | |
| # then resolves @requires-gpu scenarios to skip and the rest still runs. A | |
| # GPU that IS present but stays held by a leftover serve still fails, since | |
| # that would corrupt the serve scenarios' results. | |
| - name: GPU preflight (advisory bounded wait) | |
| run: | | |
| MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-8}" | |
| CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" | |
| if ! command -v rocm-smi >/dev/null 2>&1; then | |
| echo "::warning::rocm-smi not found in this WSL distro — GPU scenarios will resolve to skip" | |
| exit 0 | |
| fi | |
| min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) | |
| deadline=$(( SECONDS + CEILING_SECS )) | |
| saw_vram=0 | |
| while [ "$SECONDS" -lt "$deadline" ]; do | |
| out=$(timeout 15 rocm-smi --showmeminfo vram 2>/dev/null) || { sleep 5; continue; } | |
| total=$(printf '%s\n' "$out" | grep -i 'VRAM Total Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) | |
| used=$(printf '%s\n' "$out" | grep -i 'VRAM Total Used Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) | |
| if [ -z "$total" ] || [ -z "$used" ]; then | |
| sleep 5; continue | |
| fi | |
| saw_vram=1 | |
| free=$(( total - used )) | |
| if [ "$free" -ge "$min_free" ]; then | |
| echo "GPU ready: $(( free / 1024 / 1024 / 1024 )) GiB free (>= ${MIN_FREE_GIB} GiB)." | |
| exit 0 | |
| fi | |
| echo "waiting: $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB)…" | |
| sleep 5 | |
| done | |
| if [ "$saw_vram" -eq 0 ]; then | |
| echo "::warning::rocm-smi reported no VRAM figures under WSL — GPU scenarios will resolve to skip" | |
| exit 0 | |
| fi | |
| echo "::error::GPU preflight failed after ${CEILING_SECS}s: VRAM never dropped below the floor — a serve is likely still holding the GPU" | |
| exit 1 | |
| # Bootstrap rustup with --no-modify-path so it never writes $HOME/.profile | |
| # (setup-rust-toolchain doesn't expose that flag). rust-toolchain.toml pins | |
| # the exact toolchain, installed on first cargo use. Idempotent. | |
| - name: Ensure Rust toolchain | |
| run: | | |
| if ! command -v cargo >/dev/null 2>&1 && [ ! -x "$HOME/.cargo/bin/cargo" ]; then | |
| curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \ | |
| | sh -s -- -y --no-modify-path --default-toolchain none | |
| fi | |
| echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" | |
| - name: Run E2E tests on Strix Halo WSL2 (incl. nightly-only) | |
| run: | | |
| export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" | |
| export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" | |
| # Share ONE installed managed runtime across serve/chat scenarios and | |
| # PRE-WARM it in place (mirrors e2e-gpu-nightly-strix). Required for | |
| # correctness, not speed: `install sdk` bakes ABSOLUTE paths into the | |
| # runtime manifest, so installing into a per-scenario temp dir leaves | |
| # every later serve pointing at a deleted install root. | |
| prewarm="$RUNNER_WORKSPACE/e2e-prewarm" | |
| export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" | |
| # See the e2e-gpu lane in e2e-selfhosted.yml for why the | |
| # e2e-test-hooks feature must match what `cargo xtask e2e` would build. | |
| cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks | |
| export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" | |
| export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" | |
| # Cache, not a one-shot: refreshed when the channel index has a newer | |
| # version. See `xtask e2e-prewarm` (EAI-8057). | |
| HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ | |
| cargo xtask e2e-prewarm --channel release --prewarm-dir "$prewarm" | |
| cargo xtask e2e | |
| - name: Upload E2E report | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: e2e-gpu-strix-wsl-report | |
| path: tests/e2e-cucumber/results/ | |
| # The nightly lanes each uploaded a report nobody joined up, so reading a | |
| # nightly meant opening every artifact and comparing them by hand — on the | |
| # run that covers the most scenarios, since only nightly sets | |
| # E2E_INCLUDE_NIGHTLY. Same consolidation the per-PR self-hosted workflow | |
| # already does (e2e-selfhosted.yml), against the nightly artifacts. | |
| e2e-report-nightly: | |
| name: E2E consolidated report (nightly) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| # Narrower than the workflow-level `contents: write`, which exists for the | |
| # release-publishing jobs. This one checks out, downloads artifacts, runs an | |
| # xtask and uploads; it has no business being able to write to the repo. | |
| permissions: | |
| contents: read | |
| needs: | |
| - e2e-gpu-nightly | |
| - e2e-gpu-nightly-rad3 | |
| - e2e-gpu-nightly-strix | |
| - e2e-gpu-nightly-strix-windows | |
| - e2e-wsl-nightly | |
| # Unconditional `always()`, unlike the per-PR workflow: there is no paths | |
| # filter to consult on a schedule, and the lanes are `continue-on-error`, so | |
| # a failed lane must still be reported. A nightly that broke is precisely | |
| # when the grid is worth reading. | |
| if: always() | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 | |
| # `*-report` matches the artifacts above, and will pick up any | |
| # nightly platform added later without a change here — provided it is | |
| # named canonically (see the rename note on the upload steps). | |
| # Layout note (same as the per-PR job): download-artifact@v8 uses a | |
| # per-artifact subdir when several match but flattens into the root when | |
| # exactly one does, which is what a partially-failed nightly can produce; | |
| # `discover()` handles both, labeling a root-level report from its | |
| # platform.json slug. | |
| - name: Download all nightly E2E reports | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| pattern: '*-report' | |
| path: e2e-artifacts | |
| - name: Build consolidated report + step summary | |
| run: | | |
| mkdir -p consolidated | |
| cargo xtask e2e-report \ | |
| --artifacts-dir e2e-artifacts \ | |
| --html-out consolidated/index.html >> "$GITHUB_STEP_SUMMARY" | |
| - name: Upload consolidated report | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: e2e-consolidated-report-nightly | |
| path: consolidated/ |