Nightly #49
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly | |
| on: | |
| schedule: | |
| - cron: "0 6 * * *" | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| env: | |
| CARGO_TERM_COLOR: always | |
| # Production trust gate — dormant until the owner sets the repository variable | |
| # ROCM_CLI_REQUIRE_PRODUCTION_TRUST (e.g. "1") and provides the trust inputs | |
| # below. While the variable is unset/falsy, release_readiness.py skips | |
| # validate_production_trust() and behavior is unchanged. The values are public | |
| # keys and paths (not the private signing key, which stays step-scoped). When | |
| # enabling the gate, the recipe-index inputs also require a step that | |
| # materializes the hosted index + its .sig into the workspace. | |
| ROCM_CLI_REQUIRE_PRODUCTION_TRUST: ${{ vars.ROCM_CLI_REQUIRE_PRODUCTION_TRUST }} | |
| ROCM_CLI_SIGNING_PUBLIC_KEY_PEM: ${{ secrets.ROCM_CLI_SIGNING_PUBLIC_KEY_PEM }} | |
| ROCM_CLI_METADATA_PUBLIC_KEY_PEM: ${{ secrets.ROCM_CLI_METADATA_PUBLIC_KEY_PEM }} | |
| ROCM_CLI_MODEL_RECIPE_INDEX_PATH: ${{ vars.ROCM_CLI_MODEL_RECIPE_INDEX_PATH }} | |
| ROCM_CLI_MODEL_RECIPE_INDEX_PUBLIC_KEY_PATH: ${{ vars.ROCM_CLI_MODEL_RECIPE_INDEX_PUBLIC_KEY_PATH }} | |
| jobs: | |
| nightly: | |
| name: Build Linux nightly assets | |
| runs-on: ubuntu-latest | |
| outputs: | |
| skip: ${{ steps.check.outputs.skip }} | |
| date: ${{ steps.meta.outputs.date }} | |
| short_sha: ${{ steps.meta.outputs.short_sha }} | |
| staging_tag: ${{ steps.staging.outputs.tag }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 | |
| - name: Verify pinned keys match the signing key | |
| run: cargo xtask verify-pinned-keys | |
| - name: Check for changes since last nightly | |
| id: check | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| LAST=$(gh release view nightly --json createdAt --jq .createdAt 2>/dev/null || echo "1970-01-01T00:00:00Z") | |
| COMMITS=$(git log --since="${LAST}" --oneline | wc -l) | |
| echo "commits=${COMMITS}" >> "$GITHUB_OUTPUT" | |
| if [ "${COMMITS}" -eq 0 ] && [ "${{ github.event_name }}" != "workflow_dispatch" ]; then | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| echo "No new commits since last nightly — skipping" | |
| else | |
| echo "skip=false" >> "$GITHUB_OUTPUT" | |
| echo "${COMMITS} new commit(s) since last nightly" | |
| fi | |
| - name: Cache Cargo home and build artifacts | |
| if: steps.check.outputs.skip != 'true' | |
| uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| path: | | |
| ~/.cargo/bin | |
| ~/.cargo/registry/cache | |
| ~/.cargo/registry/index | |
| ~/.cargo/registry/src | |
| ~/.cargo/rustup | |
| target | |
| key: Linux-manylinux-release-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }} | |
| restore-keys: | | |
| Linux-manylinux-release- | |
| - name: Build release binaries | |
| if: steps.check.outputs.skip != 'true' | |
| run: | | |
| # The bind-mount source must exist and be owned by the runner user; | |
| # Docker would otherwise create it as root, leaving the non-root | |
| # container user unable to write into CARGO_HOME/RUSTUP_HOME. | |
| mkdir -p "$HOME/.cargo" | |
| # Install system build deps as root: yum needs to write to | |
| # /var/cache/yum and the RPM db, which a non-root container user | |
| # cannot do. These land in the container's ephemeral layer (not the | |
| # bind mount), so this runs every time; only the Cargo/rustup state | |
| # below is cached. | |
| docker run --rm \ | |
| -v "$PWD":/work \ | |
| -w /work \ | |
| quay.io/pypa/manylinux2014_x86_64 \ | |
| /bin/bash -lc ' | |
| set -euo pipefail | |
| yum install -y pkgconfig libcap-devel | |
| ' | |
| # Build as the runner user so cached Cargo/rustup files are owned by | |
| # the runner (the host-side cache save step must be able to read them). | |
| docker run --rm \ | |
| -u "$(id -u):$(id -g)" \ | |
| -e CARGO_HOME=/host-cargo \ | |
| -e RUSTUP_HOME=/host-cargo/rustup \ | |
| -v "$HOME/.cargo":/host-cargo \ | |
| -v "$PWD":/work \ | |
| -w /work \ | |
| quay.io/pypa/manylinux2014_x86_64 \ | |
| /bin/bash -lc ' | |
| set -euo pipefail | |
| if [ ! -f /host-cargo/bin/cargo ]; then | |
| curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain none --no-modify-path | |
| fi | |
| export PATH="/host-cargo/bin:$PATH" | |
| cargo build --release -p rocm -p rocmd -p xtask | |
| ' | |
| - name: Set nightly metadata | |
| id: meta | |
| if: steps.check.outputs.skip != 'true' | |
| run: | | |
| echo "date=$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT" | |
| echo "short_sha=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" | |
| - name: Package nightly bundle | |
| if: steps.check.outputs.skip != 'true' | |
| env: | |
| ROCM_CLI_SIGNING_PRIVATE_KEY_PEM: ${{ secrets.ROCM_CLI_SIGNING_PRIVATE_KEY_PEM }} | |
| ROCM_CLI_REQUIRE_SIGNATURE: "1" | |
| run: | | |
| DATE="${{ steps.meta.outputs.date }}" | |
| SHORT_SHA="${{ steps.meta.outputs.short_sha }}" | |
| DIST="rocm-cli-nightly-${DATE}-${SHORT_SHA}-linux-amd64" | |
| chmod +x install.sh scripts/package-linux-release.sh | |
| ./scripts/package-linux-release.sh "${DIST}" | |
| cp "dist/${DIST}.tar.gz" dist/rocm-cli-nightly-linux-amd64.tar.gz | |
| (cd dist && sha256sum rocm-cli-nightly-linux-amd64.tar.gz > rocm-cli-nightly-linux-amd64.tar.gz.sha256) | |
| if [ -f "dist/${DIST}.tar.gz.sig" ]; then | |
| cp "dist/${DIST}.tar.gz.sig" dist/rocm-cli-nightly-linux-amd64.tar.gz.sig | |
| else | |
| echo "nightly signature was not produced" >&2 | |
| exit 1 | |
| fi | |
| python scripts/release_readiness.py \ | |
| --dist dist \ | |
| --require-signatures \ | |
| --require-rocm-asset-names \ | |
| --require-exact-assets \ | |
| --asset "${DIST}.tar.gz" \ | |
| --asset rocm-cli-nightly-linux-amd64.tar.gz | |
| echo "DIST=${DIST}" >> "$GITHUB_ENV" | |
| echo "DATE=${DATE}" >> "$GITHUB_ENV" | |
| echo "SHORT_SHA=${SHORT_SHA}" >> "$GITHUB_ENV" | |
| - name: Create staging nightly release | |
| id: staging | |
| if: steps.check.outputs.skip != 'true' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| STAGING_TAG="nightly-staging-${DATE}-${SHORT_SHA}" | |
| gh release delete "${STAGING_TAG}" --yes --cleanup-tag 2>/dev/null || true | |
| gh release create "${STAGING_TAG}" \ | |
| --draft \ | |
| --title "rocm-cli nightly (${DATE} · ${SHORT_SHA})" \ | |
| --notes "$(cat <<EOF | |
| Automated nightly build from \`main\` at $(date -u +"%Y-%m-%d %H:%M UTC"). | |
| Commit: [\`${SHORT_SHA}\`](https://github.com/${{ github.repository }}/commit/$(git rev-parse HEAD)) | |
| ${{ steps.check.outputs.commits }} commit(s) since the previous nightly. | |
| Install the latest nightly on Linux x86_64: | |
| \`\`\`bash | |
| curl -fsSL https://raw.githubusercontent.com/${{ github.repository }}/main/install.sh | sh -s -- nightly | |
| \`\`\` | |
| Install the latest nightly on Windows x86_64 from PowerShell: | |
| \`\`\`powershell | |
| \$script = "\$env:TEMP\install-rocm-cli.ps1"; irm https://raw.githubusercontent.com/${{ github.repository }}/main/install.ps1 -OutFile \$script; powershell -ExecutionPolicy Bypass -File \$script nightly | |
| \`\`\` | |
| EOF | |
| )" \ | |
| --prerelease \ | |
| dist/*.tar.gz dist/*.tar.gz.sha256 dist/*.sig | |
| echo "tag=${STAGING_TAG}" >> "$GITHUB_OUTPUT" | |
| windows-nightly: | |
| name: Build and upload Windows nightly | |
| runs-on: windows-latest | |
| needs: nightly | |
| if: needs.nightly.outputs.skip != 'true' | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 | |
| - name: Verify pinned keys match the signing key | |
| shell: pwsh | |
| run: cargo xtask verify-pinned-keys | |
| - name: Build release binaries | |
| shell: pwsh | |
| run: cargo build --release -p rocm -p rocmd -p xtask | |
| - name: Package Windows bundle | |
| shell: pwsh | |
| env: | |
| ROCM_CLI_SIGNING_PRIVATE_KEY_PEM: ${{ secrets.ROCM_CLI_SIGNING_PRIVATE_KEY_PEM }} | |
| ROCM_CLI_REQUIRE_SIGNATURE: "1" | |
| run: | | |
| $date = "${{ needs.nightly.outputs.date }}" | |
| $shortSha = "${{ needs.nightly.outputs.short_sha }}" | |
| $dist = "rocm-cli-nightly-$date-$shortSha-windows-amd64" | |
| .\scripts\package-windows-release.ps1 $dist | |
| Copy-Item "dist\$dist.zip" "dist\rocm-cli-nightly-windows-amd64.zip" | |
| $hash = (Get-FileHash -Algorithm SHA256 -LiteralPath "dist\rocm-cli-nightly-windows-amd64.zip").Hash.ToLowerInvariant() | |
| Set-Content -LiteralPath "dist\rocm-cli-nightly-windows-amd64.zip.sha256" -Value "$hash rocm-cli-nightly-windows-amd64.zip" -Encoding ascii | |
| if (Test-Path "dist\$dist.zip.sig") { | |
| Copy-Item "dist\$dist.zip.sig" "dist\rocm-cli-nightly-windows-amd64.zip.sig" | |
| } else { | |
| throw "nightly signature was not produced" | |
| } | |
| python .\scripts\release_readiness.py ` | |
| --dist dist ` | |
| --require-signatures ` | |
| --require-rocm-asset-names ` | |
| --require-exact-assets ` | |
| --asset "$dist.zip" ` | |
| --asset rocm-cli-nightly-windows-amd64.zip | |
| - name: Upload Windows nightly assets | |
| shell: pwsh | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| $stagingTag = "${{ needs.nightly.outputs.staging_tag }}" | |
| if (-not $stagingTag) { | |
| throw "nightly staging release tag was not produced" | |
| } | |
| if (-not (Get-ChildItem dist -File -Include *.sig -Recurse)) { | |
| throw "nightly signatures are required" | |
| } | |
| $assets = Get-ChildItem dist -File -Include *.zip,*.sha256,*.sig -Recurse | ForEach-Object { $_.FullName } | |
| gh release upload $stagingTag @assets --clobber | |
| publish-nightly: | |
| name: Publish nightly | |
| runs-on: ubuntu-latest | |
| needs: | |
| - nightly | |
| - windows-nightly | |
| if: needs.nightly.outputs.skip != 'true' | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Publish staged nightly | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GH_REPO: ${{ github.repository }} | |
| run: | | |
| STAGING_TAG="${{ needs.nightly.outputs.staging_tag }}" | |
| DATE="${{ needs.nightly.outputs.date }}" | |
| SHORT_SHA="${{ needs.nightly.outputs.short_sha }}" | |
| ASSET_DIR="$(mktemp -d)" | |
| if [ -z "${STAGING_TAG}" ]; then | |
| echo "nightly staging release tag was not produced" >&2 | |
| exit 1 | |
| fi | |
| gh release download "${STAGING_TAG}" --dir "${ASSET_DIR}" | |
| python scripts/release_readiness.py \ | |
| --dist "${ASSET_DIR}" \ | |
| --require-signatures \ | |
| --require-rocm-asset-names \ | |
| --require-exact-assets \ | |
| --asset "rocm-cli-nightly-${DATE}-${SHORT_SHA}-linux-amd64.tar.gz" \ | |
| --asset rocm-cli-nightly-linux-amd64.tar.gz \ | |
| --asset "rocm-cli-nightly-${DATE}-${SHORT_SHA}-windows-amd64.zip" \ | |
| --asset rocm-cli-nightly-windows-amd64.zip | |
| TITLE="$(gh release view "${STAGING_TAG}" --json name -q .name)" | |
| NOTES="$(gh release view "${STAGING_TAG}" --json body -q .body)" | |
| gh release delete nightly --yes --cleanup-tag 2>/dev/null || true | |
| gh release create nightly \ | |
| --title "${TITLE}" \ | |
| --notes "${NOTES}" \ | |
| --prerelease \ | |
| "${ASSET_DIR}"/* | |
| # Convenience assets for quick curl-and-run, attached after the | |
| # readiness gate (NOT part of the signed exact-asset set): | |
| # rocm / rocm.exe - standalone CLI binaries | |
| # linux-binaries.tar.gz - byte-identical alias of the signed | |
| # rocm-cli-nightly-linux-amd64.tar.gz bundle | |
| # windows-binaries.zip - byte-identical alias of the signed | |
| # rocm-cli-nightly-windows-amd64.zip bundle | |
| # Integrity-conscious users should prefer the signed *-amd64 archives. | |
| EXTRA="$(mktemp -d)" | |
| mkdir -p "${EXTRA}/lin" "${EXTRA}/win" | |
| tar -xzf "${ASSET_DIR}/rocm-cli-nightly-linux-amd64.tar.gz" -C "${EXTRA}/lin" | |
| unzip -o -q "${ASSET_DIR}/rocm-cli-nightly-windows-amd64.zip" -d "${EXTRA}/win" | |
| cp "$(find "${EXTRA}/lin" -type f -name rocm)" "${EXTRA}/rocm" | |
| cp "$(find "${EXTRA}/win" -type f -name rocm.exe)" "${EXTRA}/rocm.exe" | |
| cp "${ASSET_DIR}/rocm-cli-nightly-linux-amd64.tar.gz" "${EXTRA}/linux-binaries.tar.gz" | |
| cp "${ASSET_DIR}/rocm-cli-nightly-windows-amd64.zip" "${EXTRA}/windows-binaries.zip" | |
| gh release upload nightly \ | |
| "${EXTRA}/rocm" "${EXTRA}/rocm.exe" \ | |
| "${EXTRA}/linux-binaries.tar.gz" "${EXTRA}/windows-binaries.zip" --clobber | |
| # The staging release is a draft, which never creates a git tag, so | |
| # --cleanup-tag would 422 on a nonexistent ref. Just delete the draft. | |
| gh release delete "${STAGING_TAG}" --yes 2>/dev/null || true | |
| cleanup-staging-nightly: | |
| name: Clean failed nightly staging release | |
| runs-on: ubuntu-latest | |
| needs: | |
| - nightly | |
| - windows-nightly | |
| if: always() && needs.nightly.outputs.skip != 'true' && needs.windows-nightly.result != 'success' && needs.nightly.outputs.staging_tag != '' | |
| steps: | |
| - name: Delete staging release | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GH_REPO: ${{ github.repository }} | |
| run: gh release delete "${{ needs.nightly.outputs.staging_tag }}" --yes --cleanup-tag 2>/dev/null || true | |
| # Full E2E on app-dev MI300X INCLUDING the expensive `@nightly` scenarios (the | |
| # large-model serve), which the per-PR ci.yml `e2e-gpu` job skips to stay fast. | |
| # Runs unconditionally on the schedule/dispatch (independent of the nightly | |
| # BUILD skip above — a regression run is worth doing even with no new commits). | |
| # Non-blocking, mirrors ci.yml's e2e-gpu job but sets E2E_INCLUDE_NIGHTLY. | |
| e2e-gpu-nightly: | |
| name: E2E tests (GPU, incl. nightly-only) | |
| timeout-minutes: 90 | |
| runs-on: [self-hosted, linux, amd-gpu] | |
| continue-on-error: true | |
| env: | |
| # Include @nightly scenarios (the large-model serve). See src/expectation.rs. | |
| E2E_INCLUDE_NIGHTLY: "1" | |
| # Serve-readiness floor; a `@serve-timeout` tag lengthens it for the large | |
| # model (its cold ~54 GiB load exceeds this default). | |
| E2E_SERVE_TIMEOUT_SECS: "300" | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| # Reclaim the GPU before running: kill only e2e leftovers scoped to | |
| # /tmp/rocm-e2e-* and the e2e-target/e2e-shared trees — never the runner or | |
| # any /workload manual-testing processes (see ci.yml e2e-gpu for the same). | |
| - name: Reclaim GPU from stray E2E processes | |
| run: | | |
| pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true | |
| pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true | |
| pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true | |
| pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true | |
| pkill -f 'e2e-target/release/rocm daemon' 2>/dev/null || true | |
| rm -rf /tmp/rocm-e2e-* 2>/dev/null || true | |
| echo "reclaimed" | |
| # cache: false — this self-hosted runner persists the build cache itself via | |
| # CARGO_TARGET_DIR (below), so the action's rust-cache is pure overhead. | |
| - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 | |
| with: | |
| cache: false | |
| - name: Run full E2E on GPU hardware (incl. nightly-only) | |
| run: | | |
| export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" | |
| export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" | |
| # Share uv's wheel cache off the near-full PVC, on the roomy `/` overlay | |
| # (see ci.yml e2e-gpu): warm `rocm install sdk` ~34s vs cold ~160s. | |
| export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache" | |
| # Share ONE installed managed runtime across serve/chat scenarios so | |
| # install sdk runs once per runner, not per scenario (see ci.yml e2e-gpu). | |
| # The shared dir IS the pre-warm's own data/runtimes; NEVER move it — a | |
| # post-install mv invalidates the absolute paths install sdk bakes into | |
| # the runtime manifest and every serve fails instantly (run 29320025393). | |
| prewarm="$RUNNER_WORKSPACE/e2e-prewarm" | |
| export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" | |
| # Build once; reuse for pre-warm + suite so xtask doesn't rebuild. | |
| cargo build --release -p rocm | |
| export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" | |
| # Pre-warm the shared runtime once/serially before the suite; skipped | |
| # once populated (persists across runs). Install in place (no mv) so the | |
| # manifest's absolute paths stay valid. See ci.yml e2e-gpu for rationale. | |
| if [ ! -d "$E2E_SHARED_RUNTIMES_DIR/registry" ]; then | |
| echo "pre-warming shared runtime (first run on this runner)…" | |
| mkdir -p "$prewarm"/{data,config,cache} | |
| ROCM_CLI_CONFIG_DIR="$prewarm/config" \ | |
| ROCM_CLI_DATA_DIR="$prewarm/data" \ | |
| ROCM_CLI_CACHE_DIR="$prewarm/cache" \ | |
| HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ | |
| UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ | |
| "$ROCM_CLI_BINARY" install sdk | |
| fi | |
| cargo xtask e2e | |
| - name: Upload E2E report | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: e2e-gpu-nightly-report | |
| path: tests/e2e-cucumber/results/ | |
| # Strix Halo counterpart to e2e-gpu-nightly. The same @nightly large-model | |
| # scenario serves the platform-specific Lemonade GGUF on this host. Keep the | |
| # runner's storage and runtime setup aligned with ci.yml's proven Strix Ubuntu job. | |
| e2e-gpu-nightly-strix: | |
| name: E2E tests (Strix Halo, incl. nightly-only) | |
| timeout-minutes: 90 | |
| runs-on: [self-hosted, linux, strix-halo] | |
| continue-on-error: true | |
| env: | |
| HOME: /home/ubuntu/actions-runner/e2e-home | |
| CARGO_HOME: /home/ubuntu/actions-runner/.cargo | |
| RUSTUP_HOME: /home/ubuntu/actions-runner/.rustup | |
| TMPDIR: /home/ubuntu/actions-runner/tmp | |
| PIP_CACHE_DIR: /home/ubuntu/actions-runner/pip-cache | |
| E2E_INCLUDE_NIGHTLY: "1" | |
| E2E_SERVE_TIMEOUT_SECS: "300" | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Prepare writable dirs on the nvme + reclaim GPU from stray E2E procs | |
| run: | | |
| mkdir -p "$HOME" "$TMPDIR" "$PIP_CACHE_DIR" | |
| pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true | |
| pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true | |
| pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true | |
| pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true | |
| rm -rf /tmp/rocm-e2e-* 2>/dev/null || true | |
| echo "prepared + reclaimed" | |
| - name: GPU preflight (bounded wait for an available GPU) | |
| run: | | |
| MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-8}" | |
| CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" | |
| min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) | |
| deadline=$(( SECONDS + CEILING_SECS )) | |
| reason="rocm-smi never returned within its timeout (driver wedged or GPU absent)" | |
| while [ "$SECONDS" -lt "$deadline" ]; do | |
| out=$(timeout 15 rocm-smi --showmeminfo vram 2>/dev/null) || { sleep 5; continue; } | |
| total=$(printf '%s\n' "$out" | grep -i 'VRAM Total Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) | |
| used=$(printf '%s\n' "$out" | grep -i 'VRAM Total Used Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) | |
| if [ -z "$total" ] || [ -z "$used" ]; then | |
| reason="rocm-smi returned no VRAM figures (no AMD GPU detected)" | |
| sleep 5; continue | |
| fi | |
| free=$(( total - used )) | |
| if [ "$free" -ge "$min_free" ]; then | |
| echo "GPU ready: $(( free / 1024 / 1024 / 1024 )) GiB free (>= ${MIN_FREE_GIB} GiB)." | |
| exit 0 | |
| fi | |
| reason="VRAM never dropped below the floor: only $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB) — a serve is likely still holding the GPU" | |
| echo "waiting: $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB)…" | |
| sleep 5 | |
| done | |
| echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" | |
| exit 1 | |
| - name: Ensure Rust toolchain | |
| run: | | |
| if ! command -v cargo >/dev/null 2>&1 && [ ! -x "$CARGO_HOME/bin/cargo" ]; then | |
| curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs \ | |
| | sh -s -- -y --no-modify-path --default-toolchain none | |
| fi | |
| echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" | |
| - name: Run E2E tests on Strix Halo (incl. nightly-only) | |
| run: | | |
| export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" | |
| export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" | |
| prewarm="$RUNNER_WORKSPACE/e2e-prewarm" | |
| export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" | |
| cargo build --release -p rocm | |
| export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" | |
| if [ ! -d "$E2E_SHARED_RUNTIMES_DIR/registry" ]; then | |
| echo "pre-warming shared runtime (first run on this runner)…" | |
| mkdir -p "$prewarm"/{data,config,cache} | |
| ROCM_CLI_CONFIG_DIR="$prewarm/config" \ | |
| ROCM_CLI_DATA_DIR="$prewarm/data" \ | |
| ROCM_CLI_CACHE_DIR="$prewarm/cache" \ | |
| HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ | |
| "$ROCM_CLI_BINARY" install sdk | |
| if [ -d "$E2E_SHARED_RUNTIMES_DIR/registry" ]; then | |
| echo "shared runtime pre-warmed at $E2E_SHARED_RUNTIMES_DIR" | |
| else | |
| echo "pre-warm did not produce a runtimes registry; scenarios will install their own" >&2 | |
| fi | |
| else | |
| echo "shared runtime already present at $E2E_SHARED_RUNTIMES_DIR — skipping pre-warm" | |
| fi | |
| cargo xtask e2e | |
| - name: Upload E2E report | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: e2e-gpu-nightly-strix-report | |
| path: tests/e2e-cucumber/results/ | |
| # Windows counterpart to e2e-gpu-nightly-strix. Keep the PowerShell-native | |
| # setup aligned with ci.yml's proven Strix Windows job while opting into the | |
| # same platform-adaptive @nightly large-model scenario as the Linux runner. | |
| e2e-gpu-nightly-strix-windows: | |
| name: E2E tests (Strix Halo, Windows, incl. nightly-only) | |
| timeout-minutes: 90 | |
| runs-on: [self-hosted, windows, strix-halo] | |
| continue-on-error: true | |
| env: | |
| E2E_INCLUDE_NIGHTLY: "1" | |
| E2E_SERVE_TIMEOUT_SECS: "300" | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Reclaim GPU from stray E2E processes | |
| shell: powershell | |
| run: | | |
| Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | | |
| Where-Object { $_.CommandLine -match 'rocm-e2e|__engine-serve-http|e2e-target|e2e-shared' -and $_.CommandLine -match 'llama-server|vllm|rocm ' } | | |
| ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue } | |
| Write-Host "reclaimed" | |
| # Best-effort when rocm-smi is absent; when present, fail within the bounded | |
| # ceiling instead of letting a held or wedged GPU consume the full job cap. | |
| - name: GPU preflight (bounded wait for an available GPU) | |
| shell: powershell | |
| run: | | |
| $minFreeGiB = if ($env:GPU_PREFLIGHT_MIN_FREE_GIB) { [int]$env:GPU_PREFLIGHT_MIN_FREE_GIB } else { 8 } | |
| $ceilingSecs = if ($env:GPU_PREFLIGHT_CEILING_SECS) { [int]$env:GPU_PREFLIGHT_CEILING_SECS } else { 90 } | |
| if (-not (Get-Command rocm-smi -ErrorAction SilentlyContinue)) { | |
| Write-Host "rocm-smi not found on this Windows runner; skipping GPU preflight (best-effort)." | |
| exit 0 | |
| } | |
| $minFree = [int64]$minFreeGiB * 1GB | |
| $deadline = (Get-Date).AddSeconds($ceilingSecs) | |
| $reason = "rocm-smi never returned usable VRAM figures" | |
| while ((Get-Date) -lt $deadline) { | |
| $out = (rocm-smi --showmeminfo vram 2>$null | Out-String) | |
| $total = ([regex]::Matches($out, 'VRAM Total Memory \(B\):\s*(\d+)') | Select-Object -First 1).Groups[1].Value | |
| $used = ([regex]::Matches($out, 'VRAM Total Used Memory \(B\):\s*(\d+)') | Select-Object -First 1).Groups[1].Value | |
| if (-not $total -or -not $used) { | |
| $reason = "rocm-smi returned no VRAM figures (no AMD GPU detected)" | |
| Start-Sleep -Seconds 5; continue | |
| } | |
| $free = [int64]$total - [int64]$used | |
| if ($free -ge $minFree) { | |
| Write-Host "GPU ready: $([math]::Floor($free/1GB)) GiB free (>= $minFreeGiB GiB)." | |
| exit 0 | |
| } | |
| $reason = "VRAM never dropped below the floor: only $([math]::Floor($free/1GB)) GiB free (< $minFreeGiB GiB) - a serve is likely still holding the GPU" | |
| Write-Host "waiting: $([math]::Floor($free/1GB)) GiB free (< $minFreeGiB GiB)..." | |
| Start-Sleep -Seconds 5 | |
| } | |
| Write-Host "::error::GPU preflight failed after ${ceilingSecs}s: $reason" | |
| exit 1 | |
| # setup-rust-toolchain needs bash, which is unavailable on this runner. | |
| - name: Ensure Rust toolchain (PowerShell) | |
| shell: powershell | |
| run: | | |
| if (-not (Get-Command cargo -ErrorAction SilentlyContinue)) { | |
| Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile $env:TEMP\rustup-init.exe | |
| & $env:TEMP\rustup-init.exe -y --default-toolchain none | |
| "$env:USERPROFILE\.cargo\bin" | Out-File -FilePath $env:GITHUB_PATH -Append | |
| } | |
| - name: Run E2E tests on Strix Halo Windows (incl. nightly-only) | |
| shell: powershell | |
| run: | | |
| # Install the shared runtime in place because its manifest contains | |
| # absolute paths. The persistent runner workspace keeps those paths valid | |
| # across scenarios and subsequent runs. | |
| $prewarm = "$env:RUNNER_WORKSPACE\e2e-prewarm" | |
| $env:E2E_SHARED_RUNTIMES_DIR = "$prewarm\data\runtimes" | |
| cargo build --release -p rocm | |
| if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } | |
| $targetDir = if ($env:CARGO_TARGET_DIR) { $env:CARGO_TARGET_DIR } else { "target" } | |
| $env:ROCM_CLI_BINARY = "$targetDir\release\rocm.exe" | |
| if (-not (Test-Path "$env:E2E_SHARED_RUNTIMES_DIR\registry")) { | |
| Write-Host "pre-warming shared runtime (first run on this runner)..." | |
| New-Item -ItemType Directory -Force -Path "$prewarm\data","$prewarm\config","$prewarm\cache" | Out-Null | |
| $env:ROCM_CLI_CONFIG_DIR = "$prewarm\config" | |
| $env:ROCM_CLI_DATA_DIR = "$prewarm\data" | |
| $env:ROCM_CLI_CACHE_DIR = "$prewarm\cache" | |
| & $env:ROCM_CLI_BINARY install sdk | |
| Remove-Item Env:\ROCM_CLI_CONFIG_DIR,Env:\ROCM_CLI_DATA_DIR,Env:\ROCM_CLI_CACHE_DIR -ErrorAction SilentlyContinue | |
| if (Test-Path "$env:E2E_SHARED_RUNTIMES_DIR\registry") { | |
| Write-Host "shared runtime pre-warmed at $env:E2E_SHARED_RUNTIMES_DIR" | |
| } else { | |
| Write-Host "pre-warm did not produce a runtimes registry; scenarios will install their own" | |
| } | |
| } else { | |
| Write-Host "shared runtime already present at $env:E2E_SHARED_RUNTIMES_DIR - skipping pre-warm" | |
| } | |
| cargo xtask e2e | |
| - name: Upload E2E report | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: e2e-gpu-nightly-strix-windows-report | |
| path: tests/e2e-cucumber/results/ |