ci(rocm): add ROCm CI workflow for MI300X self-hosted runners

.github/workflows/_run-ci-rocm.yml

@@ -0,0 +1,175 @@
+name: CI Job (ROCm)
+
+on:
+  workflow_call:
+    inputs:
+      execute_command:
+        type: string
+        required: true
+      runs_on:
+        type: string
+        required: true
+        description: 'JSON-encoded list of runner labels (all required). e.g. ''["amd", "mi300x", "4gpu"]'' picks runners that carry ALL the listed labels.'
+      container_image:
+        # No default: pr-test-rocm.yml's resolve-ci-image is the single
+        # source of the image name, so a dated tag can't rot here.
+        type: string
+        required: true
+      checkout_ref:
+        description: Git ref to test. PR callers pass the merge ref explicitly.
+        type: string
+        required: true
+      skip_dependency_install:
+        # Default true: the ROCm image ships SGLang / Megatron-LM baked in.
+        # Pass false to fetch and install refs (PR body ci-megatron-pr: /
+        # ci-sglang-pr:, else miles-main / sglang-miles).
+        type: boolean
+        required: false
+        default: true
+    secrets:
+      WANDB_API_KEY:
+        required: false
+
+jobs:
+  run:
+    # fromJSON parses the input list so `runs-on` receives an actual array.
+    # Without this, GitHub treats the string verbatim as one label literal
+    # (e.g. the label name '["amd", "mi300x", "4gpu"]' with brackets), which no
+    # runner carries, and the job queues forever.
+    runs-on: ${{ fromJSON(inputs.runs_on) }}
+    container:
+      image: ${{ inputs.container_image }}
+      options: >
+        --device=/dev/kfd
+        --device=/dev/dri
+        --env HIP_VISIBLE_DEVICES
+        --security-opt seccomp=unconfined
+        --group-add video
+        --group-add render
+        --cap-add=SYS_PTRACE
+        --ipc=host
+        --shm-size=32g
+        --ulimit memlock=-1
+        --ulimit stack=67108864
+        --memory=0
+        --memory-swap=0
+        -v /data/miles_ci:/data/miles_ci
+        -v /data/miles_ci/models:/root/models
+        -v /data/miles_ci/datasets:/root/datasets
+        -v /data/miles_ci/hf_cache:/root/.cache/huggingface
+        --privileged
+        --ulimit nofile=65535:65535
+    defaults:
+      run:
+        working-directory: ${{ github.workspace }}
+    env:
+      PYTHONPATH: ${{ github.workspace }}
+      GITHUB_COMMIT_NAME: ${{ github.event.pull_request.head.sha || github.sha }}_${{ github.event.pull_request.number || 'non-pr' }}
+      WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.checkout_ref }}
+          persist-credentials: false
+
+      - name: Cleanup Ray processes
+        shell: bash
+        run: |
+          pkill -9 -f 'ray::' 2>/dev/null || true
+          pkill -9 -f raylet 2>/dev/null || true
+          pkill -9 -f gcs_server 2>/dev/null || true
+          pkill -9 -f 'ray-dashboard' 2>/dev/null || true
+          pkill -9 sglang 2>/dev/null || true
+          ray stop --force 2>/dev/null || true
+          sleep 3
+
+      - name: Wait for GPU ready
+        shell: bash
+        run: |
+          # When a self-hosted runner accepts a new job seconds after the
+          # previous job's container exits, the AMD GPU driver subsystem may
+          # still be tearing down its prior context. Block here until
+          # rocm-smi succeeds, with a bounded retry budget so a
+          # genuinely-broken GPU fails fast.
+          for i in $(seq 1 60); do
+            if rocm-smi --showproductname >/dev/null 2>&1; then
+              echo "GPU ready after $((i-1)) retry(s)"
+              rocm-smi --showproductname 2>&1 | head -20 || true
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "ERROR: rocm-smi did not become ready after 120s" >&2
+          rocm-smi 2>&1 | tail -20 || true
+          exit 1
+
+      - name: Resolve dependency refs
+        if: ${{ !inputs.skip_dependency_install }}
+        id: resolve-refs
+        shell: bash
+        env:
+          PR_BODY: ${{ github.event.pull_request.body || '' }}
+          INPUT_MEGATRON_PR: ${{ github.event.inputs.ci_megatron_pr || '' }}
+          INPUT_SGLANG_PR: ${{ github.event.inputs.ci_sglang_pr || '' }}
+        run: |
+          MEGATRON_PR="${INPUT_MEGATRON_PR}"
+          SGLANG_PR="${INPUT_SGLANG_PR}"
+          if [ -n "$PR_BODY" ]; then
+            PR_MEGATRON_PR=$(echo "$PR_BODY" | grep -m1 -oP '^ci-megatron-pr:\s+\K\S+' || true)
+            PR_SGLANG_PR=$(echo "$PR_BODY" | grep -m1 -oP '^ci-sglang-pr:\s+\K\S+' || true)
+            [ -z "$MEGATRON_PR" ] && [ -n "$PR_MEGATRON_PR" ] && MEGATRON_PR="$PR_MEGATRON_PR"
+            [ -z "$SGLANG_PR" ] && [ -n "$PR_SGLANG_PR" ] && SGLANG_PR="$PR_SGLANG_PR"
+          fi
+          [ -z "$MEGATRON_PR" ] && MEGATRON_PR="miles-main"
+          [ -z "$SGLANG_PR" ] && SGLANG_PR="sglang-miles"
+          resolve_fetch_ref() {
+            local ref="$1"
+            if [[ "$ref" =~ ^#([0-9]+)$ ]]; then
+              echo "refs/pull/${BASH_REMATCH[1]}/head"
+            else
+              echo "$ref"
+            fi
+          }
+          MEGATRON_FETCH=$(resolve_fetch_ref "$MEGATRON_PR")
+          SGLANG_FETCH=$(resolve_fetch_ref "$SGLANG_PR")
+          echo "ci_megatron_pr=$MEGATRON_FETCH" >> $GITHUB_OUTPUT
+          echo "ci_sglang_pr=$SGLANG_FETCH" >> $GITHUB_OUTPUT
+          echo "Resolved: megatron=$MEGATRON_PR -> fetch=$MEGATRON_FETCH, sglang=$SGLANG_PR -> fetch=$SGLANG_FETCH"
+
+      - name: Install dependencies
+        if: ${{ !inputs.skip_dependency_install }}
+        shell: bash
+        env:
+          MEGATRON_PR: ${{ steps.resolve-refs.outputs.ci_megatron_pr }}
+          SGLANG_PR: ${{ steps.resolve-refs.outputs.ci_sglang_pr }}
+        run: |
+          if command -v rustup >/dev/null 2>&1; then
+            rustup default stable
+          fi
+          if ! command -v protoc >/dev/null 2>&1; then
+            apt-get update
+            apt-get install -y protobuf-compiler
+          fi
+          rustc --version
+          protoc --version
+          cd /sgl-workspace/sglang && git reset --hard HEAD && git clean -fd && git fetch origin "$SGLANG_PR" && git checkout -f FETCH_HEAD && git log --oneline -1 && pip install -e python --no-deps --break-system-packages
+          cd /root/Megatron-LM && git reset --hard HEAD && git clean -fd && git fetch origin "$MEGATRON_PR" && git checkout -f FETCH_HEAD && git log --oneline -1 && pip install -e . --no-deps --break-system-packages
+          cd $GITHUB_WORKSPACE && pip install -e . --no-deps --break-system-packages
+          pip install polars zstandard xxhash blake3 --break-system-packages
+
+      - name: Install (miles only)
+        if: ${{ inputs.skip_dependency_install }}
+        shell: bash
+        run: |
+          cd $GITHUB_WORKSPACE && pip install -e . --no-deps --break-system-packages
+          pip install polars zstandard xxhash blake3 --break-system-packages
+
+      - name: Resolve suite plan
+        shell: bash
+        run: ${{ inputs.execute_command }} --list-only
+
+      - name: Run tests
+        shell: bash
+        run: ${{ inputs.execute_command }}

.github/workflows/pr-test-rocm.yml

@@ -0,0 +1,130 @@
+# doc-dev: docs/ci/00-stage.md
+name: PR Test (ROCm)
+
+# ROCm counterpart to pr-test.yml. Same structure, different GPU stage:
+# calls _run-ci-rocm.yml instead of _run-ci.yml, using the ROCm container
+# with HIP/CUDA compatibility.
+#
+# Test selection is explicit rather than inherited from the CUDA suites. The
+# shared CI policy selects matching register_rocm_ci() labels for PR/nightly
+# runs; manual dispatch keeps the full regular MI300X suite.
+#
+# The MI300X host is split into two 4-GPU runners, so this stage is 4-GPU.
+# CUDA tests that need 8 GPUs get a standalone test_amd_<name>.py with its
+# own 4-GPU CaseConfig rather than an IS_HIP branch in the original.
+#
+# pull_request_target keeps the workflow and fork authorization on the trusted
+# base branch. A fork reaches the privileged self-hosted stage only after a
+# maintainer adds a canonical `run-ci-*` label.
+#
+# The reusable job then checks out the PR merge ref explicitly. Fork PR jobs
+# receive no repository secrets; the accepted label still authorizes their
+# code to run against the MI300X host mounts.
+#
+# SGLang / Megatron-LM versions are baked into the ROCm image
+# (_run-ci-rocm.yml defaults skip_dependency_install to true), so unlike
+# pr-test.yml there are no ci_megatron_pr / ci_sglang_pr inputs here.
+
+on:
+  pull_request_target:
+    types: [opened, synchronize, reopened, ready_for_review, labeled]
+  schedule:
+    # Same nightly cadence as pr-test.yml; ci_policy.py maps this exact cron.
+    - cron: '0 15 * * *'
+  workflow_dispatch:
+    inputs:
+      ci_image_tag:
+        description: 'ROCm Docker image tag (default: miles-rocm700-mi30x-20260708)'
+        required: false
+        type: string
+        default: 'miles-rocm700-mi30x-20260708'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.schedule || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  resolve-ci-policy:
+    runs-on: ubuntu-latest
+    outputs:
+      allow_self_hosted: ${{ steps.authorize.outputs.allow_self_hosted }}
+      cadence: ${{ steps.resolve.outputs.cadence }}
+      raw_labels: ${{ steps.resolve.outputs.raw_labels }}
+      bypass_fastfail: ${{ steps.resolve.outputs.bypass_fastfail }}
+    steps:
+      - name: Authorize self-hosted PR execution
+        id: authorize
+        shell: bash
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name || '' }}
+          BASE_REPOSITORY: ${{ github.repository }}
+          PR_LABELS_JSON: ${{ toJSON(github.event.pull_request.labels.*.name) }}
+        run: |
+          ALLOW_SELF_HOSTED=true
+          if [[ "$EVENT_NAME" == "pull_request_target" && "$HEAD_REPOSITORY" != "$BASE_REPOSITORY" ]]; then
+            ALLOW_SELF_HOSTED=false
+            if jq -e 'any(.[]; test("^run-ci-[A-Za-z0-9][A-Za-z0-9_.-]*$"))' <<<"$PR_LABELS_JSON" >/dev/null; then
+              ALLOW_SELF_HOSTED=true
+            fi
+          fi
+          echo "allow_self_hosted=${ALLOW_SELF_HOSTED}" >> "$GITHUB_OUTPUT"
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Resolve CI policy inputs
+        id: resolve
+        env:
+          EVENT_NAME: ${{ github.event_name == 'pull_request_target' && 'pull_request' || github.event_name }}
+          SCHEDULE: ${{ github.event.schedule || '' }}
+          PR_LABELS_JSON: ${{ toJSON(github.event.pull_request.labels.*.name) }}
+        run: python -m tests.ci.ci_policy
+
+  resolve-ci-image:
+    runs-on: ubuntu-latest
+    outputs:
+      container_image: ${{ steps.resolve.outputs.container_image }}
+    steps:
+      - name: Resolve CI image
+        id: resolve
+        shell: bash
+        env:
+          INPUT_CI_IMAGE_TAG: ${{ github.event.inputs.ci_image_tag || '' }}
+        run: |
+          CI_IMAGE_TAG="${INPUT_CI_IMAGE_TAG}"
+          [ -z "$CI_IMAGE_TAG" ] && CI_IMAGE_TAG="miles-rocm700-mi30x-20260708"
+          if [[ ! "$CI_IMAGE_TAG" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]]; then
+            echo "::error::ci-image-tag must be a Docker tag, not a full image name: $CI_IMAGE_TAG"
+            exit 1
+          fi
+          echo "container_image=rocm/sgl-dev:${CI_IMAGE_TAG}" >> "$GITHUB_OUTPUT"
+          echo "Resolved CI image: rocm/sgl-dev:${CI_IMAGE_TAG}"
+
+  stage-c-4-gpu-mi300x:
+    needs: [resolve-ci-policy, resolve-ci-image]
+    if: needs.resolve-ci-policy.outputs.allow_self_hosted == 'true'
+    # One shard per 4-GPU runner, so the pair works the suite in parallel.
+    # run_suite.py balances the shards by est_time. fail-fast: false so one
+    # shard's failure does not cancel the other mid-run.
+    strategy:
+      fail-fast: false
+      matrix:
+        partition_id: [0, 1]
+    uses: ./.github/workflows/_run-ci-rocm.yml
+    with:
+      runs_on: '["self-hosted", "amd", "mi300x", "4gpu"]'
+      container_image: ${{ needs.resolve-ci-image.outputs.container_image }}
+      checkout_ref: ${{ github.event_name == 'pull_request_target' && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
+      execute_command: >-
+        python tests/ci/run_suite.py --hw rocm --suite stage-c-4-gpu-mi300x
+        --auto-partition-id ${{ matrix.partition_id }} --auto-partition-size 2
+        --cadence ${{ needs.resolve-ci-policy.outputs.cadence }}
+        --labels ${{ needs.resolve-ci-policy.outputs.raw_labels }}
+        ${{ github.event_name == 'workflow_dispatch' && '--match-all-labels' || '' }}
+        --enable-retry --max-attempts 2
+    secrets:
+      WANDB_API_KEY: ${{ (github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name == github.repository) && secrets.WANDB_API_KEY || '' }}