_run-ci.yml

# doc-dev: docs/developer/ci/00-stage.md
name: CI Job

on:
  workflow_call:
    inputs:
      execute_command:
        type: string
        required: true
      runs_on:
        type: string
        required: false
        description: 'JSON-encoded list of runner labels (all required). e.g. ''["h100", "8gpu"]'' picks runners that carry BOTH labels. No default: GPU jobs must pass this explicitly.'
      container_image:
        type: string
        required: false
        default: 'radixark/miles:dev'
      num_gpus:
        description: 'Exact CUDA-visible device count for a dynamically allocated B200 job.'
        type: number
        required: false
        default: 0
      timeout_minutes:
        description: 'Job budget, including GPU admission waiting, setup, execution, and cleanup.'
        type: number
        required: false
        default: 360
      skip_dependency_install:
        type: boolean
        required: false
        default: false
      ref:
        description: 'Git ref to check out (empty = the triggering commit). Set by release CI to test a release branch.'
        type: string
        required: false
        default: ''
      plan_already_resolved:
        description: 'Skip hosted planning when a trusted caller already selected an enabled test at an exact ref.'
        type: boolean
        required: false
        default: false
    secrets:
      WANDB_API_KEY:
        required: false
      NEON_DATABASE_URL:
        required: false

jobs:
  plan:
    if: ${{ !inputs.plan_already_resolved }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
    outputs:
      has_tests: ${{ steps.select.outputs.has_tests }}
      ref: ${{ steps.select.outputs.ref }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref }}
          persist-credentials: false
      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'
      - name: Resolve suite plan
        id: select
        env:
          PYTHONPATH: ${{ github.workspace }}
        run: |
          echo "ref=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
          ${{ inputs.execute_command }} --list-only --github-output "$GITHUB_OUTPUT"

  run:
    needs: plan
    if: |
      !cancelled() &&
      (inputs.plan_already_resolved ||
        (needs.plan.result == 'success' && needs.plan.outputs.has_tests == 'true'))
    # fromJSON parses the input list so `runs-on` receives an actual array.
    # Without this, GitHub treats the string verbatim as one label literal
    # (e.g. the label name '["h100", "8gpu"]' with brackets), which no
    # runner carries, and the job queues forever.
    runs-on: ${{ fromJSON(inputs.runs_on) }}
    timeout-minutes: ${{ inputs.timeout_minutes }}
    container:
      image: ${{ inputs.container_image }}
      options: >
        --gpus all
        --env CUDA_VISIBLE_DEVICES
        --ipc=host
        --shm-size=32g
        --ulimit memlock=-1
        --ulimit stack=67108864
        --memory=0
        --memory-swap=0
        -v /data/miles_ci:/data/miles_ci
        -v /data/miles_ci/models:/root/models
        -v /data/miles_ci/datasets:/root/datasets
        -v /data/miles_ci/hf_cache:/root/.cache/huggingface
        --privileged
        --ulimit nofile=65535:65535
    defaults:
      run:
        working-directory: ${{ github.workspace }}
    env:
      SGLANG_SOURCE_ROOT: /sgl-workspace/sglang/python
      MEGATRON_SOURCE_ROOT: /root/Megatron-LM
      MILES_TEST_DUMPS_ROOT: /data/miles_ci/dumps
      GITHUB_COMMIT_NAME: ${{ inputs.ref || github.sha }}_${{ github.event.pull_request.number || github.event.inputs.pull_number || 'non-pr' }}
      WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
      NEON_DATABASE_URL: ${{ secrets.NEON_DATABASE_URL }}
      MILES_TEST_ENABLE_INFINITE_RUN: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.infinite_run) || 'false' }}
      # Persist shared JIT caches across job containers; see tests/ci/README.md.
      SGLANG_CACHE_DIR: /data/miles_ci/jit
      # DeepGEMM's PID-based subdirectories must not collide across containers.
      SGLANG_DG_CACHE_DIR: /tmp/deep_gemm
      TRITON_CACHE_DIR: /data/miles_ci/jit/triton
      TORCHINDUCTOR_CACHE_DIR: /data/miles_ci/jit/inductor
      # TileLang 0.1.14 does not key on the headers patched by our image.
      TILELANG_CACHE_DIR: /tmp/tilelang_cache
      CUDA_CACHE_PATH: /data/miles_ci/jit/nv
      FLASHINFER_WORKSPACE_BASE: /data/miles_ci/jit
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.plan_already_resolved && inputs.ref || needs.plan.outputs.ref }}

      # local actions load from the checked-out ref, which may predate them (/rerun-test, release CI)
      - name: Checkout CI actions
        uses: actions/checkout@v4
        with:
          ref: ${{ github.sha }}
          path: .ci-actions
          sparse-checkout: .github/actions

      - name: Configure source paths
        shell: bash
        run: echo "PYTHONPATH=$GITHUB_WORKSPACE:$SGLANG_SOURCE_ROOT:$MEGATRON_SOURCE_ROOT" >> "$GITHUB_ENV"

      - name: Keep NVSHMEM on NVLink (RoCE-only runners)
        # NVSHMEM cannot connect over RoCE from this bridge-network container: a NIC
        # name that matches nothing keeps it off RDMA. See docs/developer/ci/00-stage.md.
        shell: bash
        run: |
          if [ "$(cat /sys/class/infiniband/*/ports/*/link_layer 2>/dev/null | sort -u)" = Ethernet ]; then
            echo "Every HCA is RoCE: setting NVSHMEM_HCA_LIST"
            echo "NVSHMEM_HCA_LIST=none_single_node_nvlink_only" >> "$GITHUB_ENV"
          fi

      - name: Cleanup Ray processes
        shell: bash
        run: |
          pkill -9 -f 'ray::' 2>/dev/null || true
          pkill -9 -f raylet 2>/dev/null || true
          pkill -9 -f gcs_server 2>/dev/null || true
          pkill -9 -f 'ray-dashboard' 2>/dev/null || true
          pkill -9 sglang 2>/dev/null || true
          ray stop --force 2>/dev/null || true
          sleep 3

      - name: Wait for GPU ready
        shell: bash
        run: |
          # When a self-hosted runner accepts a new job seconds after the
          # previous job's container exits, the NVIDIA driver subsystem may
          # still be tearing down its prior CUDA context. The next process
          # to call cudaGetDeviceCount() then hits Error 802 ("system not
          # yet initialized"). Block here until nvidia-smi succeeds, with
          # a bounded retry budget so a genuinely-broken GPU fails fast.
          for i in $(seq 1 60); do
            if nvidia-smi -L >/dev/null 2>&1; then
              echo "GPU ready after $((i-1)) retry(s)"
              nvidia-smi --query-gpu=index,name,memory.used --format=csv,noheader
              exit 0
            fi
            sleep 2
          done
          echo "ERROR: nvidia-smi did not become ready after 120s" >&2
          nvidia-smi 2>&1 | tail -20 || true
          exit 1

      - name: Verify B200 CUDA partition
        if: contains(fromJSON(inputs.runs_on), 'b200')
        env:
          EXPECTED_GPUS: ${{ inputs.num_gpus || (contains(fromJSON(inputs.runs_on), '4gpu') && 4 || 8) }}
        run: |
          python3 - <<'PY'
          import os
          import torch

          expected = int(os.environ["EXPECTED_GPUS"])
          actual = torch.cuda.device_count()
          assert actual == expected, (
              f"B200 stage requires {expected} CUDA-visible GPUs, found {actual}; "
              "fix the runner's CUDA_VISIBLE_DEVICES partition"
          )
          PY

      - name: Resolve dependency refs
        if: ${{ !inputs.skip_dependency_install }}
        id: resolve-refs
        uses: ./.ci-actions/.github/actions/resolve-dependency-refs
        with:
          megatron_ref: ${{ github.event.inputs.ci_megatron_pr || '' }}
          sglang_ref: ${{ github.event.inputs.ci_sglang_pr || '' }}
          pr_body: ${{ github.event.pull_request.body || '' }}

      - name: Reconcile Miles dependencies
        shell: bash
        run: |
          python tests/ci/reconcile_dependencies.py \
            examples/multi_lora/requirements.txt requirements.txt

      - name: Sync dependency sources
        if: ${{ !inputs.skip_dependency_install }}
        shell: bash
        env:
          MEGATRON_PR: ${{ steps.resolve-refs.outputs.ci_megatron_pr }}
          SGLANG_PR: ${{ steps.resolve-refs.outputs.ci_sglang_pr }}
        run: |
          cd /sgl-workspace/sglang && git reset --hard HEAD && git clean -fd && git fetch origin "$SGLANG_PR" && git checkout -f FETCH_HEAD && git log --oneline -1
          cd /root/Megatron-LM && git reset --hard HEAD && git clean -fd && git fetch origin "$MEGATRON_PR" && git checkout -f FETCH_HEAD && git log --oneline -1

      - name: Install Megatron Bridge override
        if: ${{ !inputs.skip_dependency_install && (github.event.inputs.ci_megatron_bridge_pr != '' || contains(github.event.pull_request.body, 'ci-megatron-bridge-pr:')) }}
        uses: ./.ci-actions/.github/actions/install-megatron-bridge
        with:
          ref: ${{ github.event.inputs.ci_megatron_bridge_pr || '' }}
          pr_body: ${{ github.event.pull_request.body || '' }}

      - name: Verify source resolution
        shell: bash
        run: python tests/ci/verify_source_resolution.py

      - name: Run tests
        shell: bash
        run: ${{ inputs.execute_command }}