test-multi-gpu-pytest.yaml

# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
# All rights reserved.
#
# SPDX-License-Identifier: BSD-3-Clause

# Multi-GPU unit-test workflow
#
# Runs the non-default-GPU subset of unit tests across the multi-GPU runner
# pool's GPUs in parallel. Uses the same ECR-pulled isaac-lab image as the
# single-GPU CI; only diffs are the runner label and the env vars that pin
# Kit and the test parametrize to the shard's non-default GPU.
#
# Adding a new test to multi-GPU coverage: give its device parametrize a
# non-default-capable scope — argless ``isaaclab.test.utils.test_devices()``
# (cpu + cuda:0 + non-default GPUs) or any ``"..X"`` mask. The workflow
# auto-discovers it.

name: Multi-GPU pytest

on:
  pull_request:
    paths:
      - "source/isaaclab/isaaclab/test/utils/**"
      - "source/isaaclab/isaaclab/app/app_launcher.py"
      - "source/**/test/**/test_*.py"
      - ".github/workflows/test-multi-gpu-pytest.yaml"
      - ".github/actions/run-package-tests/**"
      - ".github/actions/run-tests/**"
      - ".github/actions/ecr-build-push-pull/**"
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  CI_IMAGE_TAG: isaac-lab-ci:${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.ref_name }}-${{ github.sha }}

jobs:
  config:
    name: Load Config
    runs-on: ubuntu-latest
    outputs:
      isaacsim_image_name: ${{ steps.load.outputs.isaacsim_image_name }}
      isaacsim_image_tag: ${{ steps.load.outputs.isaacsim_image_tag }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
        with:
          fetch-depth: 1
          sparse-checkout: .github/workflows/config.yaml
          sparse-checkout-cone-mode: false
      - id: load
        run: |
          set -euo pipefail
          f=.github/workflows/config.yaml
          echo "isaacsim_image_name=$(yq -r .isaacsim_image_name "$f")" >> "$GITHUB_OUTPUT"
          echo "isaacsim_image_tag=$(yq -r .isaacsim_image_tag "$f")" >> "$GITHUB_OUTPUT"

  build:
    name: Build / cache image
    needs: [config]
    # Must run on the SAME pool as the test job. The ECR cache repo is resolved
    # per runner pool (single-GPU `gpu` runners -> gitci-docker-cache; multi-GPU
    # runners -> multigpu-docker-cache). If this built on `[self-hosted, gpu]`
    # the image would land in gitci-docker-cache, which the multi-GPU test job
    # cannot see, so it would rebuild from scratch on the scarce multi-GPU
    # runner. Building here populates multigpu-docker-cache so the test job's
    # pull hits.
    runs-on: [self-hosted, linux, x64, multi-gpu]
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
        with:
          fetch-depth: 1
          lfs: true

      # Pre-populates the ECR exact-commit tag from deps-cache (registry-side
      # alias). Without this prior step, run-package-tests' internal
      # ecr-build-push-pull hits exact-cache-miss + deps-cache-hit and leaves
      # no local image, causing `docker run` to fail with `pull access
      # denied`. Mirrors the build → test split in build.yaml.
      - uses: ./.github/actions/ecr-build-push-pull
        env:
          NGC_API_KEY: ${{ secrets.NGC_API_KEY }}
        with:
          image-tag: ${{ env.CI_IMAGE_TAG }}
          isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
          isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
          dockerfile-path: docker/Dockerfile.base
          cache-tag: cache-base

  test-multi-gpu-pytest:
    name: Multi-GPU unit tests
    needs: [config, build]
    if: needs.build.result == 'success'
    # The ``multi-gpu`` label is the multi-GPU pool. Do NOT add ``gpu`` here:
    # in this fleet ``gpu`` tags single-GPU runners, so requiring it routes the
    # job onto a 1-GPU box and the shard runner aborts with "Need at least 2
    # visible devices; found 1".
    runs-on: [self-hosted, linux, x64, multi-gpu]
    timeout-minutes: 60
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
        with:
          fetch-depth: 1
          lfs: true

      - name: Discover opt-in test files
        # Auto-discovery: any test_*.py with a non-default-capable scope — an
        # argless ``test_devices()``, a named scope containing non-default GPUs,
        # or a string mask with a trailing ``X`` — is in scope. Adding a test to
        # multi-GPU CI needs no workflow edit; opting a file out is just narrowing
        # its scope to ``DeviceScope.CPU_AND_DEFAULT_CUDA`` (or mask ``"110"``).
        #
        # Within a discovered file, tests that are NOT parametrized over the
        # ``device`` argument are deselected at collection time by the
        # ``mgpu_shard_select`` plugin (injected per shard by ``tools/conftest.py``):
        # single-GPU CI already covers them on ``cuda:0`` and re-running on every
        # non-default shard adds wall-time without surfacing any new failure mode.
        id: discover
        run: |
          # File-level opt-out: a test file can exclude itself from multi-GPU CI
          # by declaring a module-level ``MULTI_GPU_SKIP_REASON = "..."`` line.
          # Used for files with known Kit/Isaac-Sim concurrency issues; the file
          # still runs in single-GPU CI. Excluded files are reported as a notice
          # below for visibility. No workflow edit needed to add/remove a file.
          device_scope_pattern='test_devices\(\)|test_devices\([^)]*DeviceScope\.(ALL|CUDA|NON_DEFAULT_CUDA)|test_devices\("[^"]*X"\)'
          mapfile -t candidates < <(grep -rlE "$device_scope_pattern" source/ --include='test_*.py' | sort -u)
          discovered=()
          skipped=()
          for f in "${candidates[@]}"; do
            if grep -q '^MULTI_GPU_SKIP_REASON' "$f"; then
              skipped+=("$f")
            else
              discovered+=("$f")
            fi
          done
          for f in "${skipped[@]}"; do
            reason=$(grep -m1 '^MULTI_GPU_SKIP_REASON' "$f" | sed -E 's/^MULTI_GPU_SKIP_REASON[[:space:]]*=[[:space:]]*//; s/^"//; s/"$//')
            echo "::notice::multi-GPU skipped: $f — $reason"
          done
          if [ ${#discovered[@]} -eq 0 ]; then
            echo "::error::No tests with a non-default-capable device scope were discovered"
            exit 1
          fi
          basenames=$(printf '%s\n' "${discovered[@]}" | xargs -n1 basename | sort -u | paste -sd,)
          echo "include=$basenames" >> "$GITHUB_OUTPUT"
          # Full relative paths too, to seed the shared work queue (the run step
          # needs runnable paths, not just basenames).
          echo "paths=$(printf '%s,' "${discovered[@]}")" >> "$GITHUB_OUTPUT"
          echo "::notice::Discovered ${#discovered[@]} opt-in test files"
          printf '  %s\n' "${discovered[@]}"

      - name: Pull image from ECR
        # Pulls the per-commit image the build job pushed. ecr-build-push-pull
        # handles ECR auth via the EC2 IAM role, and on exact-cache-hit (which
        # the build job's deps-cache-hit registry-tag created for this SHA)
        # it pulls the image locally and tags it as ``$CI_IMAGE_TAG`` for our
        # parallel ``docker run`` block below.
        uses: ./.github/actions/ecr-build-push-pull
        env:
          NGC_API_KEY: ${{ secrets.NGC_API_KEY }}
        with:
          image-tag: ${{ env.CI_IMAGE_TAG }}
          isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
          isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
          dockerfile-path: docker/Dockerfile.base
          cache-tag: cache-base

      - name: Run shards in parallel on local GPUs (1-docker N-shard)
        # ONE container hosts all N pytest shards as parallel subshells, each
        # pinned to its own non-default cuda:N and pulling from a shared work
        # queue; the host reconciles the queue afterward and exports
        # MGPU_RUNTIME_DIR for the summary step. Full rationale + the
        # 1-docker-N-shard tradeoffs live in the script header.
        env:
          IMAGE_TAG: ${{ env.CI_IMAGE_TAG }}
          INCLUDE_FILES: ${{ steps.discover.outputs.include }}
          PATHS: ${{ steps.discover.outputs.paths }}
        run: bash .github/actions/multi-gpu/multi_gpu_host_launcher.sh

      - name: Aggregated test summary
        # Per-shard + per-file pass/total/walltime, plus a combined table, also
        # written to $GITHUB_STEP_SUMMARY. Runs on success or failure.
        if: always()
        env:
          RUNTIME_DIR: ${{ env.MGPU_RUNTIME_DIR }}
        run: python3 .github/actions/multi-gpu/aggregate_test_summary.py