build.yaml

# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
# All rights reserved.
#
# SPDX-License-Identifier: BSD-3-Clause

# region help
# Every test job runs on every PR and merge queue candidate. Pushes to protected branches run the
# post-merge integration tests.
#
# =============================================================================
# CI DEBUGGING TIPS
# =============================================================================
#
# 1. DISABLE A TEST JOB (to isolate a specific job):
#    Change the job's `if:` clause to `if: false`
#    Example:
#      test-isaaclab-tasks:
#        ...
#        if: false  # TEMP: Disabled for debugging
#
# 2. RUN ONLY SPECIFIC TEST FILES (within a job):
#    Add `include-files:` parameter to run-package-tests action
#    Example:
#      - uses: ./.github/actions/run-package-tests
#        with:
#          ...
#          include-files: "test_rigid_object_collection.py"  # Comma-separated
#
# 3. SKIP CONCURRENCY WAIT (run immediately without waiting for other runs):
#    Comment out the concurrency block:
#      # concurrency:
#      #   group: ${{ github.workflow }}-${{ github.ref }}
#      #   cancel-in-progress: true
#
# 4. TEST LOCALLY LIKE CI:
#    docker run -it --rm --gpus all --network=host --entrypoint bash \
#      -e OMNI_KIT_ACCEPT_EULA=yes -e ACCEPT_EULA=Y -e ISAAC_SIM_HEADLESS=1 \
#      -e TEST_FILTER_PATTERN="isaaclab_physx" \
#      -e TEST_INCLUDE_FILES="test_rigid_object_collection.py" \
#      -v "$PWD":/workspace/isaaclab isaac-lab-base:latest \
#      -c 'cd /workspace/isaaclab && /isaac-sim/python.sh -m pytest tools -v'
#
# Remember to REVERT all temporary changes before merging!
# =============================================================================
#endregion

name: Docker + Tests

on:
  push:
    branches:
      - main
      - develop
      - 'release/**'
  pull_request:
    types: [opened, synchronize, reopened]
    branches:
      - main
      - develop
      - 'release/**'
  workflow_dispatch:

# Concurrency control to prevent parallel runs on the same PR
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

permissions:
  contents: read
  pull-requests: write
  checks: write
  issues: read

env:
  NGC_API_KEY: ${{ secrets.NGC_API_KEY }}

jobs:
  changes:
    name: Detect Changes
    runs-on: ubuntu-latest
    outputs:
      run_docker_tests: ${{ steps.detect.outputs.run_docker_tests }}
    steps:
    - id: detect
      env:
        GH_TOKEN: ${{ github.token }}
        PR_NUMBER: ${{ github.event.pull_request.number }}
        EVENT_NAME: ${{ github.event_name }}
        REPO: ${{ github.repository }}
      run: |
        set -euo pipefail

        # Docker test jobs run only when paths in the patterns table change.
        # Otherwise they skip via `if:` and report green to branch protection,
        # which is why we don't use a workflow-level `paths:` filter (a
        # not-triggered required check would block the PR forever).
        # config.yaml is included because it controls the base image names and
        # tags consumed by the Docker build jobs.
        patterns=(
          $'^source/\tLibrary source code'
          $'^docker/\tContainer build inputs'
          $'^tools/\tBuild tooling'
          $'^apps/\tStandalone apps'
          $'^scripts/\tStandalone scripts'
          $'^\\.github/workflows/build\\.yaml$\tThis workflow file'
          $'^\\.github/workflows/config\\.yaml$\tBase image config'
          $'^\\.github/actions/\tCI actions'
          $'^\\.github/test-subsets/\tCI test subset config'
        )
        if [ "$EVENT_NAME" = "push" ]; then
          triggered_jobs="Docker base build job + rendering-correctness + rendering-correctness-kitless"
        else
          triggered_jobs="Docker build jobs + all test-* matrix jobs"
        fi

        render_table() {
          local files="$1" entry regex desc count sample shown
          echo "| Pattern | What it covers | Matched files |"
          echo "|---|---|---|"
          for entry in "${patterns[@]}"; do
            IFS=$'\t' read -r regex desc <<< "$entry"
            # escape | so it doesn't end the markdown table cell mid-regex
            shown="${regex//|/\\|}"
            count=$(grep -cE "$regex" <<< "$files" || true)
            if [ "$count" -gt 0 ]; then
              sample=$(grep -m 3 -E "$regex" <<< "$files" | paste -sd ', ' -)
              [ "$count" -gt 3 ] && sample="$sample (and $((count - 3)) more)"
              echo "| \`$shown\` | $desc | $sample |"
            else
              echo "| \`$shown\` | $desc | - |"
            fi
          done
        }

        any_match() {
          local files="$1" entry regex
          for entry in "${patterns[@]}"; do
            IFS=$'\t' read -r regex _ <<< "$entry"
            if grep -qE "$regex" <<< "$files"; then
              return 0
            fi
          done
          return 1
        }

        decide() {
          local decision="$1" reason="$2" files="${3:-}"
          echo "Decision: run_docker_tests=$decision ($reason)"
          echo "run_docker_tests=$decision" >> "$GITHUB_OUTPUT"
          {
            echo "## Docker test gating"
            echo ""
            if [ "$decision" = "true" ]; then
              echo "Docker tests will **run**: $reason."
            else
              echo "Docker tests will be **skipped**: $reason."
            fi
            echo ""
            echo "Triggered jobs: $triggered_jobs."
            if [ -n "$files" ]; then
              echo ""
              render_table "$files"
            fi
          } >> "$GITHUB_STEP_SUMMARY"
        }

        if [ "$EVENT_NAME" != "pull_request" ]; then
          decide true "non-PR event ($EVENT_NAME)"
          exit 0
        fi

        if ! changed_files="$(gh api --paginate "repos/$REPO/pulls/$PR_NUMBER/files" --jq '.[].filename')"; then
          # Fail-safe: a transient API error must not block merge. Default to running.
          echo "::warning::Could not list changed files; defaulting to running tests"
          decide true "fail-safe (could not list changed files)"
          exit 0
        fi

        printf '%s\n' "$changed_files"

        if any_match "$changed_files"; then
          decide true "relevant paths changed" "$changed_files"
        else
          decide false "no relevant paths changed" "$changed_files"
        fi

  config:
    name: Load Config
    runs-on: ubuntu-latest
    outputs:
      isaacsim_image_name: ${{ steps.load.outputs.isaacsim_image_name }}
      isaacsim_image_tag: ${{ steps.load.outputs.isaacsim_image_tag }}
      isaaclab_image_name: ${{ steps.load.outputs.isaaclab_image_name }}
      ovphysx_wheelhouse_resource: ${{ steps.load.outputs.ovphysx_wheelhouse_resource }}
      ci_image_tag: ${{ steps.image_tag.outputs.ci_image_tag }}
    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 1
        sparse-checkout: .github/workflows/config.yaml
        sparse-checkout-cone-mode: false
    - id: load
      run: |
        set -euo pipefail
        f=.github/workflows/config.yaml
        echo "isaacsim_image_name=$(yq -r .isaacsim_image_name "$f")" >> "$GITHUB_OUTPUT"
        echo "isaacsim_image_tag=$(yq -r .isaacsim_image_tag "$f")" >> "$GITHUB_OUTPUT"
        echo "isaaclab_image_name=$(yq -r .isaaclab_image_name "$f")" >> "$GITHUB_OUTPUT"
        echo "ovphysx_wheelhouse_resource=$(yq -r .ovphysx_wheelhouse_resource "$f")" >> "$GITHUB_OUTPUT"
    - id: image_tag
      shell: bash
      env:
        EVENT_NAME: ${{ github.event_name }}
        PR_NUMBER: ${{ github.event.pull_request.number }}
        REF_NAME: ${{ github.ref_name }}
        SHA: ${{ github.sha }}
      run: |
        set -euo pipefail

        if [ "$EVENT_NAME" = "pull_request" ]; then
          ref_component="pr-${PR_NUMBER}"
        else
          ref_component="$REF_NAME"
        fi

        # Sanitize the ref name for use as a Docker tag suffix.
        sanitized_ref=$(echo "$ref_component" | sed 's/[^a-zA-Z0-9._-]/-/g')
        echo "ci_image_tag=isaac-lab-ci:${sanitized_ref}-${SHA}" >> "$GITHUB_OUTPUT"
        echo "CI image tag: isaac-lab-ci:${sanitized_ref}-${SHA}"

  #region build jobs
  build:
    name: Build Base Docker Image
    runs-on: [self-hosted, gpu]
    needs: [changes, config]
    if: needs.changes.outputs.run_docker_tests == 'true'
    steps:
    - name: Checkout Code
      uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true

    - name: Build and push to ECR
      uses: ./.github/actions/ecr-build-push-pull
      with:
        image-tag: ${{ needs.config.outputs.ci_image_tag }}
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
        dockerfile-path: docker/Dockerfile.base
        cache-tag: cache-base

  build-curobo:
    name: Build cuRobo Docker Image
    runs-on: [self-hosted, gpu]
    needs: [changes, config]
    if: >-
      github.event_name != 'push' &&
      needs.changes.outputs.run_docker_tests == 'true'
    steps:
    - name: Checkout Code
      uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true

    - name: Build and push to ECR
      uses: ./.github/actions/ecr-build-push-pull
      with:
        image-tag: ${{ needs.config.outputs.ci_image_tag }}-curobo
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
        dockerfile-path: docker/Dockerfile.curobo
        cache-tag: cache-curobo

  # aarch64 build + marker-gated tests on NVIDIA DGX Spark self-hosted runners.
  # Build and test must share one runner because ECR is not wired for arm64 —
  # the locally-built image cannot be handed off across machines.
  arm-ci:
    name: arm-ci
    runs-on: [self-hosted, arm64]
    needs: [changes, config]
    if: needs.changes.outputs.run_docker_tests == 'true'
    timeout-minutes: 60
    continue-on-error: true
    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true

    - name: Build base image (linux/arm64)
      uses: ./.github/actions/docker-build
      with:
        image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
        dockerfile-path: docker/Dockerfile.base
        platform: linux/arm64
        # The Spark runner is long-lived self-hosted with no ECR, so its local
        # deps-cache tags accumulate; evict ones older than 14 days each run.
        evict-stale-cache: "true"

    - name: Run arm_ci marker tests
      uses: ./.github/actions/run-tests
      with:
        test-path: tools
        result-file: arm-ci-report.xml
        container-name: isaac-lab-arm-ci-${{ github.run_id }}-${{ github.run_attempt }}
        image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64
        extra-pip-packages: "ovrtx ovphysx==0.4.13"
        # ovphysx-backed test params require ovphysx >= 0.5.1, which has no aarch64
        # wheel yet; keep the newton/ovrtx rendering coverage, mirroring the public
        # pip-index fallback of rendering-correctness-kitless. Running them anyway
        # exhausts ovrtx SyncScopeIds (>15 renderer creations in one process).
        test-k-expr: not ovphysx
        ci-marker: arm_ci
        volume-mount-source: ${{ github.workspace }}

    - name: Run shared Cartpole smoke
      uses: ./.github/actions/run-tests
      with:
        test-path: source/isaaclab/test/install_ci/misc/cartpole_training_smoke.py
        result-file: arm-ci-cartpole-smoke-report.xml
        container-name: isaac-lab-arm-ci-cartpole-${{ github.run_id }}-${{ github.run_attempt }}
        image-tag: ${{ needs.config.outputs.ci_image_tag }}-arm64
        volume-mount-source: ${{ github.workspace }}

    - name: Upload test reports
      if: always()
      uses: actions/upload-artifact@v7
      with:
        name: arm-ci-reports
        path: reports/
        retention-days: 7

  #endregion

  #region test jobs
  test-isaaclab-tasks:
    name: isaaclab_tasks [1/3]
    runs-on: [self-hosted, gpu]
    timeout-minutes: 180
    continue-on-error: true
    needs: [build, config]
    if: >-
      github.event_name != 'push' &&
      needs.build.result == 'success'
    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true
    - uses: ./.github/actions/run-package-tests
      with:
        image-tag: ${{ needs.config.outputs.ci_image_tag }}
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
        filter-pattern: "isaaclab_tasks"
        exclude-pattern: "test_rendering_"
        shard-index: "0"
        shard-count: "3"
        container-name: isaac-lab-tasks-1-test

  test-isaaclab-tasks-2:
    name: isaaclab_tasks [2/3]
    runs-on: [self-hosted, gpu]
    timeout-minutes: 180
    continue-on-error: true
    needs: [build, config]
    if: >-
      github.event_name != 'push' &&
      needs.build.result == 'success'
    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true
    - uses: ./.github/actions/run-package-tests
      with:
        image-tag: ${{ needs.config.outputs.ci_image_tag }}
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
        filter-pattern: "isaaclab_tasks"
        exclude-pattern: "test_rendering_"
        shard-index: "1"
        shard-count: "3"
        container-name: isaac-lab-tasks-2-test

  test-isaaclab-tasks-3:
    name: isaaclab_tasks [3/3]
    runs-on: [self-hosted, gpu]
    timeout-minutes: 180
    continue-on-error: true
    needs: [build, config]
    if: >-
      github.event_name != 'push' &&
      needs.build.result == 'success'
    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true
    - uses: ./.github/actions/run-package-tests
      with:
        image-tag: ${{ needs.config.outputs.ci_image_tag }}
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
        filter-pattern: "isaaclab_tasks"
        exclude-pattern: "test_rendering_"
        shard-index: "2"
        shard-count: "3"
        container-name: isaac-lab-tasks-3-test

  test-isaaclab-core:
    name: isaaclab (core) [1/3]
    runs-on: [self-hosted, gpu]
    timeout-minutes: 180
    needs: [build, config]
    if: >-
      github.event_name != 'push' &&
      needs.build.result == 'success'
    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true
    - uses: ./.github/actions/run-package-tests
      with:
        image-tag: ${{ needs.config.outputs.ci_image_tag }}
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
        filter-pattern: "not isaaclab_"
        shard-index: "0"
        shard-count: "3"
        container-name: isaac-lab-core-1-test

  test-isaaclab-core-2:
    name: isaaclab (core) [2/3]
    runs-on: [self-hosted, gpu]
    timeout-minutes: 180
    needs: [build, config]
    if: >-
      github.event_name != 'push' &&
      needs.build.result == 'success'
    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true
    - uses: ./.github/actions/run-package-tests
      with:
        image-tag: ${{ needs.config.outputs.ci_image_tag }}
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
        filter-pattern: "not isaaclab_"
        shard-index: "1"
        shard-count: "3"
        container-name: isaac-lab-core-2-test

  test-isaaclab-core-3:
    name: isaaclab (core) [3/3]
    runs-on: [self-hosted, gpu]
    timeout-minutes: 180
    needs: [build, config]
    if: >-
      github.event_name != 'push' &&
      needs.build.result == 'success'
    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true
    - uses: ./.github/actions/run-package-tests
      with:
        image-tag: ${{ needs.config.outputs.ci_image_tag }}
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
        filter-pattern: "not isaaclab_"
        shard-index: "2"
        shard-count: "3"
        container-name: isaac-lab-core-3-test

  test-isaaclab-rl:
    name: isaaclab_rl
    runs-on: [self-hosted, gpu]
    timeout-minutes: 180
    needs: [build, config]
    if: >-
      github.event_name != 'push' &&
      needs.build.result == 'success'
    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true
    - uses: ./.github/actions/run-package-tests
      with:
        image-tag: ${{ needs.config.outputs.ci_image_tag }}
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
        filter-pattern: "isaaclab_rl"
        extra-pip-packages: "leapp"
        container-name: isaac-lab-rl-test

  test-isaaclab-mimic:
    name: isaaclab_mimic
    runs-on: [self-hosted, gpu]
    timeout-minutes: 180
    needs: [build, config]
    if: >-
      github.event_name != 'push' &&
      needs.build.result == 'success'
    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true
    - uses: ./.github/actions/run-package-tests
      with:
        image-tag: ${{ needs.config.outputs.ci_image_tag }}
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
        filter-pattern: "isaaclab_mimic"
        container-name: isaac-lab-mimic-test

  test-isaaclab-contrib:
    name: isaaclab_contrib
    runs-on: [self-hosted, gpu]
    timeout-minutes: 180
    needs: [build, config]
    if: >-
      github.event_name != 'push' &&
      needs.build.result == 'success'
    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true
    - uses: ./.github/actions/run-package-tests
      with:
        image-tag: ${{ needs.config.outputs.ci_image_tag }}
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
        filter-pattern: "isaaclab_contrib"
        container-name: isaac-lab-contrib-test

  test-isaaclab-teleop:
    name: isaaclab_teleop
    runs-on: [self-hosted, gpu]
    timeout-minutes: 180
    needs: [build, config]
    if: >-
      github.event_name != 'push' &&
      needs.build.result == 'success'
    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true
    - uses: ./.github/actions/run-package-tests
      with:
        image-tag: ${{ needs.config.outputs.ci_image_tag }}
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
        filter-pattern: "isaaclab_teleop"
        container-name: isaac-lab-teleop-test

  test-isaaclab-visualizers:
    name: isaaclab_visualizers
    runs-on: [self-hosted, gpu]
    timeout-minutes: 180
    needs: [build, config]
    if: >-
      github.event_name != 'push' &&
      needs.build.result == 'success'
    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true
    - uses: ./.github/actions/run-package-tests
      with:
        image-tag: ${{ needs.config.outputs.ci_image_tag }}
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
        filter-pattern: "isaaclab_visualizers"
        container-name: isaac-lab-visualizers-test

  test-isaaclab-assets:
    name: isaaclab_assets
    runs-on: [self-hosted, gpu]
    timeout-minutes: 180
    needs: [build, config]
    if: >-
      github.event_name != 'push' &&
      needs.build.result == 'success'
    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true
    - uses: ./.github/actions/run-package-tests
      with:
        image-tag: ${{ needs.config.outputs.ci_image_tag }}
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
        filter-pattern: "isaaclab_assets"
        container-name: isaac-lab-assets-test

  test-isaaclab-newton:
    name: isaaclab_newton
    runs-on: [self-hosted, gpu]
    timeout-minutes: 180
    needs: [build, config]
    if: >-
      github.event_name != 'push' &&
      needs.build.result == 'success'
    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true
    - uses: ./.github/actions/run-package-tests
      with:
        image-tag: ${{ needs.config.outputs.ci_image_tag }}
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
        filter-pattern: "isaaclab_newton"
        container-name: isaac-lab-newton-test

  test-isaaclab-physx:
    name: isaaclab_physx
    runs-on: [self-hosted, gpu]
    timeout-minutes: 180
    needs: [build, config]
    if: >-
      github.event_name != 'push' &&
      needs.build.result == 'success'
    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true
    - uses: ./.github/actions/run-package-tests
      with:
        image-tag: ${{ needs.config.outputs.ci_image_tag }}
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
        filter-pattern: "isaaclab_physx"
        container-name: isaac-lab-physx-test

  test-isaaclab-ov:
    name: isaaclab_ov
    runs-on: [self-hosted, gpu]
    timeout-minutes: 180
    needs: [build, config]
    if: >-
      github.event_name != 'push' &&
      needs.build.result == 'success'
    env:
      USE_OVPHYSX_WHEELHOUSE: ${{ needs.config.outputs.ovphysx_wheelhouse_resource != '' && ((github.event_name == 'pull_request' && github.base_ref == 'develop' && github.event.pull_request.head.repo.full_name == github.repository) || (github.event_name != 'pull_request' && github.ref_name == 'develop')) }}
    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true
    - uses: ./.github/actions/run-package-tests
      with:
        image-tag: ${{ needs.config.outputs.ci_image_tag }}
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
        filter-pattern: "isaaclab_ov"
        # isaaclab_ovphysx requires ovphysx >= 0.5.1, which is only available from the
        # NGC wheelhouse; untrusted runs (fork PRs) fall back to the public pip index
        # and keep only the ovrtx-based isaaclab_ov coverage.
        exclude-pattern: ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && '' || 'isaaclab_ovphysx' }}
        extra-pip-packages: ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && 'ovrtx' || 'ovrtx ovphysx' }}
        wheelhouse-resource: ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && needs.config.outputs.ovphysx_wheelhouse_resource || '' }}
        wheelhouse-packages: ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && 'ovphysx' || '' }}
        container-name: isaac-lab-ov-test

    # Folded from the former standalone verify-base-non-root job: reuses the
    # base image already pulled by run-package-tests to keep the regression
    # check without burning a separate runner.
    - name: Run Dockerfile non-root regression test
      shell: bash
      run: |
        set -euo pipefail
        docker run --rm \
          -v "$PWD":/workspace/isaaclab \
          --entrypoint bash \
          "${{ needs.config.outputs.ci_image_tag }}" \
          -lc 'cd /workspace/isaaclab && /isaac-sim/python.sh -m pytest docker/test/test_dockerfile_nonroot.py -q'

    - name: Verify Base runtime user is non-root
      shell: bash
      run: |
        set -euo pipefail
        runtime_identity="$(docker run --rm --entrypoint bash "${{ needs.config.outputs.ci_image_tag }}" \
          -lc 'printf "%s %s %s\n" "$(id -u)" "$(id -g)" "$(id -un 2>/dev/null || true)"')"
        read -r runtime_uid runtime_gid runtime_user <<< "${runtime_identity}"
        echo "Base runtime identity: uid=${runtime_uid} gid=${runtime_gid} user=${runtime_user}"
        if [ "${runtime_uid}" = "0" ]; then
          echo "::error::Base Docker image must not run as root by default."
          exit 1
        fi

  test-curobo:
    name: test-curobo
    runs-on: [self-hosted, gpu]
    timeout-minutes: 120
    continue-on-error: true
    needs: [build-curobo, config]
    if: >-
      github.event_name != 'push' &&
      needs.build-curobo.result == 'success'
    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true
    - uses: ./.github/actions/run-package-tests
      with:
        image-tag: ${{ needs.config.outputs.ci_image_tag }}-curobo
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
        dockerfile-path: docker/Dockerfile.curobo
        cache-tag: cache-curobo
        include-files: "test_curobo_planner_franka.py,test_curobo_planner_cube_stack.py,test_pink_ik.py"
        container-name: isaac-lab-curobo-test

    # Folded from the former standalone verify-curobo-non-root job: reuses
    # the curobo image already pulled by run-package-tests to keep the
    # regression check without burning a separate runner.
    - name: Run Dockerfile non-root regression test
      shell: bash
      run: |
        set -euo pipefail
        docker run --rm \
          -v "$PWD":/workspace/isaaclab \
          --entrypoint bash \
          "${{ needs.config.outputs.ci_image_tag }}-curobo" \
          -lc 'cd /workspace/isaaclab && /isaac-sim/python.sh -m pytest docker/test/test_dockerfile_nonroot.py -q'

    - name: Verify cuRobo runtime user is non-root
      shell: bash
      run: |
        set -euo pipefail
        runtime_identity="$(docker run --rm --entrypoint bash "${{ needs.config.outputs.ci_image_tag }}-curobo" \
          -lc 'printf "%s %s %s\n" "$(id -u)" "$(id -g)" "$(id -un 2>/dev/null || true)"')"
        read -r runtime_uid runtime_gid runtime_user <<< "${runtime_identity}"
        echo "cuRobo runtime identity: uid=${runtime_uid} gid=${runtime_gid} user=${runtime_user}"
        if [ "${runtime_uid}" = "0" ]; then
          echo "::error::cuRobo Docker image must not run as root by default."
          exit 1
        fi

  test-skillgen:
    name: test-skillgen
    runs-on: [self-hosted, gpu]
    timeout-minutes: 120
    continue-on-error: true
    needs: [build-curobo, config]
    if: >-
      github.event_name != 'push' &&
      needs.build-curobo.result == 'success'
    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true
    - uses: ./.github/actions/run-package-tests
      with:
        image-tag: ${{ needs.config.outputs.ci_image_tag }}-curobo
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
        dockerfile-path: docker/Dockerfile.curobo
        cache-tag: cache-curobo
        include-files: "test_generate_dataset_skillgen.py,test_environments_skillgen.py,test_environments_automate.py"
        container-name: isaac-lab-skillgen-test

  test-environments-training:
    name: "environments_training"
    runs-on: [self-hosted, gpu]
    timeout-minutes: 300
    continue-on-error: true
    needs: [build, config]
    if: >-
      github.event_name != 'push' &&
      needs.build.result == 'success'
    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true
    - uses: ./.github/actions/run-package-tests
      with:
        image-tag: ${{ needs.config.outputs.ci_image_tag }}
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
        filter-pattern: "isaaclab_tasks"
        include-files: "test_environments_training.py"
        container-name: isaac-lab-environments-training-test
        omni-github-test-type: training-e2e

  test-rendering-correctness:
    name: "rendering-correctness"
    runs-on: [self-hosted, gpu]
    timeout-minutes: 120
    continue-on-error: ${{ github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }}
    needs: [build, config]
    if: needs.build.result == 'success'
    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true
    - uses: ./.github/actions/run-package-tests
      with:
        image-tag: ${{ needs.config.outputs.ci_image_tag }}
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
        filter-pattern: "isaaclab_tasks"
        include-files: >-
          test_rendering_cartpole.py,
          test_rendering_dexsuite_kuka_hetero.py,
          test_rendering_dexsuite_kuka_homo.py,
          test_rendering_franka_cloth.py,
          test_rendering_franka_soft.py,
          test_rendering_registered_tasks.py,
          test_rendering_shadow_hand.py
        test-node-ids-file: ${{ github.event_name == 'push' && '.github/test-subsets/postmerge-rendering.toml' || '' }}
        test-node-ids-key: ${{ github.event_name == 'push' && 'rendering-correctness' || '' }}
        container-name: isaac-lab-rendering-correctness-test
        omni-github-test-type: rendering-correctness

  test-rendering-correctness-kitless:
    name: "rendering-correctness-kitless"
    runs-on: [self-hosted, gpu]
    timeout-minutes: 120
    continue-on-error: ${{ github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }}
    needs: [build, config]
    if: needs.build.result == 'success'
    env:
      USE_OVPHYSX_WHEELHOUSE: ${{ needs.config.outputs.ovphysx_wheelhouse_resource != '' && ((github.event_name == 'pull_request' && github.base_ref == 'develop' && github.event.pull_request.head.repo.full_name == github.repository) || (github.event_name != 'pull_request' && github.ref_name == 'develop')) }}
    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 1
        lfs: true
    - uses: ./.github/actions/run-package-tests
      with:
        image-tag: ${{ needs.config.outputs.ci_image_tag }}
        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
        filter-pattern: "isaaclab_tasks"
        # ovphysx-backed test params require ovphysx >= 0.5.1, which is only available
        # from the NGC wheelhouse; untrusted runs (fork PRs) fall back to the public pip
        # index and keep the newton/ovrtx kitless rendering coverage.
        test-k-expr: ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && '' || 'not ovphysx' }}
        extra-pip-packages: ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && 'ovrtx' || 'ovrtx ovphysx' }}
        wheelhouse-resource: ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && needs.config.outputs.ovphysx_wheelhouse_resource || '' }}
        wheelhouse-packages: ${{ env.USE_OVPHYSX_WHEELHOUSE == 'true' && 'ovphysx' || '' }}
        include-files: >-
          test_rendering_cartpole_kitless.py,
          test_rendering_dexsuite_kuka_hetero_kitless.py,
          test_rendering_dexsuite_kuka_homo_kitless.py,
          test_rendering_franka_cloth_kitless.py,
          test_rendering_franka_soft_kitless.py,
          test_rendering_shadow_hand_kitless.py
        test-node-ids-file: ${{ github.event_name == 'push' && '.github/test-subsets/postmerge-rendering.toml' || '' }}
        test-node-ids-key: ${{ github.event_name == 'push' && 'rendering-correctness-kitless' || '' }}
        container-name: isaac-lab-rendering-correctness-kitless-test
        omni-github-test-type: rendering-correctness-kitless
  #endregion

#region disabled quarantined tests
# test-quarantined:
#   name: "Quarantined Tests"
#   runs-on: [self-hosted, gpu]
#   timeout-minutes: 180
#   continue-on-error: true
#   needs: [build, config]
#   if: >-
#     needs.build.result == 'success' &&
#     vars.RUN_QUARANTINED_TESTS == 'true'
#   steps:
#   - uses: actions/checkout@v6
#     with:
#       fetch-depth: 1
#       lfs: true
#   - uses: ./.github/actions/run-package-tests
#     with:
#       image-tag: ${{ needs.config.outputs.ci_image_tag }}
#       isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
#       isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
#       quarantined-only: "true"
#       container-name: isaac-lab-quarantined-test
#endregion