Using more specific GPU runners (#69)

Changing GHA to use more specific GPU runners, so nightly tests split
among 1xGPU and 4xGPU, and pytests run on 1xGPU or 4xGPU as appopriate

Author: clayrosenthal

.github/workflows/nightly-training.yaml

@@ -15,7 +15,9 @@ permissions:
 
 jobs:
   training:
-    runs-on: codebuild-holosoma-gpu-build-${{ github.run_id }}-${{ github.run_attempt }}
+    runs-on: codebuild-holosoma-l4-${{ matrix.multigpu == 'True' && 'x4' || 'x1' }}-gpu-build-${{ github.run_id }}-${{ github.run_attempt }}
+    #TODO: test nightly on a10g/other GPUs
+    # runs-on: codebuild-holosoma-a10g-${{ matrix.multigpu == 'True' && 'x4' || 'x1' }}-gpu-build-${{ github.run_id }}-${{ github.run_attempt }}
     continue-on-error: true
     strategy:
       fail-fast: false # continue on one experiment failing

.github/workflows/pytest.yaml

@@ -18,15 +18,23 @@ on:
 jobs:
   tests:
     strategy:
+      fail-fast: false # continue on one test failing
       matrix:
         simulator: [isaacgym, isaacsim]
-    name: Run ${{ matrix.simulator }} Tests
-    runs-on: codebuild-holosoma-gpu-build-${{ github.run_id }}-${{ github.run_attempt }}
+        multigpu: ["False"]
+        # no multigpu isaacsim tests for now
+        include:
+        - simulator: isaacgym
+          multigpu: "True"
+    name: Run ${{ matrix.simulator }} Tests with ${{ matrix.multigpu == 'True' && 'multi-gpu' || 'single-gpu' }}
+    runs-on:
+    - codebuild-holosoma-a10g-${{ matrix.multigpu == 'True' && 'x4' || 'x1' }}-gpu-build-${{ github.run_id }}-${{ github.run_attempt }}
+    continue-on-error: true
     container:
-      image: 982423663241.dkr.ecr.us-west-2.amazonaws.com/holosoma:latest
+      image: 982423663241.dkr.ecr.us-west-2.amazonaws.com/holosoma-${{ matrix.simulator }}:latest
       options: "--gpus all --runtime=nvidia --shm-size=12g"
-      volumes:
-        - "precommit-cache:/github/home/.cache/pre-commit"
+      env:
+        HOLOSOMA_MULTIGPU: ${{ matrix.multigpu }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v6
@@ -41,11 +49,42 @@ jobs:
           fi
           bash tests/ci/${{ matrix.simulator }}_ci_tests.sh
 
+  e2e-tests:
+    strategy:
+      fail-fast: false # continue on one test failing
+      # e2e only on isaacgym single gpu for now
+      matrix:
+        simulator: [isaacgym]
+        multigpu: ["False"]
+    name: Run ${{ matrix.simulator }} Tests end-to-end with ${{ matrix.multigpu == 'True' && 'multi-gpu' || 'single-gpu' }}
+    runs-on:
+    - codebuild-holosoma-a10g-${{ matrix.multigpu == 'True' && 'x4' || 'x1' }}-gpu-build-${{ github.run_id }}-${{ github.run_attempt }}
+    continue-on-error: true
+    container:
+      # needs both hsgym and hsinference envs
+      image: 982423663241.dkr.ecr.us-west-2.amazonaws.com/holosoma:latest
+      options: "--gpus all --runtime=nvidia --shm-size=12g"
+      env:
+        HOLOSOMA_MULTIGPU: ${{ matrix.multigpu }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+      - name: Pytest
+        id: pytest
+        shell: bash
+        run: |
+          ln -s /root/.holosoma_deps "$HOME/.holosoma_deps"
+          if [[ ! -L /workspace/holosoma ]]; then
+            rm -rf /workspace/holosoma
+            ln -sfF "$GITHUB_WORKSPACE" /workspace/holosoma
+          fi
+          bash tests/ci/${{ matrix.simulator }}_e2e_ci_tests.sh
+
   inference-tests:
     name: Run Inference Unit Tests
     runs-on: codebuild-holosoma-cpu-build-${{ github.run_id }}-${{ github.run_attempt }}
     container:
-      # TODO: Switch to holosoma-inference:latest once it's available in ECR
+      # TODO: Switch to `holosoma-inference` once it's been confirmed functional
       # image: 982423663241.dkr.ecr.us-west-2.amazonaws.com/holosoma-inference:latest
       # Until then, the inference tests will run in isaacsim env.
       image: 982423663241.dkr.ecr.us-west-2.amazonaws.com/holosoma:latest

conftest.py

@@ -4,11 +4,21 @@
 from holosoma.utils.safe_torch_import import torch  # noqa: F401
 
 
+def mark_str(marker, msg):
+    return f"{marker}: marks tests as requiring {msg} (deselect with '-m \"not {marker}\"')"
+
+
 def pytest_configure(config):
     """Register custom markers for pytest."""
     config.addinivalue_line(
-        "markers", "isaacsim: marks tests as requiring Isaac Sim (deselect with '-m \"not isaacsim\"')"
+        "markers",
+        mark_str("isaacsim", "Isaac Sim"),
+    )
+    config.addinivalue_line(
+        "markers",
+        mark_str("multi_gpu", "multiple GPUs"),
     )
     config.addinivalue_line(
-        "markers", "multi_gpu: marks tests as requiring multiple GPUs (deselect with '-m \"not multi_gpu\"')"
+        "markers",
+        mark_str("requires_inference", "inference environment"),
     )

scripts/source_isaacgym_setup.sh

@@ -8,6 +8,6 @@ fi
 CONDA_ENV_NAME=${CONDA_ENV_NAME:-hsgym}
 echo "conda environment name is set to: $CONDA_ENV_NAME"
 
-source ${SCRIPT_DIR}/source_common.sh
-source ${CONDA_ROOT}/bin/activate $CONDA_ENV_NAME
+source "${SCRIPT_DIR}/source_common.sh"
+source "${CONDA_ROOT}/bin/activate" "$CONDA_ENV_NAME"
 export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${CONDA_ROOT}/envs/$CONDA_ENV_NAME/lib

tests/ci/isaacgym_ci_tests.sh

@@ -8,4 +8,11 @@ source scripts/source_isaacgym_setup.sh
 pip install -e 'src/holosoma[unitree,booster]'
 pip install -e src/holosoma_inference
 
-pytest -s --ignore=thirdparty --ignore=src/holosoma_inference -m "not isaacsim"
+marker="not isaacsim and not requires_inference"
+if [[ "$HOLOSOMA_MULTIGPU" == "True" ]]; then
+   marker="$marker and multi_gpu"
+elif [[ "$HOLOSOMA_MULTIGPU" == "False" ]]; then
+   marker="$marker and not multi_gpu"
+fi
+
+pytest -s --ignore=thirdparty --ignore=src/holosoma_inference -m "$marker"

tests/ci/isaacgym_e2e_ci_tests.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# CI runs this inside holosoma docker
+set -ex
+
+cd /workspace/holosoma
+
+source scripts/source_isaacgym_setup.sh
+pip install -e 'src/holosoma[unitree,booster]'
+pip install -e src/holosoma_inference
+
+marker="not isaacsim and requires_inference"
+if [[ "$HOLOSOMA_MULTIGPU" == "True" ]]; then
+   marker="$marker and multi_gpu"
+elif [[ "$HOLOSOMA_MULTIGPU" == "False" ]]; then
+   marker="$marker and not multi_gpu"
+fi
+
+pytest -s --ignore=thirdparty --ignore=src/holosoma_inference -m "$marker"

tests/ci/isaacsim_ci_tests.sh

@@ -9,4 +9,11 @@ source scripts/source_isaacsim_setup.sh
 python -m pip install -e 'src/holosoma[unitree,booster]'
 python -m pip install -e src/holosoma_inference
 
-python -m pytest -s -m "isaacsim" --ignore=holosoma/holosoma/envs/legged_base_task/tests/ --ignore=thirdparty --ignore=src/holosoma_inference
+marker="isaacsim and not requires_inference"
+if [[ "$HOLOSOMA_MULTIGPU" == "True" ]]; then
+   marker="$marker and multi_gpu"
+elif [[ "$HOLOSOMA_MULTIGPU" == "False" ]]; then
+   marker="$marker and not multi_gpu"
+fi
+
+python -m pytest -s -m "$marker" --ignore=holosoma/holosoma/envs/legged_base_task/tests/ --ignore=thirdparty --ignore=src/holosoma_inference

tests/e2e/test_run_policy.py

@@ -99,6 +99,7 @@ def assert_run_policy_with_hsinference(config_name: str, model_path: str, timeou
             pass
 
 
+@pytest.mark.requires_inference
 @pytest.mark.parametrize(
     ("workflow_name", "config_name"),
     [