From 605576eb295fc38e0167e0dd751d2028d61dcf89 Mon Sep 17 00:00:00 2001 From: "Diego Vieira (SiDi)" Date: Mon, 15 Jun 2026 12:56:29 -0400 Subject: [PATCH 1/6] update: dockerfile rheo --- workflows/rheo/docker/Dockerfile.x86 | 90 ++++++++++++++++++++-------- 1 file changed, 66 insertions(+), 24 deletions(-) diff --git a/workflows/rheo/docker/Dockerfile.x86 b/workflows/rheo/docker/Dockerfile.x86 index 6bf53bbf..796d658f 100644 --- a/workflows/rheo/docker/Dockerfile.x86 +++ b/workflows/rheo/docker/Dockerfile.x86 @@ -24,7 +24,7 @@ ARG INSTALL_GROOT=false ARG WORKDIR="/workspaces" ENV WORKDIR=${WORKDIR} WORKDIR "${WORKDIR}" -ENV PYTHONPATH=${WORKDIR}/workflows/rheo/scripts:${WORKDIR}/workflows/rheo/scripts/simulation/rl +ENV PYTHONPATH=${WORKDIR}/workflows/rheo:${WORKDIR}/workflows/rheo/scripts:${WORKDIR}/workflows/rheo/scripts/simulation/rl USER root @@ -40,7 +40,10 @@ RUN apt-get update && apt-get install -y \ cmake \ sudo \ ffmpeg \ - python3-pip + python3-pip \ + pkg-config \ + libfreetype6-dev \ + libpng-dev # Skip pip upgrade - not needed in Docker and conflicts with PEP 668 in Python 3.12 # RUN pip3 install --upgrade pip @@ -52,40 +55,79 @@ COPY ./third_party/IsaacLab ${WORKDIR}/third_party/IsaacLab ENV ISAACLAB_PATH=${WORKDIR}/third_party/IsaacLab ENV TERM=xterm RUN ln -s /isaac-sim/ ${WORKDIR}/third_party/IsaacLab/_isaac_sim -# Install IsaacLab dependencies -RUN for DIR in ${WORKDIR}/third_party/IsaacLab/source/isaaclab*/; do pip install --no-deps -e "$DIR" --break-system-packages; done # Set permissions for Isaac Sim directories that need write access RUN chmod 777 -R /isaac-sim/ -# Pre-install flatdict into kit Python: its setup.py imports pkg_resources which -# is absent from pip's isolated build env, causing isaaclab core install to fail. -RUN /isaac-sim/python.sh -m pip install --no-build-isolation flatdict==4.0.1 -# Install isaaclab -RUN ${ISAACLAB_PATH}/isaaclab.sh -i -# Verify isaaclab core was installed (isaaclab.sh silently swallows failures) -RUN /isaac-sim/python.sh -c "import isaaclab; print(isaaclab.__file__)" +# Build deps and apt packages required by Isaac Lab extensions (see IsaacLab/docker/Dockerfile.base) +RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install toml +RUN ${ISAACLAB_PATH}/isaaclab.sh -p ${ISAACLAB_PATH}/tools/install_deps.py apt ${ISAACLAB_PATH}/source +# flatdict must be pre-installed without build isolation (pkg_resources / setuptools issue). +# Do NOT install setuptools/wheel here — it corrupts pip's vendored packaging in the kit. +# Pre-install isaaclab deps in stages with pinned wheels. Use --no-deps for packages that +# re-trigger pip matplotlib backtracking (pytransform3d, dex-retargeting) after deps exist. +RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary \ + flatdict==4.0.1 \ + "numpy<2" \ + matplotlib==3.10.3 \ + scipy \ + lxml \ + pyyaml \ + anytree \ + trimesh \ + nlopt==2.7.1 \ + pin==2.7.0 \ + qpsolvers==4.8.1 \ + quadprog \ + loop-rate-limiters +RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary --no-deps \ + pytransform3d==3.14.4 +RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary --no-deps \ + pin-pink==3.1.0 && \ + ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary \ + daqp==0.7.2 +RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary --no-deps \ + dex-retargeting==0.4.6 +RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary \ + prettytable==3.3.0 \ + gymnasium==1.2.1 \ + pillow==11.3.0 \ + hidapi==0.14.0.post2 \ + starlette==0.49.1 \ + "pyglet<2" +RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary "onnx>=1.18.0" +RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary warp-lang +RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary einops +# isaaclab_mimic runtime deps (install editable packages with --no-deps below) +RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary \ + tomli ipywidgets==8.1.5 +# transformers omitted here — bulk install segfaults kit Python; GR00T stage installs it when needed. +# Install Isaac Lab core and extensions with --no-deps (runtime deps pre-installed above). +RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --no-deps -e "${ISAACLAB_PATH}/source/isaaclab" && \ + /isaac-sim/python.sh -c "import isaaclab; print(isaaclab.__file__)" && \ + for DIR in ${WORKDIR}/third_party/IsaacLab/source/isaaclab_*/; do \ + ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --no-deps -e "$DIR" || exit 1; \ + done && \ + /isaac-sim/python.sh -c "import isaaclab; import isaaclab_mimic; print(isaaclab.__file__)" # Patch for osqp RUN if python -c "import qpsolvers; print(qpsolvers.available_solvers)" | grep -q "osqp"; then \ echo "OSQP is installed."; \ else \ echo "OSQP missing, installing..."; \ - /isaac-sim/python.sh -m pip install --upgrade pip; \ /isaac-sim/python.sh -m pip install qpsolvers==4.8.1; \ fi -# Install pip dependencies -RUN /isaac-sim/python.sh -m pip install --upgrade pip && \ - /isaac-sim/python.sh -m pip install \ +# Workflow pip deps in one layer. Do not pin typing_extensions or install jupyter here — +# they install standalone `packaging` and break pip's vendored copy in kit Python. +# WebRTC stack required by scripts/utils/webrtc_cam.py (imported at module load by runners). +RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary \ pytest \ - jupyter \ - typing_extensions==4.12.2 \ - onnxruntime - -# lwlabs deps -RUN /isaac-sim/python.sh -m pip install --upgrade pip && \ - /isaac-sim/python.sh -m pip install \ + onnxruntime \ vuer[all] \ - lightwheel-sdk + lightwheel-sdk \ + aiohttp \ + av==12.3.0 \ + aiortc==1.10.1 && \ + ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --force-reinstall --no-deps pip==24.3.1 ENV LW_API_ENDPOINT="https://api-dev.lightwheel.net" @@ -207,7 +249,7 @@ COPY ./third_party/RLinf ${WORKDIR}/third_party/RLinf ################################ COPY ./workflows/rheo ${WORKDIR}/workflows/rheo -RUN echo "export PYTHONPATH=${WORKDIR}/workflows/rheo/scripts:${WORKDIR}/workflows/rheo/scripts/simulation/rl:\$PYTHONPATH" >> /etc/bash.bashrc +RUN echo "export PYTHONPATH=${WORKDIR}/workflows/rheo:${WORKDIR}/workflows/rheo/scripts:${WORKDIR}/workflows/rheo/scripts/simulation/rl:\$PYTHONPATH" >> /etc/bash.bashrc # Set aliases RUN echo "alias python='/isaac-sim/python.sh'" >> /etc/bash.bashrc From 9850d931b8a52e840b0217c74c65aa8007da0506 Mon Sep 17 00:00:00 2001 From: "Diego Vieira (SiDi)" Date: Mon, 15 Jun 2026 13:19:38 -0400 Subject: [PATCH 2/6] update: dockerfile rheo --- workflows/rheo/docker/Dockerfile.x86 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workflows/rheo/docker/Dockerfile.x86 b/workflows/rheo/docker/Dockerfile.x86 index 796d658f..a0156dca 100644 --- a/workflows/rheo/docker/Dockerfile.x86 +++ b/workflows/rheo/docker/Dockerfile.x86 @@ -233,7 +233,8 @@ RUN if [ "$INSTALL_GROOT" = "true" ]; then \ # Set CUDA 12.8 library path permanently for runtime (prepend to existing path) ENV LD_LIBRARY_PATH=/usr/local/cuda-12.8/lib64:/usr/local/cuda/lib64:/usr/lib/x86_64-linux-gnu -################################ + +################################# # Install IsaacLab Arena ################################ COPY ./third_party/IsaacLab-Arena ${WORKDIR}/third_party/IsaacLab-Arena From 24cd4c42cb6fb7d721428600e23b994258d75356 Mon Sep 17 00:00:00 2001 From: "Diego Vieira (SiDi)" Date: Tue, 16 Jun 2026 14:35:06 -0400 Subject: [PATCH 3/6] update: dockerfile rheo --- workflows/rheo/docker/Dockerfile.x86 | 23 ++++++- workflows/rheo/docker/run_docker.sh | 28 ++++++-- .../simulation/examples/observe_runner.py | 16 +++-- .../simulation/examples/policy_runner.py | 14 +++- .../examples/triggered_policy_runner.py | 18 +++-- .../scripts/simulation/register_and_patch.py | 66 +++++++++++++++++-- 6 files changed, 141 insertions(+), 24 deletions(-) diff --git a/workflows/rheo/docker/Dockerfile.x86 b/workflows/rheo/docker/Dockerfile.x86 index a0156dca..ee2eef2b 100644 --- a/workflows/rheo/docker/Dockerfile.x86 +++ b/workflows/rheo/docker/Dockerfile.x86 @@ -25,6 +25,7 @@ ARG WORKDIR="/workspaces" ENV WORKDIR=${WORKDIR} WORKDIR "${WORKDIR}" ENV PYTHONPATH=${WORKDIR}/workflows/rheo:${WORKDIR}/workflows/rheo/scripts:${WORKDIR}/workflows/rheo/scripts/simulation/rl +ENV HEADLESS=1 USER root @@ -94,11 +95,21 @@ RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer starlette==0.49.1 \ "pyglet<2" RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary "onnx>=1.18.0" -RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary warp-lang +# Do NOT pip install warp-lang — Isaac Sim 5.1 ships Warp via omni.warp-1.8.2. A newer +# warp-lang from PyPI shadows the kit copy and breaks wp.types.array / wp.context at startup. RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary einops # isaaclab_mimic runtime deps (install editable packages with --no-deps below) RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary \ tomli ipywidgets==8.1.5 +# isaaclab / isaaclab_tasks / isaaclab_rl runtime deps skipped by --no-deps editable installs +RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary \ + h5py \ + hydra-core \ + tensorboard \ + moviepy \ + numba \ + "protobuf>=4.25.8,!=5.26.0" \ + "packaging<24" # transformers omitted here — bulk install segfaults kit Python; GR00T stage installs it when needed. # Install Isaac Lab core and extensions with --no-deps (runtime deps pre-installed above). RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --no-deps -e "${ISAACLAB_PATH}/source/isaaclab" && \ @@ -109,7 +120,7 @@ RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --no-dep /isaac-sim/python.sh -c "import isaaclab; import isaaclab_mimic; print(isaaclab.__file__)" # Patch for osqp -RUN if python -c "import qpsolvers; print(qpsolvers.available_solvers)" | grep -q "osqp"; then \ +RUN if /isaac-sim/python.sh -c "import qpsolvers; print(qpsolvers.available_solvers)" | grep -q "osqp"; then \ echo "OSQP is installed."; \ else \ echo "OSQP missing, installing..."; \ @@ -118,7 +129,7 @@ RUN if python -c "import qpsolvers; print(qpsolvers.available_solvers)" | grep - # Workflow pip deps in one layer. Do not pin typing_extensions or install jupyter here — # they install standalone `packaging` and break pip's vendored copy in kit Python. -# WebRTC stack required by scripts/utils/webrtc_cam.py (imported at module load by runners). +# WebRTC stack (aiortc/av/aiohttp) — loaded lazily by runners when --webrtc_cam is set. RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary \ pytest \ onnxruntime \ @@ -261,6 +272,12 @@ RUN echo "alias pytest='/isaac-sim/python.sh -m pytest'" >> /etc/bash.bashrc RUN pip3 install debugpy --break-system-packages RUN echo "alias debugpy='python -Xfrozen_modules=off -m debugpy --listen localhost:5678 --wait-for-client'" >> /etc/bash.bashrc +# Strip pip warp-lang after all pip installs — PyPI warp shadows omni.warp-1.8.2 and breaks Kit. +# pip uninstall fails on kit Python (cross-device link / .pyc permissions) — remove directories as root. +RUN KIT_SITE="/isaac-sim/kit/python/lib/python3.11/site-packages" && \ + chmod -R u+w "${KIT_SITE}/warp" "${KIT_SITE}/~arp" 2>/dev/null || true && \ + rm -rf "${KIT_SITE}/warp" "${KIT_SITE}"/warp_lang-*.dist-info "${KIT_SITE}"/~arp* 2>/dev/null || true + # Prompt RUN echo "PS1='[RHEO] \[\e[0;32m\]~\u \[\e[0;34m\]\w\[\e[0m\] \$ '" >> /etc/bash.bashrc RUN echo "alias ll='ls -alF --color=auto'" >> /etc/bash.bashrc diff --git a/workflows/rheo/docker/run_docker.sh b/workflows/rheo/docker/run_docker.sh index 6f3b091e..24ac644f 100755 --- a/workflows/rheo/docker/run_docker.sh +++ b/workflows/rheo/docker/run_docker.sh @@ -3,6 +3,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# Re-exec with bash when invoked as `sh run_docker.sh` (dash does not support [[ or arrays). +if [ -z "${BASH_VERSION:-}" ]; then + exec bash "$0" "$@" +fi + # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -183,6 +188,11 @@ add_volume_if_it_exists() { [ -d "$src" ] && echo "-v $src:$dst" } +# Lightwheel SDK cache (nested dirs required for version metadata .txt files) +mkdir -p "$HOME/.cache/lightwheel_sdk/object/CoffeeMachine037" +mkdir -p "$HOME/.cache/lightwheel_sdk/object/Microwave039" +mkdir -p "$HOME/.cache/lightwheel_sdk/object" + # Run container if [ "$NEW_CONTAINER" = false ] && [ "$( docker container inspect -f '{{.State.Running}}' $CONTAINER_NAME 2>/dev/null)" = "true" ]; then if [ $# -ge 1 ]; then @@ -247,10 +257,20 @@ else fi fi - # X11 - if [ -n "$DISPLAY" ]; then - echo "Allowing X11 connections" - xhost +local:docker > /dev/null + # X11 (optional). Do not abort if xhost fails (SSH/headless); unset DISPLAY so Isaac Sim + # runs headless instead of failing on an invalid display. + if [ -n "${DISPLAY:-}" ]; then + echo "Allowing X11 connections for DISPLAY=${DISPLAY}" + if xhost +local:docker > /dev/null 2>&1; then + echo "X11 forwarding enabled." + else + echo "Warning: xhost failed (no X server on ${DISPLAY}). Running headless." + echo " Use --webrtc_cam and open http://localhost:8080 to view the scene." + unset DISPLAY + fi + fi + if [ -z "${DISPLAY:-}" ]; then + export HEADLESS=1 fi if [ $# -ge 1 ]; then diff --git a/workflows/rheo/scripts/simulation/examples/observe_runner.py b/workflows/rheo/scripts/simulation/examples/observe_runner.py index 3362e5b2..ca89c283 100644 --- a/workflows/rheo/scripts/simulation/examples/observe_runner.py +++ b/workflows/rheo/scripts/simulation/examples/observe_runner.py @@ -22,7 +22,6 @@ from isaaclab_arena.cli.isaaclab_arena_cli import get_isaaclab_arena_cli_parser from isaaclab_arena.utils.isaaclab_utils.simulation_app import SimulationAppContext from isaaclab_arena_environments.cli import add_example_environments_cli_args, get_arena_builder_from_cli -from scripts.utils.webrtc_cam import setup_webrtc_cam from simulation.examples.webrtc_runner_cli import add_webrtc_cli_args from simulation.register_and_patch import register_workflow_assets, register_workflow_cli @@ -44,9 +43,11 @@ def main(): add_example_environments_cli_args(args_parser) args_cli = args_parser.parse_args() - # Start the simulation app + if getattr(args_cli, "enable_pinocchio", False): + import pinocchio # noqa: F401 + + # Start the simulation app (omni.* modules require SimulationApp before isaaclab imports). with SimulationAppContext(args_cli): - # Register Workflow-specific assets register_workflow_assets() # Build environment configuration @@ -83,8 +84,13 @@ def main(): env = gym.make(env_name, cfg=env_cfg).unwrapped print(f"[INFO] Environment created: {env_name}") - # Optional WebRTC livestream of the room camera. - _maybe_publish_webrtc = setup_webrtc_cam(args_cli, camera_name="room_camera") + # Optional WebRTC livestream (lazy import: aiortc/asyncio must not load during Kit startup). + if getattr(args_cli, "webrtc_cam", False): + from scripts.utils.webrtc_cam import setup_webrtc_cam + + _maybe_publish_webrtc = setup_webrtc_cam(args_cli, camera_name="room_camera") + else: + _maybe_publish_webrtc = lambda _obs: None if args_cli.seed is not None: env.seed(args_cli.seed) diff --git a/workflows/rheo/scripts/simulation/examples/policy_runner.py b/workflows/rheo/scripts/simulation/examples/policy_runner.py index d35402b8..cbab55a6 100644 --- a/workflows/rheo/scripts/simulation/examples/policy_runner.py +++ b/workflows/rheo/scripts/simulation/examples/policy_runner.py @@ -23,8 +23,6 @@ from isaaclab_arena.cli.isaaclab_arena_cli import get_isaaclab_arena_cli_parser from isaaclab_arena.utils.isaaclab_utils.simulation_app import SimulationAppContext from isaaclab_arena_environments.cli import get_arena_builder_from_cli -from scripts.utils.policy_tasks import create_success_hold_wrapper -from simulation.examples.policy_runner_cli import create_policy, setup_policy_argument_parser, validate_policy_args from simulation.register_and_patch import register_workflow_assets, register_workflow_cli register_workflow_cli() @@ -36,9 +34,19 @@ def main(): # We do this as the parser is shared between the example environment and policy runner args_cli, _ = args_parser.parse_known_args() + if getattr(args_cli, "enable_pinocchio", False): + import pinocchio # noqa: F401 + # Start the simulation app with SimulationAppContext(args_cli): - # Add policy-related arguments to the parser + # Lazy imports: policy_runner_cli pulls isaaclab (warp) — must load after Kit starts. + from scripts.utils.policy_tasks import create_success_hold_wrapper + from simulation.examples.policy_runner_cli import ( + create_policy, + setup_policy_argument_parser, + validate_policy_args, + ) + register_workflow_assets() args_parser = setup_policy_argument_parser(args_parser) args_cli = args_parser.parse_args() diff --git a/workflows/rheo/scripts/simulation/examples/triggered_policy_runner.py b/workflows/rheo/scripts/simulation/examples/triggered_policy_runner.py index 19f36374..685cb992 100644 --- a/workflows/rheo/scripts/simulation/examples/triggered_policy_runner.py +++ b/workflows/rheo/scripts/simulation/examples/triggered_policy_runner.py @@ -23,14 +23,11 @@ import torch import tqdm from isaaclab_arena.cli.isaaclab_arena_cli import get_isaaclab_arena_cli_parser -from isaaclab_arena.examples.policy_runner_cli import get_isaaclab_arena_environments_cli_parser from isaaclab_arena.utils.isaaclab_utils.simulation_app import SimulationAppContext from isaaclab_arena_environments.cli import get_arena_builder_from_cli from scripts.utils.keyboard_env_reseter import KeyboardHandler, disable_terminations_and_recorders from scripts.utils.trigger_server import RemoteTrigger -from scripts.utils.webrtc_cam import setup_webrtc_cam from simulation.examples.webrtc_runner_cli import add_trigger_cli_args, add_webrtc_cli_args -from simulation.gr00t_closedloop_policy import CustomGr00tClosedloopPolicy from simulation.register_and_patch import register_workflow_assets, register_workflow_cli @@ -136,6 +133,8 @@ def _build_idle_action(obs, env) -> torch.Tensor: def _load_policy(policy_name: str, device: str) -> tuple: """Load a policy by name. Returns (policy, max_steps).""" + from simulation.gr00t_closedloop_policy import CustomGr00tClosedloopPolicy + cfg = POLICY_CONFIGS[policy_name] print(f"[INFO] Loading policy: {policy_name}") policy = CustomGr00tClosedloopPolicy( @@ -168,7 +167,12 @@ def main(): args_parser = get_isaaclab_arena_cli_parser() args_cli, _ = args_parser.parse_known_args() + if getattr(args_cli, "enable_pinocchio", False): + import pinocchio # noqa: F401 + with SimulationAppContext(args_cli): + from isaaclab_arena.examples.policy_runner_cli import get_isaaclab_arena_environments_cli_parser + register_workflow_assets() args_parser = get_isaaclab_arena_environments_cli_parser(args_parser) add_webrtc_cli_args(args_parser) @@ -187,7 +191,13 @@ def main(): env = gym.make(env_name, cfg=env_cfg).unwrapped print(f"[INFO] Environment created: {env_name}") - _maybe_publish_cam = setup_webrtc_cam(args_cli, camera_name="room_camera") + # Lazy import: aiortc/asyncio must not load during Kit startup. + if getattr(args_cli, "webrtc_cam", False): + from scripts.utils.webrtc_cam import setup_webrtc_cam + + _maybe_publish_cam = setup_webrtc_cam(args_cli, camera_name="room_camera") + else: + _maybe_publish_cam = lambda _obs: None if args_cli.seed is not None: env.seed(args_cli.seed) diff --git a/workflows/rheo/scripts/simulation/register_and_patch.py b/workflows/rheo/scripts/simulation/register_and_patch.py index f733b376..9f01698e 100644 --- a/workflows/rheo/scripts/simulation/register_and_patch.py +++ b/workflows/rheo/scripts/simulation/register_and_patch.py @@ -43,9 +43,65 @@ def register_workflow_cli(): ) +def _patch_lightwheel_cache_writes(): + """Ensure nested Lightwheel cache dirs exist before the SDK writes metadata. + + The SDK calls ``open(path, "w")`` for ``{object}/{version}.txt`` without always + creating ``{object}/`` first. Host caches may also mix flat and nested layouts. + """ + import builtins + import os + + if getattr(builtins, "_rheo_lightwheel_open_patch", False): + return + + _orig_open = builtins.open + + def _open_with_lightwheel_cache_dirs(file, *args, **kwargs): + if isinstance(file, (str, bytes, os.PathLike)): + path = os.fspath(file) + marker = f"{os.sep}.cache{os.sep}lightwheel_sdk{os.sep}object{os.sep}" + if marker in path: + parent = os.path.dirname(path) + if parent: + os.makedirs(parent, exist_ok=True) + return _orig_open(file, *args, **kwargs) + + builtins.open = _open_with_lightwheel_cache_dirs + builtins._rheo_lightwheel_open_patch = True + + def register_workflow_assets(): - """Register workflow assets into the global registries for CLI before the simulation app is started.""" - import scripts.teleop_devices # noqa: F401 - import simulation.assets.background_library # noqa: F401 - import simulation.assets.object_library # noqa: F401 - import simulation.embodiments.g1_patched # noqa: F401 + """Register Rheo-specific assets into the global AssetRegistry. + + Arena's ``@register_asset`` decorator calls ``is_registered()``, which normally + triggers ``ensure_assets_registered()`` and imports the full Arena library + (Lightwheel SDK downloads, G1/Pinocchio modules, etc.). That cascade while + SimulationApp is running has caused SIGSEGV in Kit Python. We register workflow + assets first; the full Arena registry loads later via ``get_asset_by_name()``. + """ + from contextlib import contextmanager + + import isaaclab_arena.assets.asset_registry as asset_registry + + @contextmanager + def _skip_eager_arena_registry(): + def _is_registered_local(self, name: str) -> bool: + return name in self._components + + orig_asset = asset_registry.AssetRegistry.is_registered + orig_device = asset_registry.DeviceRegistry.is_registered + asset_registry.AssetRegistry.is_registered = _is_registered_local + asset_registry.DeviceRegistry.is_registered = _is_registered_local + try: + yield + finally: + asset_registry.AssetRegistry.is_registered = orig_asset + asset_registry.DeviceRegistry.is_registered = orig_device + + with _skip_eager_arena_registry(): + _patch_lightwheel_cache_writes() + import scripts.teleop_devices.motion_controllers # noqa: F401 + import simulation.assets.background_library # noqa: F401 + import simulation.assets.object_library # noqa: F401 + import simulation.embodiments.g1_patched # noqa: F401 From 389fd0838de4f377159de8f3cec1bcc95a3b4acf Mon Sep 17 00:00:00 2001 From: "Diego Vieira (SiDi)" Date: Fri, 19 Jun 2026 08:24:05 -0400 Subject: [PATCH 4/6] Updated rheo fine tunning documentation --- workflows/agentic/docs/fine tunning.drawio | 247 ++++++++++++++ .../agentic/docs/fine tunning_eng.drawio | 247 ++++++++++++++ workflows/rheo/README.md | 76 ++++- .../rheo/docs/.$finetune-pipeline.drawio.bkp | 247 ++++++++++++++ .../rheo/docs/finetune-pipeline-en.drawio | 306 ++++++++++++++++++ workflows/rheo/docs/finetune-pipeline.drawio | 247 ++++++++++++++ 6 files changed, 1363 insertions(+), 7 deletions(-) create mode 100644 workflows/agentic/docs/fine tunning.drawio create mode 100644 workflows/agentic/docs/fine tunning_eng.drawio create mode 100644 workflows/rheo/docs/.$finetune-pipeline.drawio.bkp create mode 100644 workflows/rheo/docs/finetune-pipeline-en.drawio create mode 100644 workflows/rheo/docs/finetune-pipeline.drawio diff --git a/workflows/agentic/docs/fine tunning.drawio b/workflows/agentic/docs/fine tunning.drawio new file mode 100644 index 00000000..d072cb71 --- /dev/null +++ b/workflows/agentic/docs/fine tunning.drawio @@ -0,0 +1,247 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/workflows/agentic/docs/fine tunning_eng.drawio b/workflows/agentic/docs/fine tunning_eng.drawio new file mode 100644 index 00000000..a2782f81 --- /dev/null +++ b/workflows/agentic/docs/fine tunning_eng.drawio @@ -0,0 +1,247 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/workflows/rheo/README.md b/workflows/rheo/README.md index 2e5d0241..85a26de8 100644 --- a/workflows/rheo/README.md +++ b/workflows/rheo/README.md @@ -22,6 +22,7 @@ The workflow provides an end-to-end development pipeline for Physical AI in clin - [🚀 Quick Start](#-quick-start) - [🏠 Environment Requirements](#-environment-requirements) - [⚡ Running Workflows](#-running-workflows) +- [🖥 Viewing Isaac Sim](#-viewing-isaac-sim) - [🛠 Troubleshooting](#-troubleshooting) - [📚 Attribution & Citation](#-attribution-and-citation) @@ -105,6 +106,61 @@ hf download nvidia/GR00T-N1.5-RL-Rheo-AssembleTrocar --local-dir $HOME/models/GR ## ⚡ Running Workflows +### 🖥 Viewing Isaac Sim + +Isaac Sim does **not** open when you only enter the Docker shell (`[RHEO] $`). You must run a runner script (for example `observe_runner.py`). The Rheo Docker image sets `HEADLESS=1` by default, so you need an explicit override to show the native GUI window. + +#### Step 1 — Host display setup (run once per desktop session) + +On the machine with a monitor (not a plain SSH session without X11), allow Docker to use your display: + +```bash +xhost +local:docker +echo $DISPLAY # should print :0 or :1 +``` + +#### Step 2 — Run with the Isaac Sim GUI window + +Prefix the `run_docker.sh` command with `HEADLESS=0`, pass `--enable_cameras`, and **do not** pass `--headless`. The window appears after the environment finishes loading (first launch can take several minutes). + +```bash +cd i4h-workflows + +HEADLESS=0 ./workflows/rheo/docker/run_docker.sh \ + python scripts/simulation/examples/observe_runner.py \ + --num_steps 15000 \ + --enable_cameras \ + observe_object \ + --object surgical_tray_no_lid \ + --embodiment g1_wbc_pink +``` + +Apply the same pattern to other runners (policy evaluation, teleop, and so on): `HEADLESS=0` on the host, no `--headless` on the script. + +#### Browser stream (WebRTC alternative) + +If you are on SSH, have no local display, or the GUI crashes on startup, use headless mode with WebRTC and open `http://localhost:8080` in a browser: + +```bash +./workflows/rheo/docker/run_docker.sh \ + python scripts/simulation/examples/observe_runner.py \ + --headless \ + --num_steps 15000 \ + --enable_cameras \ + --webrtc_cam \ + --webrtc_host 0.0.0.0 \ + --webrtc_port 8080 \ + --webrtc_fps 30 \ + observe_object \ + --object surgical_tray_no_lid \ + --embodiment g1_wbc_pink +``` + +| Mode | Host | Script flags | Where you see the scene | +| --- | --- | --- | --- | +| GUI window | `HEADLESS=0`, `xhost +local:docker` | `--enable_cameras` (no `--headless`) | Isaac Sim / Kit window | +| WebRTC | default | `--headless --webrtc_cam ...` | Browser at `http://localhost:8080` | + ### Running Agent Workflow #### Run Physical Agent @@ -126,17 +182,15 @@ This is a minimal example to run the physical agent in Isaac Sim, in which the c --embodiment g1_wbc_joint ``` -OR simply observe the surgical tray: +OR simply observe the surgical tray — see [Viewing Isaac Sim](#-viewing-isaac-sim) for GUI vs WebRTC. GUI example: ```bash -./workflows/rheo/docker/run_docker.sh -g1.6 \ +xhost +local:docker + +HEADLESS=0 ./workflows/rheo/docker/run_docker.sh \ python scripts/simulation/examples/observe_runner.py \ --num_steps 15000 \ --enable_cameras \ - --webrtc_cam \ - --webrtc_host 0.0.0.0 \ - --webrtc_port 8080 \ - --webrtc_fps 30 \ observe_object \ --object surgical_tray_no_lid \ --embodiment g1_wbc_pink @@ -176,7 +230,8 @@ hf download nvidia/GR00T-N1.5-RL-Rheo-AssembleTrocar --local-dir $HOME/models/GR Notes: -- **`--headless`** and **`--enable_cameras`** are Isaac Lab / AppLauncher options (pass them if you need cameras/rendering). +- **GUI vs headless**: The Docker image sets `HEADLESS=1`. Use `HEADLESS=0` on the host and omit `--headless` to open the Isaac Sim window; see [Viewing Isaac Sim](#-viewing-isaac-sim). Use `--headless` (and optionally `--webrtc_cam`) for SSH or browser viewing. +- **`--enable_cameras`** is required when you need camera observations or rendering. - **`--rl_ckpt`** automatically applies runtime patches to `Gr00tPolicy` to ensure consistency with RL post-training modifications made by RLinf. **If you are not using RL-trained checkpoints, DO NOT pass this flag.** The patch modifies: - **Eagle input padding**: Pads `eagle_input_ids` and `eagle_attention_mask` to a fixed length of 850 - **Dropout removal**: Replaces all dropout layers with `nn.Identity()` for deterministic inference @@ -386,6 +441,13 @@ After fine-tuning or reinforcement learning, you can evaluate the success rate o ## 🛠 Troubleshooting +- **Isaac Sim window does not appear**: + - Entering the container alone (`./workflows/rheo/docker/run_docker.sh` with no command) only opens a shell; run a runner script such as `observe_runner.py`. + - The image defaults to `HEADLESS=1`. Set `HEADLESS=0` before `run_docker.sh` and do not pass `--headless` on the script. + - On the host, run `xhost +local:docker` and confirm `echo $DISPLAY` is set (`:0` or `:1`). + - The GUI may take several minutes to appear on first scene load; wait for `[INFO] Environment created:` in the logs. + - If the process segfaults during RTX startup (common on some RTX 50-series GPUs), use the WebRTC path in [Viewing Isaac Sim](#-viewing-isaac-sim) instead. + - **Resetting the VLM Agent**: - After changing the agent configuration or a system restart, you need to reset the UI container to reload the agent configurations. Otherwise, it could lead to unexpected behavior by using the default agent in the VLM-Surgical-Agent-Framework repository. - To reset the UI container, you can run the following command: `./tools/env_setup/install_vlm_surgical_agent_fx.sh -r` diff --git a/workflows/rheo/docs/.$finetune-pipeline.drawio.bkp b/workflows/rheo/docs/.$finetune-pipeline.drawio.bkp new file mode 100644 index 00000000..3478b429 --- /dev/null +++ b/workflows/rheo/docs/.$finetune-pipeline.drawio.bkp @@ -0,0 +1,247 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/workflows/rheo/docs/finetune-pipeline-en.drawio b/workflows/rheo/docs/finetune-pipeline-en.drawio new file mode 100644 index 00000000..b67619e5 --- /dev/null +++ b/workflows/rheo/docs/finetune-pipeline-en.drawio @@ -0,0 +1,306 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/workflows/rheo/docs/finetune-pipeline.drawio b/workflows/rheo/docs/finetune-pipeline.drawio new file mode 100644 index 00000000..051e983d --- /dev/null +++ b/workflows/rheo/docs/finetune-pipeline.drawio @@ -0,0 +1,247 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From e1c8e02e543958de856267d2de058923dada19f4 Mon Sep 17 00:00:00 2001 From: "Diego Vieira (SiDi)" Date: Thu, 25 Jun 2026 10:20:41 -0400 Subject: [PATCH 5/6] Add Blackwell render workarounds and MP4 recording for Rheo runners. Mitigate RTX 50-series crashes via Kit args, conservative render presets, and headless DISPLAY handling; add --save_video to observe/policy runners. --- workflows/rheo/README.md | 1 + workflows/rheo/docker/run_docker.sh | 9 ++ .../examples/eval_assemble_trocar.py | 3 + .../simulation/examples/observe_runner.py | 24 +++++- .../simulation/examples/policy_runner.py | 23 ++++- .../simulation/examples/policy_runner_cli.py | 2 + .../rheo/scripts/simulation/examples/utils.py | 80 ++++++++++++++++- .../simulation/examples/video_runner_cli.py | 30 +++++++ .../record_demos_assemble_trocar.py | 3 + .../scripts/simulation/register_and_patch.py | 2 + .../simulation/rl/rlinf_ext/__init__.py | 7 +- .../g1_assemble_trocar_env_cfg.py | 8 +- .../tasks/g1_observe_object_task.py | 14 +-- .../simulation/tasks/g1_push_cart_task.py | 7 +- .../tasks/g1_tray_pick_and_place_task.py | 7 +- .../rheo/scripts/utils/blackwell_render.py | 85 +++++++++++++++++++ 16 files changed, 276 insertions(+), 29 deletions(-) create mode 100644 workflows/rheo/scripts/simulation/examples/video_runner_cli.py create mode 100644 workflows/rheo/scripts/utils/blackwell_render.py diff --git a/workflows/rheo/README.md b/workflows/rheo/README.md index 85a26de8..7e335aa2 100644 --- a/workflows/rheo/README.md +++ b/workflows/rheo/README.md @@ -447,6 +447,7 @@ After fine-tuning or reinforcement learning, you can evaluate the success rate o - On the host, run `xhost +local:docker` and confirm `echo $DISPLAY` is set (`:0` or `:1`). - The GUI may take several minutes to appear on first scene load; wait for `[INFO] Environment created:` in the logs. - If the process segfaults during RTX startup (common on some RTX 50-series GPUs), use the WebRTC path in [Viewing Isaac Sim](#-viewing-isaac-sim) instead. + - Rheo auto-injects conservative Kit/render settings for Blackwell (RTX 50-series) and unsets `DISPLAY` in Docker unless `HEADLESS=0` and `RHEO_KEEP_DISPLAY=1`. Opt out with `RHEO_DISABLE_BLACKWELL_RENDER_PATCH=1`. Clear shader cache: `rm -rf ~/.cache/ov ~/.nv/ComputeCache`. - **Resetting the VLM Agent**: - After changing the agent configuration or a system restart, you need to reset the UI container to reload the agent configurations. Otherwise, it could lead to unexpected behavior by using the default agent in the VLM-Surgical-Agent-Framework repository. diff --git a/workflows/rheo/docker/run_docker.sh b/workflows/rheo/docker/run_docker.sh index 24ac644f..59ff3169 100755 --- a/workflows/rheo/docker/run_docker.sh +++ b/workflows/rheo/docker/run_docker.sh @@ -257,6 +257,15 @@ else fi fi + # RTX 50-series (Blackwell): forwarding host DISPLAY can destabilize headless RTX/Vulkan init. + if [ "${RHEO_DISABLE_BLACKWELL_RENDER_PATCH:-}" != "1" ] && [ "${RHEO_KEEP_DISPLAY:-}" != "1" ]; then + if [ -n "${DISPLAY:-}" ] && [ "${HEADLESS:-1}" != "0" ]; then + echo "Note: Unsetting DISPLAY for container (RTX 50-series headless stability)." + echo " Use HEADLESS=0 RHEO_KEEP_DISPLAY=1 for Isaac Sim GUI." + unset DISPLAY + fi + fi + # X11 (optional). Do not abort if xhost fails (SSH/headless); unset DISPLAY so Isaac Sim # runs headless instead of failing on an invalid display. if [ -n "${DISPLAY:-}" ]; then diff --git a/workflows/rheo/scripts/simulation/examples/eval_assemble_trocar.py b/workflows/rheo/scripts/simulation/examples/eval_assemble_trocar.py index 7e4708eb..464b15eb 100644 --- a/workflows/rheo/scripts/simulation/examples/eval_assemble_trocar.py +++ b/workflows/rheo/scripts/simulation/examples/eval_assemble_trocar.py @@ -23,6 +23,9 @@ import numpy as np from isaaclab.app import AppLauncher +from scripts.utils.blackwell_render import ensure_blackwell_safe_kit_args + +ensure_blackwell_safe_kit_args() # Add command line arguments parser = argparse.ArgumentParser(description="Evaluate Gr00t Policy in IsaacLab") diff --git a/workflows/rheo/scripts/simulation/examples/observe_runner.py b/workflows/rheo/scripts/simulation/examples/observe_runner.py index ca89c283..6c103f9d 100644 --- a/workflows/rheo/scripts/simulation/examples/observe_runner.py +++ b/workflows/rheo/scripts/simulation/examples/observe_runner.py @@ -22,6 +22,8 @@ from isaaclab_arena.cli.isaaclab_arena_cli import get_isaaclab_arena_cli_parser from isaaclab_arena.utils.isaaclab_utils.simulation_app import SimulationAppContext from isaaclab_arena_environments.cli import add_example_environments_cli_args, get_arena_builder_from_cli +from simulation.examples.utils import create_video_writer_if_requested, maybe_record_video_frame +from simulation.examples.video_runner_cli import add_video_cli_args from simulation.examples.webrtc_runner_cli import add_webrtc_cli_args from simulation.register_and_patch import register_workflow_assets, register_workflow_cli @@ -39,6 +41,7 @@ def main(): ) # Add WebRTC streaming arguments add_webrtc_cli_args(args_parser) + add_video_cli_args(args_parser) # Add environment-specific CLI arguments (task, object, embodiment, etc.) add_example_environments_cli_args(args_parser) args_cli = args_parser.parse_args() @@ -84,6 +87,10 @@ def main(): env = gym.make(env_name, cfg=env_cfg).unwrapped print(f"[INFO] Environment created: {env_name}") + video_writer, video_base_name = create_video_writer_if_requested(args_cli, run_name=env_name) + if video_writer is not None: + print(f"[INFO] Video recording enabled -> {args_cli.video_dir}") + # Optional WebRTC livestream (lazy import: aiortc/asyncio must not load during Kit startup). if getattr(args_cli, "webrtc_cam", False): from scripts.utils.webrtc_cam import setup_webrtc_cam @@ -101,6 +108,12 @@ def main(): obs, _ = env.reset() _maybe_publish_webrtc(obs) + maybe_record_video_frame( + video_writer, + env, + env_index=int(getattr(args_cli, "video_env_id", 0) or 0), + overlay_text=f"observe step 0/{args_cli.num_steps}", + ) # Get action space dimensions action_dim = env.action_space.shape[-1] if len(env.action_space.shape) > 1 else env.action_space.shape[0] @@ -121,16 +134,25 @@ def main(): try: # Run simulation with initial action to maintain posture - for _ in tqdm.tqdm(range(args_cli.num_steps), desc="Running simulation"): + for step_idx in tqdm.tqdm(range(args_cli.num_steps), desc="Running simulation"): with torch.inference_mode(): # Step environment with initial action to maintain initial posture obs, _, _, _, _ = env.step(initial_action) _maybe_publish_webrtc(obs) + maybe_record_video_frame( + video_writer, + env, + env_index=int(getattr(args_cli, "video_env_id", 0) or 0), + overlay_text=f"observe step {step_idx + 1}/{args_cli.num_steps}", + ) except KeyboardInterrupt: print("\n[INFO] Simulation interrupted by user") print("\n[INFO] Simulation complete") + if video_writer is not None: + video_writer.close() + print(f"[INFO] Video saved to: {args_cli.video_dir}/{video_base_name}_*.mp4") env.close() diff --git a/workflows/rheo/scripts/simulation/examples/policy_runner.py b/workflows/rheo/scripts/simulation/examples/policy_runner.py index cbab55a6..2d7c5775 100644 --- a/workflows/rheo/scripts/simulation/examples/policy_runner.py +++ b/workflows/rheo/scripts/simulation/examples/policy_runner.py @@ -46,6 +46,7 @@ def main(): setup_policy_argument_parser, validate_policy_args, ) + from simulation.examples.utils import create_video_writer_if_requested, maybe_record_video_frame register_workflow_assets() args_parser = setup_policy_argument_parser(args_parser) @@ -61,6 +62,11 @@ def main(): env_cfg.recorders = None env = arena_builder.make_registered() + env_unwrapped = getattr(env, "unwrapped", env) + + video_writer, video_base_name = create_video_writer_if_requested(args_cli, run_name=env_name) + if video_writer is not None: + print(f"[INFO] Video recording enabled -> {args_cli.video_dir}") if args_cli.seed is not None: env.seed(args_cli.seed) @@ -69,6 +75,12 @@ def main(): random.seed(args_cli.seed) obs, _ = env.reset() + maybe_record_video_frame( + video_writer, + env_unwrapped, + env_index=int(getattr(args_cli, "video_env_id", 0) or 0), + overlay_text="policy step 0", + ) # Wrap success term with hold logic if specified (AFTER reset) if args_cli.success_hold_steps > 1: @@ -107,10 +119,16 @@ def main(): # NOTE(xinjieyao, 2025-10-07): lazy import to prevent app stalling caused by omni.kit from isaaclab_arena.metrics.metrics import compute_metrics - for _ in tqdm.tqdm(range(num_steps)): + for step_idx in tqdm.tqdm(range(num_steps)): with torch.inference_mode(): actions = policy.get_action(env, obs) obs, _, terminated, truncated, _ = env.step(actions) + maybe_record_video_frame( + video_writer, + env_unwrapped, + env_index=int(getattr(args_cli, "video_env_id", 0) or 0), + overlay_text=f"policy step {step_idx + 1}/{num_steps}", + ) if terminated.any() or truncated.any(): print( f"Resetting policy for terminated env_ids: {terminated.nonzero().flatten()}" @@ -121,6 +139,9 @@ def main(): metrics = compute_metrics(env) print(f"Metrics: {metrics}") + if video_writer is not None: + video_writer.close() + print(f"[INFO] Video saved to: {args_cli.video_dir}/{video_base_name}_*.mp4") env.close() diff --git a/workflows/rheo/scripts/simulation/examples/policy_runner_cli.py b/workflows/rheo/scripts/simulation/examples/policy_runner_cli.py index f7581a9d..2fe4a1a5 100644 --- a/workflows/rheo/scripts/simulation/examples/policy_runner_cli.py +++ b/workflows/rheo/scripts/simulation/examples/policy_runner_cli.py @@ -27,6 +27,7 @@ from isaaclab_arena.policy.policy_base import PolicyBase from isaaclab_arena.policy.replay_action_policy import ReplayActionPolicy from isaaclab_arena.policy.zero_action_policy import ZeroActionPolicy +from simulation.examples.video_runner_cli import add_video_cli_args def setup_policy_argument_parser(args_parser: argparse.ArgumentParser | None = None) -> argparse.ArgumentParser: @@ -51,6 +52,7 @@ def setup_policy_argument_parser(args_parser: argparse.ArgumentParser | None = N add_replay_arguments(args_parser) add_replay_lerobot_arguments(args_parser) add_gr00t_closedloop_arguments(args_parser) + add_video_cli_args(args_parser) return args_parser diff --git a/workflows/rheo/scripts/simulation/examples/utils.py b/workflows/rheo/scripts/simulation/examples/utils.py index 1b9a7330..14bc2657 100644 --- a/workflows/rheo/scripts/simulation/examples/utils.py +++ b/workflows/rheo/scripts/simulation/examples/utils.py @@ -207,13 +207,91 @@ def close(self) -> None: "right_wrist_camera": "video.right_wrist_view", } -# Camera view -> sensor key mapping for video recording +# Camera view -> sensor key mapping for assemble-trocar video recording _VIDEO_CAMERAS = [ ("head", "front_camera"), ("left_wrist", "left_wrist_camera"), ("right_wrist", "right_wrist_camera"), ] +# Arena / locomanip cameras (first match wins per key) +_ARENA_VIDEO_CAMERAS = [ + ("room", "room_camera"), + ("head", "robot_head_cam"), + ("head", "front_camera"), + ("left_wrist", "left_wrist_camera"), + ("right_wrist", "right_wrist_camera"), +] + + +def create_video_writer_if_requested( + args: Any, + *, + run_name: str = "run", +) -> tuple[_MultiViewConcatWriter | None, str | None]: + """Create a video writer when ``args.save_video`` is True.""" + if not bool(getattr(args, "save_video", False)): + return None, None + timestamp = time.strftime("%Y%m%d_%H%M%S") + base_name = f"{timestamp}_{run_name}" + fps = int(getattr(args, "video_fps", 30) or 30) + video_dir = str(getattr(args, "video_dir", "./eval_videos")) + return _MultiViewConcatWriter(video_dir, base_name=base_name, fps=fps), base_name + + +def _scene_has_rgb_camera(env, camera_key: str) -> bool: + try: + cam = env.scene[camera_key] + output = getattr(cam, "data", None) + if output is None or not hasattr(output, "output"): + return False + return "rgb" in output.output + except (KeyError, AttributeError, TypeError): + return False + + +def record_available_cameras( + writer: _MultiViewConcatWriter, + env, + *, + env_index: int = 0, + stream_prefix: str = "", + overlay_text: str | None = None, +) -> None: + """Record every RGB scene camera known for Arena or assemble-trocar envs.""" + recorded: set[str] = set() + first_camera = True + for view, cam_key in _ARENA_VIDEO_CAMERAS + _VIDEO_CAMERAS: + if cam_key in recorded or not _scene_has_rgb_camera(env, cam_key): + continue + recorded.add(cam_key) + stream_key = f"{stream_prefix}{view}" if stream_prefix else view + if stream_key in writer._writers: + stream_key = f"{stream_prefix}{cam_key}" if stream_prefix else cam_key + writer.write_from_scene_camera( + env, + view=view, + camera_key=cam_key, + env_index=env_index, + stream_key=stream_key, + overlay_text=overlay_text if first_camera else None, + ) + first_camera = False + + +def maybe_record_video_frame( + writer: _MultiViewConcatWriter | None, + env, + *, + env_index: int = 0, + overlay_text: str | None = None, +) -> None: + """Render the sim and append one frame per available camera.""" + if writer is None: + return + env.sim.render() + record_available_cameras(writer, env, env_index=env_index, overlay_text=overlay_text) + def process_observation(obs: Dict[str, Any], env, device: str = "cuda") -> Dict[str, torch.Tensor]: """Convert IsaacLab observations to Gr00t policy input format.""" diff --git a/workflows/rheo/scripts/simulation/examples/video_runner_cli.py b/workflows/rheo/scripts/simulation/examples/video_runner_cli.py new file mode 100644 index 00000000..3f63f74d --- /dev/null +++ b/workflows/rheo/scripts/simulation/examples/video_runner_cli.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CLI argument helpers for MP4 video recording during simulation runs.""" + +import argparse + + +def add_video_cli_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: + """Add CLI arguments for saving camera streams to MP4 files.""" + parser.add_argument("--save_video", action="store_true", help="Save simulation camera streams to MP4 files") + parser.add_argument( + "--video_dir", + type=str, + default="./eval_videos", + help="Directory for MP4 output when --save_video is set", + ) + parser.add_argument( + "--video_fps", + type=int, + default=30, + help="Frame rate for saved MP4 files when --save_video is set", + ) + parser.add_argument( + "--video_env_id", + type=int, + default=0, + help="Environment index to record when num_envs > 1 and --save_video is set", + ) + return parser diff --git a/workflows/rheo/scripts/simulation/record_demos_assemble_trocar.py b/workflows/rheo/scripts/simulation/record_demos_assemble_trocar.py index e769aa51..bf1766f2 100644 --- a/workflows/rheo/scripts/simulation/record_demos_assemble_trocar.py +++ b/workflows/rheo/scripts/simulation/record_demos_assemble_trocar.py @@ -23,6 +23,9 @@ import weakref from isaaclab.app import AppLauncher +from scripts.utils.blackwell_render import ensure_blackwell_safe_kit_args + +ensure_blackwell_safe_kit_args() parser = argparse.ArgumentParser(description="Record HDF5 demos with Meta Quest controllers") parser.add_argument("--task", type=str, default="Isaac-Assemble-Trocar-G129-Dex3-Teleop", help="task name") diff --git a/workflows/rheo/scripts/simulation/register_and_patch.py b/workflows/rheo/scripts/simulation/register_and_patch.py index 9f01698e..aa725ae7 100644 --- a/workflows/rheo/scripts/simulation/register_and_patch.py +++ b/workflows/rheo/scripts/simulation/register_and_patch.py @@ -16,6 +16,7 @@ """Helpers to register Workflow-specific assets and environments.""" from isaaclab_arena_environments.cli import ExampleEnvironments +from scripts.utils.blackwell_render import ensure_blackwell_safe_kit_args from simulation.environments.g1_locomanip_observe_object_environment import ObserveObjectEnvironment from simulation.environments.g1_locomanip_push_cart_environment import G1LocomanipPushCartEnvironment from simulation.environments.g1_locomanip_tray_pick_and_place_environment import G1LocomanipTrayPickAndPlaceEnvironment @@ -23,6 +24,7 @@ def register_workflow_cli(): """Register into the global registries for CLI before the simulation app is started.""" + ensure_blackwell_safe_kit_args() if G1LocomanipTrayPickAndPlaceEnvironment.name not in ExampleEnvironments: ExampleEnvironments.update( { diff --git a/workflows/rheo/scripts/simulation/rl/rlinf_ext/__init__.py b/workflows/rheo/scripts/simulation/rl/rlinf_ext/__init__.py index 7f3c22ec..e203de87 100644 --- a/workflows/rheo/scripts/simulation/rl/rlinf_ext/__init__.py +++ b/workflows/rheo/scripts/simulation/rl/rlinf_ext/__init__.py @@ -108,8 +108,13 @@ def __init__(self, cfg, num_envs, seed_offset, total_num_processes, worker_info) def _make_env_function(self): def make_env_isaaclab(): from isaaclab.app import AppLauncher + from scripts.utils.blackwell_render import BLACKWELL_SAFE_KIT_ARGS, blackwell_render_patch_enabled - sim_app = AppLauncher(headless=True, enable_cameras=True).app + kit_args = BLACKWELL_SAFE_KIT_ARGS if blackwell_render_patch_enabled() else None + launcher_kwargs = {"headless": True, "enable_cameras": True} + if kit_args: + launcher_kwargs["kit_args"] = kit_args + sim_app = AppLauncher(**launcher_kwargs).app import gymnasium as gym import simulation.tasks.assemble_trocar # noqa: F401 - triggers gym.register() from isaaclab_tasks.utils import load_cfg_from_registry diff --git a/workflows/rheo/scripts/simulation/tasks/assemble_trocar/g1_assemble_trocar_env_cfg.py b/workflows/rheo/scripts/simulation/tasks/assemble_trocar/g1_assemble_trocar_env_cfg.py index 4272baa7..4b31baae 100644 --- a/workflows/rheo/scripts/simulation/tasks/assemble_trocar/g1_assemble_trocar_env_cfg.py +++ b/workflows/rheo/scripts/simulation/tasks/assemble_trocar/g1_assemble_trocar_env_cfg.py @@ -26,6 +26,7 @@ from isaaclab.scene import InteractiveSceneCfg from isaaclab.sim.spawners.from_files.from_files_cfg import UsdFileCfg from isaaclab.utils import configclass +from scripts.utils.blackwell_render import apply_blackwell_safe_render_cfg from simulation.assets.assets import ( PUNCTURE_DEVICE_XFORM_USD, TRAY_TROCAR_ASSEMBLY_USD, @@ -383,12 +384,7 @@ def __post_init__(self): self.sim.dt = 1 / 200 self.sim.render_interval = self.decimation self.sim.physx.bounce_threshold_velocity = 0.01 - self.sim.render.enable_translucency = True - self.sim.render.carb_settings = { - "rtx.raytracing.fractionalCutoutOpacity": True, - } - self.sim.render.rendering_mode = "quality" - self.sim.render.antialiasing_mode = "DLAA" + apply_blackwell_safe_render_cfg(self.sim.render) @configclass diff --git a/workflows/rheo/scripts/simulation/tasks/g1_observe_object_task.py b/workflows/rheo/scripts/simulation/tasks/g1_observe_object_task.py index 33ac854a..cf3bb1d7 100644 --- a/workflows/rheo/scripts/simulation/tasks/g1_observe_object_task.py +++ b/workflows/rheo/scripts/simulation/tasks/g1_observe_object_task.py @@ -26,6 +26,7 @@ from isaaclab_arena.tasks.g1_locomanip_pick_and_place_task import G1LocomanipPickAndPlaceTask from isaaclab_arena.terms.events import set_object_pose from isaaclab_arena.utils.cameras import get_viewer_cfg_look_at_object +from scripts.utils.blackwell_render import apply_blackwell_safe_render_cfg class ObserveObjectTask(G1LocomanipPickAndPlaceTask): @@ -54,21 +55,12 @@ def get_termination_cfg(self): ) def modify_env_cfg(self, env_cfg): - """Modify environment configuration for better rendering quality and add room camera.""" + """Modify environment configuration and add room camera.""" # Call parent class method if it exists if hasattr(super(), "modify_env_cfg"): env_cfg = super().modify_env_cfg(env_cfg) - # Set rendering configuration for better visual quality - env_cfg.sim.render.rendering_mode = "quality" - env_cfg.sim.render.antialiasing_mode = "DLAA" - - # Enable translucency rendering for transparent/glass objects - env_cfg.sim.render.enable_translucency = True - # Use carb_settings for fractionalCutoutOpacity (for transparent object rendering) - if env_cfg.sim.render.carb_settings is None: - env_cfg.sim.render.carb_settings = {} - env_cfg.sim.render.carb_settings["rtx.raytracing.fractionalCutoutOpacity"] = True + apply_blackwell_safe_render_cfg(env_cfg.sim.render) # Add room camera to scene self._add_room_camera(env_cfg) diff --git a/workflows/rheo/scripts/simulation/tasks/g1_push_cart_task.py b/workflows/rheo/scripts/simulation/tasks/g1_push_cart_task.py index b9ef743d..ec4b22cd 100644 --- a/workflows/rheo/scripts/simulation/tasks/g1_push_cart_task.py +++ b/workflows/rheo/scripts/simulation/tasks/g1_push_cart_task.py @@ -28,6 +28,7 @@ from isaaclab_arena.utils.cameras import get_viewer_cfg_look_at_object from isaaclab_tasks.manager_based.manipulation.stack.mdp import franka_stack_events from simulation.tasks.terminations import object_at_destination +from scripts.utils.blackwell_render import apply_blackwell_safe_render_cfg class G1PushCartTask(G1LocomanipPickAndPlaceTask): @@ -196,14 +197,12 @@ def get_termination_cfg(self): ) def modify_env_cfg(self, env_cfg): - """Modify environment configuration for better rendering quality.""" + """Modify environment configuration for simulation rendering.""" if hasattr(super(), "modify_env_cfg"): env_cfg = super().modify_env_cfg(env_cfg) - # Set rendering configuration for better visibility - env_cfg.sim.render.rendering_mode = "quality" - env_cfg.sim.render.antialiasing_mode = "DLAA" + apply_blackwell_safe_render_cfg(env_cfg.sim.render) return env_cfg diff --git a/workflows/rheo/scripts/simulation/tasks/g1_tray_pick_and_place_task.py b/workflows/rheo/scripts/simulation/tasks/g1_tray_pick_and_place_task.py index 5fccebd4..a14e9bdc 100644 --- a/workflows/rheo/scripts/simulation/tasks/g1_tray_pick_and_place_task.py +++ b/workflows/rheo/scripts/simulation/tasks/g1_tray_pick_and_place_task.py @@ -30,6 +30,7 @@ from isaaclab_arena.tasks.terminations import objects_in_proximity from isaaclab_arena.utils.cameras import get_viewer_cfg_look_at_object from isaaclab_tasks.manager_based.manipulation.stack.mdp import franka_stack_events +from scripts.utils.blackwell_render import apply_blackwell_safe_render_cfg class G1TrayPickPlaceTask(G1LocomanipPickAndPlaceTask): @@ -221,14 +222,12 @@ def get_termination_cfg(self): ) def modify_env_cfg(self, env_cfg): - """Modify environment configuration for better rendering quality.""" + """Modify environment configuration for simulation rendering.""" if hasattr(super(), "modify_env_cfg"): env_cfg = super().modify_env_cfg(env_cfg) - # Set rendering configuration for better visibility - env_cfg.sim.render.rendering_mode = "quality" - env_cfg.sim.render.antialiasing_mode = "DLAA" + apply_blackwell_safe_render_cfg(env_cfg.sim.render) return env_cfg diff --git a/workflows/rheo/scripts/utils/blackwell_render.py b/workflows/rheo/scripts/utils/blackwell_render.py new file mode 100644 index 00000000..91b18691 --- /dev/null +++ b/workflows/rheo/scripts/utils/blackwell_render.py @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Render settings and Kit args for Blackwell GPUs (e.g. RTX 5090) on Isaac Sim 5.1.""" + +from __future__ import annotations + +import os +import sys +from typing import Any + +# Kit flags applied before SimulationApp starts. NGX/DLSS and heavy RTX features can +# segfault on RTX 50-series (Blackwell) with Isaac Sim 5.1 + Vulkan shader compilation. +_BLACKWELL_KIT_FLAGS: tuple[str, ...] = ( + "--/ngx/enabled=false", + "--/rtx-transient/dldenoiser/enabled=false", + "--/rtx-transient/dlssg/enabled=false", + "--/rtx/reflections/enabled=false", + "--/rtx/indirectDiffuse/enabled=false", + "--/rtx/translucency/enabled=false", + "--/rtx/ambientOcclusion/enabled=false", + "--/rtx/raytracing/cached/enabled=false", + "--/rtx/directLighting/sampledLighting/enabled=false", + "--/renderer/multiGpu/maxGpuCount=1", +) + +BLACKWELL_SAFE_KIT_ARGS = " ".join(_BLACKWELL_KIT_FLAGS) + + +def blackwell_render_patch_enabled() -> bool: + """Return False when the user opts out via RHEO_DISABLE_BLACKWELL_RENDER_PATCH.""" + return os.environ.get("RHEO_DISABLE_BLACKWELL_RENDER_PATCH", "").lower() not in ("1", "true", "yes") + + +def _merge_kit_arg_values(existing: str) -> str: + merged = (existing or "").strip() + for flag in _BLACKWELL_KIT_FLAGS: + if flag not in merged: + merged = f"{merged} {flag}".strip() + return merged + + +def ensure_blackwell_safe_kit_args(argv: list[str] | None = None) -> None: + """Inject Kit args to stabilize rendering unless the user already set --kit_args.""" + if not blackwell_render_patch_enabled(): + return + target = sys.argv if argv is None else argv + + for index, arg in enumerate(target): + if arg == "--kit_args": + if index + 1 < len(target): + target[index + 1] = _merge_kit_arg_values(target[index + 1]) + else: + target.insert(index + 1, BLACKWELL_SAFE_KIT_ARGS) + return + if arg.startswith("--kit_args="): + prefix, value = arg.split("=", 1) + target[index] = f"{prefix}={_merge_kit_arg_values(value)}" + return + + # Insert before env subcommands (e.g. observe_object); trailing kit_args are not seen by the parent parser. + target.insert(1, "--kit_args") + target.insert(2, BLACKWELL_SAFE_KIT_ARGS) + + +def apply_blackwell_safe_render_cfg(render_cfg: Any) -> None: + """Tune Isaac Lab RenderCfg for stability on Blackwell GPUs.""" + if not blackwell_render_patch_enabled(): + return + + render_cfg.rendering_mode = "performance" + render_cfg.antialiasing_mode = "Off" + render_cfg.enable_translucency = False + render_cfg.enable_dlssg = False + render_cfg.enable_dl_denoiser = False + render_cfg.enable_reflections = False + render_cfg.enable_global_illumination = False + render_cfg.enable_direct_lighting = False + render_cfg.enable_shadows = False + render_cfg.enable_ambient_occlusion = False + render_cfg.samples_per_pixel = 1 + + carb_settings = dict(render_cfg.carb_settings or {}) + carb_settings.pop("rtx.raytracing.fractionalCutoutOpacity", None) + render_cfg.carb_settings = carb_settings or None From 90951236b9729ec0a1784c44cdc5ed5140a9ef7a Mon Sep 17 00:00:00 2001 From: "Diego Vieira (SiDi)" Date: Thu, 25 Jun 2026 10:21:56 -0400 Subject: [PATCH 6/6] Dockerfile --- workflows/rheo/i4h-workflows.code-workspace | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 workflows/rheo/i4h-workflows.code-workspace diff --git a/workflows/rheo/i4h-workflows.code-workspace b/workflows/rheo/i4h-workflows.code-workspace new file mode 100644 index 00000000..e4e7c681 --- /dev/null +++ b/workflows/rheo/i4h-workflows.code-workspace @@ -0,0 +1,7 @@ +{ + "folders": [ + { + "path": "../.." + } + ] +} \ No newline at end of file