diff --git a/workflows/agentic/docs/fine tunning.drawio b/workflows/agentic/docs/fine tunning.drawio
new file mode 100644
index 00000000..d072cb71
--- /dev/null
+++ b/workflows/agentic/docs/fine tunning.drawio
@@ -0,0 +1,247 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/workflows/agentic/docs/fine tunning_eng.drawio b/workflows/agentic/docs/fine tunning_eng.drawio
new file mode 100644
index 00000000..a2782f81
--- /dev/null
+++ b/workflows/agentic/docs/fine tunning_eng.drawio
@@ -0,0 +1,247 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/workflows/rheo/README.md b/workflows/rheo/README.md
index 2e5d0241..7e335aa2 100644
--- a/workflows/rheo/README.md
+++ b/workflows/rheo/README.md
@@ -22,6 +22,7 @@ The workflow provides an end-to-end development pipeline for Physical AI in clin
- [🚀 Quick Start](#-quick-start)
- [🏠 Environment Requirements](#-environment-requirements)
- [⚡ Running Workflows](#-running-workflows)
+- [🖥 Viewing Isaac Sim](#-viewing-isaac-sim)
- [🛠 Troubleshooting](#-troubleshooting)
- [📚 Attribution & Citation](#-attribution-and-citation)
@@ -105,6 +106,61 @@ hf download nvidia/GR00T-N1.5-RL-Rheo-AssembleTrocar --local-dir $HOME/models/GR
## ⚡ Running Workflows
+### 🖥 Viewing Isaac Sim
+
+Isaac Sim does **not** open when you only enter the Docker shell (`[RHEO] $`). You must run a runner script (for example `observe_runner.py`). The Rheo Docker image sets `HEADLESS=1` by default, so you need an explicit override to show the native GUI window.
+
+#### Step 1 — Host display setup (run once per desktop session)
+
+On the machine with a monitor (not a plain SSH session without X11), allow Docker to use your display:
+
+```bash
+xhost +local:docker
+echo $DISPLAY # should print :0 or :1
+```
+
+#### Step 2 — Run with the Isaac Sim GUI window
+
+Prefix the `run_docker.sh` command with `HEADLESS=0`, pass `--enable_cameras`, and **do not** pass `--headless`. The window appears after the environment finishes loading (first launch can take several minutes).
+
+```bash
+cd i4h-workflows
+
+HEADLESS=0 ./workflows/rheo/docker/run_docker.sh \
+ python scripts/simulation/examples/observe_runner.py \
+ --num_steps 15000 \
+ --enable_cameras \
+ observe_object \
+ --object surgical_tray_no_lid \
+ --embodiment g1_wbc_pink
+```
+
+Apply the same pattern to other runners (policy evaluation, teleop, and so on): `HEADLESS=0` on the host, no `--headless` on the script.
+
+#### Browser stream (WebRTC alternative)
+
+If you are on SSH, have no local display, or the GUI crashes on startup, use headless mode with WebRTC and open `http://localhost:8080` in a browser:
+
+```bash
+./workflows/rheo/docker/run_docker.sh \
+ python scripts/simulation/examples/observe_runner.py \
+ --headless \
+ --num_steps 15000 \
+ --enable_cameras \
+ --webrtc_cam \
+ --webrtc_host 0.0.0.0 \
+ --webrtc_port 8080 \
+ --webrtc_fps 30 \
+ observe_object \
+ --object surgical_tray_no_lid \
+ --embodiment g1_wbc_pink
+```
+
+| Mode | Host | Script flags | Where you see the scene |
+| --- | --- | --- | --- |
+| GUI window | `HEADLESS=0`, `xhost +local:docker` | `--enable_cameras` (no `--headless`) | Isaac Sim / Kit window |
+| WebRTC | default | `--headless --webrtc_cam ...` | Browser at `http://localhost:8080` |
+
### Running Agent Workflow
#### Run Physical Agent
@@ -126,17 +182,15 @@ This is a minimal example to run the physical agent in Isaac Sim, in which the c
--embodiment g1_wbc_joint
```
-OR simply observe the surgical tray:
+OR simply observe the surgical tray — see [Viewing Isaac Sim](#-viewing-isaac-sim) for GUI vs WebRTC. GUI example:
```bash
-./workflows/rheo/docker/run_docker.sh -g1.6 \
+xhost +local:docker
+
+HEADLESS=0 ./workflows/rheo/docker/run_docker.sh \
python scripts/simulation/examples/observe_runner.py \
--num_steps 15000 \
--enable_cameras \
- --webrtc_cam \
- --webrtc_host 0.0.0.0 \
- --webrtc_port 8080 \
- --webrtc_fps 30 \
observe_object \
--object surgical_tray_no_lid \
--embodiment g1_wbc_pink
@@ -176,7 +230,8 @@ hf download nvidia/GR00T-N1.5-RL-Rheo-AssembleTrocar --local-dir $HOME/models/GR
Notes:
-- **`--headless`** and **`--enable_cameras`** are Isaac Lab / AppLauncher options (pass them if you need cameras/rendering).
+- **GUI vs headless**: The Docker image sets `HEADLESS=1`. Use `HEADLESS=0` on the host and omit `--headless` to open the Isaac Sim window; see [Viewing Isaac Sim](#-viewing-isaac-sim). Use `--headless` (and optionally `--webrtc_cam`) for SSH or browser viewing.
+- **`--enable_cameras`** is required when you need camera observations or rendering.
- **`--rl_ckpt`** automatically applies runtime patches to `Gr00tPolicy` to ensure consistency with RL post-training modifications made by RLinf. **If you are not using RL-trained checkpoints, DO NOT pass this flag.** The patch modifies:
- **Eagle input padding**: Pads `eagle_input_ids` and `eagle_attention_mask` to a fixed length of 850
- **Dropout removal**: Replaces all dropout layers with `nn.Identity()` for deterministic inference
@@ -386,6 +441,14 @@ After fine-tuning or reinforcement learning, you can evaluate the success rate o
## 🛠 Troubleshooting
+- **Isaac Sim window does not appear**:
+ - Entering the container alone (`./workflows/rheo/docker/run_docker.sh` with no command) only opens a shell; run a runner script such as `observe_runner.py`.
+ - The image defaults to `HEADLESS=1`. Set `HEADLESS=0` before `run_docker.sh` and do not pass `--headless` on the script.
+ - On the host, run `xhost +local:docker` and confirm `echo $DISPLAY` is set (`:0` or `:1`).
+ - The GUI may take several minutes to appear on first scene load; wait for `[INFO] Environment created:` in the logs.
+ - If the process segfaults during RTX startup (common on some RTX 50-series GPUs), use the WebRTC path in [Viewing Isaac Sim](#-viewing-isaac-sim) instead.
+ - Rheo auto-injects conservative Kit/render settings for Blackwell (RTX 50-series) and unsets `DISPLAY` in Docker unless `HEADLESS=0` and `RHEO_KEEP_DISPLAY=1`. Opt out with `RHEO_DISABLE_BLACKWELL_RENDER_PATCH=1`. Clear shader cache: `rm -rf ~/.cache/ov ~/.nv/ComputeCache`.
+
- **Resetting the VLM Agent**:
- After changing the agent configuration or a system restart, you need to reset the UI container to reload the agent configurations. Otherwise, it could lead to unexpected behavior by using the default agent in the VLM-Surgical-Agent-Framework repository.
- To reset the UI container, you can run the following command: `./tools/env_setup/install_vlm_surgical_agent_fx.sh -r`
diff --git a/workflows/rheo/docker/Dockerfile.x86 b/workflows/rheo/docker/Dockerfile.x86
index 6bf53bbf..ee2eef2b 100644
--- a/workflows/rheo/docker/Dockerfile.x86
+++ b/workflows/rheo/docker/Dockerfile.x86
@@ -24,7 +24,8 @@ ARG INSTALL_GROOT=false
ARG WORKDIR="/workspaces"
ENV WORKDIR=${WORKDIR}
WORKDIR "${WORKDIR}"
-ENV PYTHONPATH=${WORKDIR}/workflows/rheo/scripts:${WORKDIR}/workflows/rheo/scripts/simulation/rl
+ENV PYTHONPATH=${WORKDIR}/workflows/rheo:${WORKDIR}/workflows/rheo/scripts:${WORKDIR}/workflows/rheo/scripts/simulation/rl
+ENV HEADLESS=1
USER root
@@ -40,7 +41,10 @@ RUN apt-get update && apt-get install -y \
cmake \
sudo \
ffmpeg \
- python3-pip
+ python3-pip \
+ pkg-config \
+ libfreetype6-dev \
+ libpng-dev
# Skip pip upgrade - not needed in Docker and conflicts with PEP 668 in Python 3.12
# RUN pip3 install --upgrade pip
@@ -52,40 +56,89 @@ COPY ./third_party/IsaacLab ${WORKDIR}/third_party/IsaacLab
ENV ISAACLAB_PATH=${WORKDIR}/third_party/IsaacLab
ENV TERM=xterm
RUN ln -s /isaac-sim/ ${WORKDIR}/third_party/IsaacLab/_isaac_sim
-# Install IsaacLab dependencies
-RUN for DIR in ${WORKDIR}/third_party/IsaacLab/source/isaaclab*/; do pip install --no-deps -e "$DIR" --break-system-packages; done
# Set permissions for Isaac Sim directories that need write access
RUN chmod 777 -R /isaac-sim/
-# Pre-install flatdict into kit Python: its setup.py imports pkg_resources which
-# is absent from pip's isolated build env, causing isaaclab core install to fail.
-RUN /isaac-sim/python.sh -m pip install --no-build-isolation flatdict==4.0.1
-# Install isaaclab
-RUN ${ISAACLAB_PATH}/isaaclab.sh -i
-# Verify isaaclab core was installed (isaaclab.sh silently swallows failures)
-RUN /isaac-sim/python.sh -c "import isaaclab; print(isaaclab.__file__)"
+# Build deps and apt packages required by Isaac Lab extensions (see IsaacLab/docker/Dockerfile.base)
+RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install toml
+RUN ${ISAACLAB_PATH}/isaaclab.sh -p ${ISAACLAB_PATH}/tools/install_deps.py apt ${ISAACLAB_PATH}/source
+# flatdict must be pre-installed without build isolation (pkg_resources / setuptools issue).
+# Do NOT install setuptools/wheel here — it corrupts pip's vendored packaging in the kit.
+# Pre-install isaaclab deps in stages with pinned wheels. Use --no-deps for packages that
+# re-trigger pip matplotlib backtracking (pytransform3d, dex-retargeting) after deps exist.
+RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary \
+ flatdict==4.0.1 \
+ "numpy<2" \
+ matplotlib==3.10.3 \
+ scipy \
+ lxml \
+ pyyaml \
+ anytree \
+ trimesh \
+ nlopt==2.7.1 \
+ pin==2.7.0 \
+ qpsolvers==4.8.1 \
+ quadprog \
+ loop-rate-limiters
+RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary --no-deps \
+ pytransform3d==3.14.4
+RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary --no-deps \
+ pin-pink==3.1.0 && \
+ ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary \
+ daqp==0.7.2
+RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary --no-deps \
+ dex-retargeting==0.4.6
+RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary \
+ prettytable==3.3.0 \
+ gymnasium==1.2.1 \
+ pillow==11.3.0 \
+ hidapi==0.14.0.post2 \
+ starlette==0.49.1 \
+ "pyglet<2"
+RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary "onnx>=1.18.0"
+# Do NOT pip install warp-lang — Isaac Sim 5.1 ships Warp via omni.warp-1.8.2. A newer
+# warp-lang from PyPI shadows the kit copy and breaks wp.types.array / wp.context at startup.
+RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary einops
+# isaaclab_mimic runtime deps (install editable packages with --no-deps below)
+RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary \
+ tomli ipywidgets==8.1.5
+# isaaclab / isaaclab_tasks / isaaclab_rl runtime deps skipped by --no-deps editable installs
+RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary \
+ h5py \
+ hydra-core \
+ tensorboard \
+ moviepy \
+ numba \
+ "protobuf>=4.25.8,!=5.26.0" \
+ "packaging<24"
+# transformers omitted here — bulk install segfaults kit Python; GR00T stage installs it when needed.
+# Install Isaac Lab core and extensions with --no-deps (runtime deps pre-installed above).
+RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --no-deps -e "${ISAACLAB_PATH}/source/isaaclab" && \
+ /isaac-sim/python.sh -c "import isaaclab; print(isaaclab.__file__)" && \
+ for DIR in ${WORKDIR}/third_party/IsaacLab/source/isaaclab_*/; do \
+ ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --no-deps -e "$DIR" || exit 1; \
+ done && \
+ /isaac-sim/python.sh -c "import isaaclab; import isaaclab_mimic; print(isaaclab.__file__)"
# Patch for osqp
-RUN if python -c "import qpsolvers; print(qpsolvers.available_solvers)" | grep -q "osqp"; then \
+RUN if /isaac-sim/python.sh -c "import qpsolvers; print(qpsolvers.available_solvers)" | grep -q "osqp"; then \
echo "OSQP is installed."; \
else \
echo "OSQP missing, installing..."; \
- /isaac-sim/python.sh -m pip install --upgrade pip; \
/isaac-sim/python.sh -m pip install qpsolvers==4.8.1; \
fi
-# Install pip dependencies
-RUN /isaac-sim/python.sh -m pip install --upgrade pip && \
- /isaac-sim/python.sh -m pip install \
+# Workflow pip deps in one layer. Do not pin typing_extensions or install jupyter here —
+# they install standalone `packaging` and break pip's vendored copy in kit Python.
+# WebRTC stack (aiortc/av/aiohttp) — loaded lazily by runners when --webrtc_cam is set.
+RUN ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --no-build-isolation --prefer-binary \
pytest \
- jupyter \
- typing_extensions==4.12.2 \
- onnxruntime
-
-# lwlabs deps
-RUN /isaac-sim/python.sh -m pip install --upgrade pip && \
- /isaac-sim/python.sh -m pip install \
+ onnxruntime \
vuer[all] \
- lightwheel-sdk
+ lightwheel-sdk \
+ aiohttp \
+ av==12.3.0 \
+ aiortc==1.10.1 && \
+ ${ISAACLAB_PATH}/isaaclab.sh -p -m pip install --force-reinstall --no-deps pip==24.3.1
ENV LW_API_ENDPOINT="https://api-dev.lightwheel.net"
@@ -191,7 +244,8 @@ RUN if [ "$INSTALL_GROOT" = "true" ]; then \
# Set CUDA 12.8 library path permanently for runtime (prepend to existing path)
ENV LD_LIBRARY_PATH=/usr/local/cuda-12.8/lib64:/usr/local/cuda/lib64:/usr/lib/x86_64-linux-gnu
-################################
+
+#################################
# Install IsaacLab Arena
################################
COPY ./third_party/IsaacLab-Arena ${WORKDIR}/third_party/IsaacLab-Arena
@@ -207,7 +261,7 @@ COPY ./third_party/RLinf ${WORKDIR}/third_party/RLinf
################################
COPY ./workflows/rheo ${WORKDIR}/workflows/rheo
-RUN echo "export PYTHONPATH=${WORKDIR}/workflows/rheo/scripts:${WORKDIR}/workflows/rheo/scripts/simulation/rl:\$PYTHONPATH" >> /etc/bash.bashrc
+RUN echo "export PYTHONPATH=${WORKDIR}/workflows/rheo:${WORKDIR}/workflows/rheo/scripts:${WORKDIR}/workflows/rheo/scripts/simulation/rl:\$PYTHONPATH" >> /etc/bash.bashrc
# Set aliases
RUN echo "alias python='/isaac-sim/python.sh'" >> /etc/bash.bashrc
@@ -218,6 +272,12 @@ RUN echo "alias pytest='/isaac-sim/python.sh -m pytest'" >> /etc/bash.bashrc
RUN pip3 install debugpy --break-system-packages
RUN echo "alias debugpy='python -Xfrozen_modules=off -m debugpy --listen localhost:5678 --wait-for-client'" >> /etc/bash.bashrc
+# Strip pip warp-lang after all pip installs — PyPI warp shadows omni.warp-1.8.2 and breaks Kit.
+# pip uninstall fails on kit Python (cross-device link / .pyc permissions) — remove directories as root.
+RUN KIT_SITE="/isaac-sim/kit/python/lib/python3.11/site-packages" && \
+ chmod -R u+w "${KIT_SITE}/warp" "${KIT_SITE}/~arp" 2>/dev/null || true && \
+ rm -rf "${KIT_SITE}/warp" "${KIT_SITE}"/warp_lang-*.dist-info "${KIT_SITE}"/~arp* 2>/dev/null || true
+
# Prompt
RUN echo "PS1='[RHEO] \[\e[0;32m\]~\u \[\e[0;34m\]\w\[\e[0m\] \$ '" >> /etc/bash.bashrc
RUN echo "alias ll='ls -alF --color=auto'" >> /etc/bash.bashrc
diff --git a/workflows/rheo/docker/run_docker.sh b/workflows/rheo/docker/run_docker.sh
index 6f3b091e..59ff3169 100755
--- a/workflows/rheo/docker/run_docker.sh
+++ b/workflows/rheo/docker/run_docker.sh
@@ -3,6 +3,11 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
+# Re-exec with bash when invoked as `sh run_docker.sh` (dash does not support [[ or arrays).
+if [ -z "${BASH_VERSION:-}" ]; then
+ exec bash "$0" "$@"
+fi
+
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -183,6 +188,11 @@ add_volume_if_it_exists() {
[ -d "$src" ] && echo "-v $src:$dst"
}
+# Lightwheel SDK cache (nested dirs required for version metadata .txt files)
+mkdir -p "$HOME/.cache/lightwheel_sdk/object/CoffeeMachine037"
+mkdir -p "$HOME/.cache/lightwheel_sdk/object/Microwave039"
+mkdir -p "$HOME/.cache/lightwheel_sdk/object"
+
# Run container
if [ "$NEW_CONTAINER" = false ] && [ "$( docker container inspect -f '{{.State.Running}}' $CONTAINER_NAME 2>/dev/null)" = "true" ]; then
if [ $# -ge 1 ]; then
@@ -247,10 +257,29 @@ else
fi
fi
- # X11
- if [ -n "$DISPLAY" ]; then
- echo "Allowing X11 connections"
- xhost +local:docker > /dev/null
+ # RTX 50-series (Blackwell): forwarding host DISPLAY can destabilize headless RTX/Vulkan init.
+ if [ "${RHEO_DISABLE_BLACKWELL_RENDER_PATCH:-}" != "1" ] && [ "${RHEO_KEEP_DISPLAY:-}" != "1" ]; then
+ if [ -n "${DISPLAY:-}" ] && [ "${HEADLESS:-1}" != "0" ]; then
+ echo "Note: Unsetting DISPLAY for container (RTX 50-series headless stability)."
+ echo " Use HEADLESS=0 RHEO_KEEP_DISPLAY=1 for Isaac Sim GUI."
+ unset DISPLAY
+ fi
+ fi
+
+ # X11 (optional). Do not abort if xhost fails (SSH/headless); unset DISPLAY so Isaac Sim
+ # runs headless instead of failing on an invalid display.
+ if [ -n "${DISPLAY:-}" ]; then
+ echo "Allowing X11 connections for DISPLAY=${DISPLAY}"
+ if xhost +local:docker > /dev/null 2>&1; then
+ echo "X11 forwarding enabled."
+ else
+ echo "Warning: xhost failed (no X server on ${DISPLAY}). Running headless."
+ echo " Use --webrtc_cam and open http://localhost:8080 to view the scene."
+ unset DISPLAY
+ fi
+ fi
+ if [ -z "${DISPLAY:-}" ]; then
+ export HEADLESS=1
fi
if [ $# -ge 1 ]; then
diff --git a/workflows/rheo/docs/.$finetune-pipeline.drawio.bkp b/workflows/rheo/docs/.$finetune-pipeline.drawio.bkp
new file mode 100644
index 00000000..3478b429
--- /dev/null
+++ b/workflows/rheo/docs/.$finetune-pipeline.drawio.bkp
@@ -0,0 +1,247 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/workflows/rheo/docs/finetune-pipeline-en.drawio b/workflows/rheo/docs/finetune-pipeline-en.drawio
new file mode 100644
index 00000000..b67619e5
--- /dev/null
+++ b/workflows/rheo/docs/finetune-pipeline-en.drawio
@@ -0,0 +1,306 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/workflows/rheo/docs/finetune-pipeline.drawio b/workflows/rheo/docs/finetune-pipeline.drawio
new file mode 100644
index 00000000..051e983d
--- /dev/null
+++ b/workflows/rheo/docs/finetune-pipeline.drawio
@@ -0,0 +1,247 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/workflows/rheo/i4h-workflows.code-workspace b/workflows/rheo/i4h-workflows.code-workspace
new file mode 100644
index 00000000..e4e7c681
--- /dev/null
+++ b/workflows/rheo/i4h-workflows.code-workspace
@@ -0,0 +1,7 @@
+{
+ "folders": [
+ {
+ "path": "../.."
+ }
+ ]
+}
\ No newline at end of file
diff --git a/workflows/rheo/scripts/simulation/examples/eval_assemble_trocar.py b/workflows/rheo/scripts/simulation/examples/eval_assemble_trocar.py
index 7e4708eb..464b15eb 100644
--- a/workflows/rheo/scripts/simulation/examples/eval_assemble_trocar.py
+++ b/workflows/rheo/scripts/simulation/examples/eval_assemble_trocar.py
@@ -23,6 +23,9 @@
import numpy as np
from isaaclab.app import AppLauncher
+from scripts.utils.blackwell_render import ensure_blackwell_safe_kit_args
+
+ensure_blackwell_safe_kit_args()
# Add command line arguments
parser = argparse.ArgumentParser(description="Evaluate Gr00t Policy in IsaacLab")
diff --git a/workflows/rheo/scripts/simulation/examples/observe_runner.py b/workflows/rheo/scripts/simulation/examples/observe_runner.py
index 3362e5b2..6c103f9d 100644
--- a/workflows/rheo/scripts/simulation/examples/observe_runner.py
+++ b/workflows/rheo/scripts/simulation/examples/observe_runner.py
@@ -22,7 +22,8 @@
from isaaclab_arena.cli.isaaclab_arena_cli import get_isaaclab_arena_cli_parser
from isaaclab_arena.utils.isaaclab_utils.simulation_app import SimulationAppContext
from isaaclab_arena_environments.cli import add_example_environments_cli_args, get_arena_builder_from_cli
-from scripts.utils.webrtc_cam import setup_webrtc_cam
+from simulation.examples.utils import create_video_writer_if_requested, maybe_record_video_frame
+from simulation.examples.video_runner_cli import add_video_cli_args
from simulation.examples.webrtc_runner_cli import add_webrtc_cli_args
from simulation.register_and_patch import register_workflow_assets, register_workflow_cli
@@ -40,13 +41,16 @@ def main():
)
# Add WebRTC streaming arguments
add_webrtc_cli_args(args_parser)
+ add_video_cli_args(args_parser)
# Add environment-specific CLI arguments (task, object, embodiment, etc.)
add_example_environments_cli_args(args_parser)
args_cli = args_parser.parse_args()
- # Start the simulation app
+ if getattr(args_cli, "enable_pinocchio", False):
+ import pinocchio # noqa: F401
+
+ # Start the simulation app (omni.* modules require SimulationApp before isaaclab imports).
with SimulationAppContext(args_cli):
- # Register Workflow-specific assets
register_workflow_assets()
# Build environment configuration
@@ -83,8 +87,17 @@ def main():
env = gym.make(env_name, cfg=env_cfg).unwrapped
print(f"[INFO] Environment created: {env_name}")
- # Optional WebRTC livestream of the room camera.
- _maybe_publish_webrtc = setup_webrtc_cam(args_cli, camera_name="room_camera")
+ video_writer, video_base_name = create_video_writer_if_requested(args_cli, run_name=env_name)
+ if video_writer is not None:
+ print(f"[INFO] Video recording enabled -> {args_cli.video_dir}")
+
+ # Optional WebRTC livestream (lazy import: aiortc/asyncio must not load during Kit startup).
+ if getattr(args_cli, "webrtc_cam", False):
+ from scripts.utils.webrtc_cam import setup_webrtc_cam
+
+ _maybe_publish_webrtc = setup_webrtc_cam(args_cli, camera_name="room_camera")
+ else:
+ _maybe_publish_webrtc = lambda _obs: None
if args_cli.seed is not None:
env.seed(args_cli.seed)
@@ -95,6 +108,12 @@ def main():
obs, _ = env.reset()
_maybe_publish_webrtc(obs)
+ maybe_record_video_frame(
+ video_writer,
+ env,
+ env_index=int(getattr(args_cli, "video_env_id", 0) or 0),
+ overlay_text=f"observe step 0/{args_cli.num_steps}",
+ )
# Get action space dimensions
action_dim = env.action_space.shape[-1] if len(env.action_space.shape) > 1 else env.action_space.shape[0]
@@ -115,16 +134,25 @@ def main():
try:
# Run simulation with initial action to maintain posture
- for _ in tqdm.tqdm(range(args_cli.num_steps), desc="Running simulation"):
+ for step_idx in tqdm.tqdm(range(args_cli.num_steps), desc="Running simulation"):
with torch.inference_mode():
# Step environment with initial action to maintain initial posture
obs, _, _, _, _ = env.step(initial_action)
_maybe_publish_webrtc(obs)
+ maybe_record_video_frame(
+ video_writer,
+ env,
+ env_index=int(getattr(args_cli, "video_env_id", 0) or 0),
+ overlay_text=f"observe step {step_idx + 1}/{args_cli.num_steps}",
+ )
except KeyboardInterrupt:
print("\n[INFO] Simulation interrupted by user")
print("\n[INFO] Simulation complete")
+ if video_writer is not None:
+ video_writer.close()
+ print(f"[INFO] Video saved to: {args_cli.video_dir}/{video_base_name}_*.mp4")
env.close()
diff --git a/workflows/rheo/scripts/simulation/examples/policy_runner.py b/workflows/rheo/scripts/simulation/examples/policy_runner.py
index d35402b8..2d7c5775 100644
--- a/workflows/rheo/scripts/simulation/examples/policy_runner.py
+++ b/workflows/rheo/scripts/simulation/examples/policy_runner.py
@@ -23,8 +23,6 @@
from isaaclab_arena.cli.isaaclab_arena_cli import get_isaaclab_arena_cli_parser
from isaaclab_arena.utils.isaaclab_utils.simulation_app import SimulationAppContext
from isaaclab_arena_environments.cli import get_arena_builder_from_cli
-from scripts.utils.policy_tasks import create_success_hold_wrapper
-from simulation.examples.policy_runner_cli import create_policy, setup_policy_argument_parser, validate_policy_args
from simulation.register_and_patch import register_workflow_assets, register_workflow_cli
register_workflow_cli()
@@ -36,9 +34,20 @@ def main():
# We do this as the parser is shared between the example environment and policy runner
args_cli, _ = args_parser.parse_known_args()
+ if getattr(args_cli, "enable_pinocchio", False):
+ import pinocchio # noqa: F401
+
# Start the simulation app
with SimulationAppContext(args_cli):
- # Add policy-related arguments to the parser
+ # Lazy imports: policy_runner_cli pulls isaaclab (warp) — must load after Kit starts.
+ from scripts.utils.policy_tasks import create_success_hold_wrapper
+ from simulation.examples.policy_runner_cli import (
+ create_policy,
+ setup_policy_argument_parser,
+ validate_policy_args,
+ )
+ from simulation.examples.utils import create_video_writer_if_requested, maybe_record_video_frame
+
register_workflow_assets()
args_parser = setup_policy_argument_parser(args_parser)
args_cli = args_parser.parse_args()
@@ -53,6 +62,11 @@ def main():
env_cfg.recorders = None
env = arena_builder.make_registered()
+ env_unwrapped = getattr(env, "unwrapped", env)
+
+ video_writer, video_base_name = create_video_writer_if_requested(args_cli, run_name=env_name)
+ if video_writer is not None:
+ print(f"[INFO] Video recording enabled -> {args_cli.video_dir}")
if args_cli.seed is not None:
env.seed(args_cli.seed)
@@ -61,6 +75,12 @@ def main():
random.seed(args_cli.seed)
obs, _ = env.reset()
+ maybe_record_video_frame(
+ video_writer,
+ env_unwrapped,
+ env_index=int(getattr(args_cli, "video_env_id", 0) or 0),
+ overlay_text="policy step 0",
+ )
# Wrap success term with hold logic if specified (AFTER reset)
if args_cli.success_hold_steps > 1:
@@ -99,10 +119,16 @@ def main():
# NOTE(xinjieyao, 2025-10-07): lazy import to prevent app stalling caused by omni.kit
from isaaclab_arena.metrics.metrics import compute_metrics
- for _ in tqdm.tqdm(range(num_steps)):
+ for step_idx in tqdm.tqdm(range(num_steps)):
with torch.inference_mode():
actions = policy.get_action(env, obs)
obs, _, terminated, truncated, _ = env.step(actions)
+ maybe_record_video_frame(
+ video_writer,
+ env_unwrapped,
+ env_index=int(getattr(args_cli, "video_env_id", 0) or 0),
+ overlay_text=f"policy step {step_idx + 1}/{num_steps}",
+ )
if terminated.any() or truncated.any():
print(
f"Resetting policy for terminated env_ids: {terminated.nonzero().flatten()}"
@@ -113,6 +139,9 @@ def main():
metrics = compute_metrics(env)
print(f"Metrics: {metrics}")
+ if video_writer is not None:
+ video_writer.close()
+ print(f"[INFO] Video saved to: {args_cli.video_dir}/{video_base_name}_*.mp4")
env.close()
diff --git a/workflows/rheo/scripts/simulation/examples/policy_runner_cli.py b/workflows/rheo/scripts/simulation/examples/policy_runner_cli.py
index f7581a9d..2fe4a1a5 100644
--- a/workflows/rheo/scripts/simulation/examples/policy_runner_cli.py
+++ b/workflows/rheo/scripts/simulation/examples/policy_runner_cli.py
@@ -27,6 +27,7 @@
from isaaclab_arena.policy.policy_base import PolicyBase
from isaaclab_arena.policy.replay_action_policy import ReplayActionPolicy
from isaaclab_arena.policy.zero_action_policy import ZeroActionPolicy
+from simulation.examples.video_runner_cli import add_video_cli_args
def setup_policy_argument_parser(args_parser: argparse.ArgumentParser | None = None) -> argparse.ArgumentParser:
@@ -51,6 +52,7 @@ def setup_policy_argument_parser(args_parser: argparse.ArgumentParser | None = N
add_replay_arguments(args_parser)
add_replay_lerobot_arguments(args_parser)
add_gr00t_closedloop_arguments(args_parser)
+ add_video_cli_args(args_parser)
return args_parser
diff --git a/workflows/rheo/scripts/simulation/examples/triggered_policy_runner.py b/workflows/rheo/scripts/simulation/examples/triggered_policy_runner.py
index 19f36374..685cb992 100644
--- a/workflows/rheo/scripts/simulation/examples/triggered_policy_runner.py
+++ b/workflows/rheo/scripts/simulation/examples/triggered_policy_runner.py
@@ -23,14 +23,11 @@
import torch
import tqdm
from isaaclab_arena.cli.isaaclab_arena_cli import get_isaaclab_arena_cli_parser
-from isaaclab_arena.examples.policy_runner_cli import get_isaaclab_arena_environments_cli_parser
from isaaclab_arena.utils.isaaclab_utils.simulation_app import SimulationAppContext
from isaaclab_arena_environments.cli import get_arena_builder_from_cli
from scripts.utils.keyboard_env_reseter import KeyboardHandler, disable_terminations_and_recorders
from scripts.utils.trigger_server import RemoteTrigger
-from scripts.utils.webrtc_cam import setup_webrtc_cam
from simulation.examples.webrtc_runner_cli import add_trigger_cli_args, add_webrtc_cli_args
-from simulation.gr00t_closedloop_policy import CustomGr00tClosedloopPolicy
from simulation.register_and_patch import register_workflow_assets, register_workflow_cli
@@ -136,6 +133,8 @@ def _build_idle_action(obs, env) -> torch.Tensor:
def _load_policy(policy_name: str, device: str) -> tuple:
"""Load a policy by name. Returns (policy, max_steps)."""
+ from simulation.gr00t_closedloop_policy import CustomGr00tClosedloopPolicy
+
cfg = POLICY_CONFIGS[policy_name]
print(f"[INFO] Loading policy: {policy_name}")
policy = CustomGr00tClosedloopPolicy(
@@ -168,7 +167,12 @@ def main():
args_parser = get_isaaclab_arena_cli_parser()
args_cli, _ = args_parser.parse_known_args()
+ if getattr(args_cli, "enable_pinocchio", False):
+ import pinocchio # noqa: F401
+
with SimulationAppContext(args_cli):
+ from isaaclab_arena.examples.policy_runner_cli import get_isaaclab_arena_environments_cli_parser
+
register_workflow_assets()
args_parser = get_isaaclab_arena_environments_cli_parser(args_parser)
add_webrtc_cli_args(args_parser)
@@ -187,7 +191,13 @@ def main():
env = gym.make(env_name, cfg=env_cfg).unwrapped
print(f"[INFO] Environment created: {env_name}")
- _maybe_publish_cam = setup_webrtc_cam(args_cli, camera_name="room_camera")
+ # Lazy import: aiortc/asyncio must not load during Kit startup.
+ if getattr(args_cli, "webrtc_cam", False):
+ from scripts.utils.webrtc_cam import setup_webrtc_cam
+
+ _maybe_publish_cam = setup_webrtc_cam(args_cli, camera_name="room_camera")
+ else:
+ _maybe_publish_cam = lambda _obs: None
if args_cli.seed is not None:
env.seed(args_cli.seed)
diff --git a/workflows/rheo/scripts/simulation/examples/utils.py b/workflows/rheo/scripts/simulation/examples/utils.py
index 1b9a7330..14bc2657 100644
--- a/workflows/rheo/scripts/simulation/examples/utils.py
+++ b/workflows/rheo/scripts/simulation/examples/utils.py
@@ -207,13 +207,91 @@ def close(self) -> None:
"right_wrist_camera": "video.right_wrist_view",
}
-# Camera view -> sensor key mapping for video recording
+# Camera view -> sensor key mapping for assemble-trocar video recording
_VIDEO_CAMERAS = [
("head", "front_camera"),
("left_wrist", "left_wrist_camera"),
("right_wrist", "right_wrist_camera"),
]
+# Arena / locomanip cameras (first match wins per key)
+_ARENA_VIDEO_CAMERAS = [
+ ("room", "room_camera"),
+ ("head", "robot_head_cam"),
+ ("head", "front_camera"),
+ ("left_wrist", "left_wrist_camera"),
+ ("right_wrist", "right_wrist_camera"),
+]
+
+
+def create_video_writer_if_requested(
+ args: Any,
+ *,
+ run_name: str = "run",
+) -> tuple[_MultiViewConcatWriter | None, str | None]:
+ """Create a video writer when ``args.save_video`` is True."""
+ if not bool(getattr(args, "save_video", False)):
+ return None, None
+ timestamp = time.strftime("%Y%m%d_%H%M%S")
+ base_name = f"{timestamp}_{run_name}"
+ fps = int(getattr(args, "video_fps", 30) or 30)
+ video_dir = str(getattr(args, "video_dir", "./eval_videos"))
+ return _MultiViewConcatWriter(video_dir, base_name=base_name, fps=fps), base_name
+
+
+def _scene_has_rgb_camera(env, camera_key: str) -> bool:
+ try:
+ cam = env.scene[camera_key]
+ output = getattr(cam, "data", None)
+ if output is None or not hasattr(output, "output"):
+ return False
+ return "rgb" in output.output
+ except (KeyError, AttributeError, TypeError):
+ return False
+
+
+def record_available_cameras(
+ writer: _MultiViewConcatWriter,
+ env,
+ *,
+ env_index: int = 0,
+ stream_prefix: str = "",
+ overlay_text: str | None = None,
+) -> None:
+ """Record every RGB scene camera known for Arena or assemble-trocar envs."""
+ recorded: set[str] = set()
+ first_camera = True
+ for view, cam_key in _ARENA_VIDEO_CAMERAS + _VIDEO_CAMERAS:
+ if cam_key in recorded or not _scene_has_rgb_camera(env, cam_key):
+ continue
+ recorded.add(cam_key)
+ stream_key = f"{stream_prefix}{view}" if stream_prefix else view
+ if stream_key in writer._writers:
+ stream_key = f"{stream_prefix}{cam_key}" if stream_prefix else cam_key
+ writer.write_from_scene_camera(
+ env,
+ view=view,
+ camera_key=cam_key,
+ env_index=env_index,
+ stream_key=stream_key,
+ overlay_text=overlay_text if first_camera else None,
+ )
+ first_camera = False
+
+
+def maybe_record_video_frame(
+ writer: _MultiViewConcatWriter | None,
+ env,
+ *,
+ env_index: int = 0,
+ overlay_text: str | None = None,
+) -> None:
+ """Render the sim and append one frame per available camera."""
+ if writer is None:
+ return
+ env.sim.render()
+ record_available_cameras(writer, env, env_index=env_index, overlay_text=overlay_text)
+
def process_observation(obs: Dict[str, Any], env, device: str = "cuda") -> Dict[str, torch.Tensor]:
"""Convert IsaacLab observations to Gr00t policy input format."""
diff --git a/workflows/rheo/scripts/simulation/examples/video_runner_cli.py b/workflows/rheo/scripts/simulation/examples/video_runner_cli.py
new file mode 100644
index 00000000..3f63f74d
--- /dev/null
+++ b/workflows/rheo/scripts/simulation/examples/video_runner_cli.py
@@ -0,0 +1,30 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""CLI argument helpers for MP4 video recording during simulation runs."""
+
+import argparse
+
+
+def add_video_cli_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
+ """Add CLI arguments for saving camera streams to MP4 files."""
+ parser.add_argument("--save_video", action="store_true", help="Save simulation camera streams to MP4 files")
+ parser.add_argument(
+ "--video_dir",
+ type=str,
+ default="./eval_videos",
+ help="Directory for MP4 output when --save_video is set",
+ )
+ parser.add_argument(
+ "--video_fps",
+ type=int,
+ default=30,
+ help="Frame rate for saved MP4 files when --save_video is set",
+ )
+ parser.add_argument(
+ "--video_env_id",
+ type=int,
+ default=0,
+ help="Environment index to record when num_envs > 1 and --save_video is set",
+ )
+ return parser
diff --git a/workflows/rheo/scripts/simulation/record_demos_assemble_trocar.py b/workflows/rheo/scripts/simulation/record_demos_assemble_trocar.py
index e769aa51..bf1766f2 100644
--- a/workflows/rheo/scripts/simulation/record_demos_assemble_trocar.py
+++ b/workflows/rheo/scripts/simulation/record_demos_assemble_trocar.py
@@ -23,6 +23,9 @@
import weakref
from isaaclab.app import AppLauncher
+from scripts.utils.blackwell_render import ensure_blackwell_safe_kit_args
+
+ensure_blackwell_safe_kit_args()
parser = argparse.ArgumentParser(description="Record HDF5 demos with Meta Quest controllers")
parser.add_argument("--task", type=str, default="Isaac-Assemble-Trocar-G129-Dex3-Teleop", help="task name")
diff --git a/workflows/rheo/scripts/simulation/register_and_patch.py b/workflows/rheo/scripts/simulation/register_and_patch.py
index f733b376..aa725ae7 100644
--- a/workflows/rheo/scripts/simulation/register_and_patch.py
+++ b/workflows/rheo/scripts/simulation/register_and_patch.py
@@ -16,6 +16,7 @@
"""Helpers to register Workflow-specific assets and environments."""
from isaaclab_arena_environments.cli import ExampleEnvironments
+from scripts.utils.blackwell_render import ensure_blackwell_safe_kit_args
from simulation.environments.g1_locomanip_observe_object_environment import ObserveObjectEnvironment
from simulation.environments.g1_locomanip_push_cart_environment import G1LocomanipPushCartEnvironment
from simulation.environments.g1_locomanip_tray_pick_and_place_environment import G1LocomanipTrayPickAndPlaceEnvironment
@@ -23,6 +24,7 @@
def register_workflow_cli():
"""Register into the global registries for CLI before the simulation app is started."""
+ ensure_blackwell_safe_kit_args()
if G1LocomanipTrayPickAndPlaceEnvironment.name not in ExampleEnvironments:
ExampleEnvironments.update(
{
@@ -43,9 +45,65 @@ def register_workflow_cli():
)
+def _patch_lightwheel_cache_writes():
+ """Ensure nested Lightwheel cache dirs exist before the SDK writes metadata.
+
+ The SDK calls ``open(path, "w")`` for ``{object}/{version}.txt`` without always
+ creating ``{object}/`` first. Host caches may also mix flat and nested layouts.
+ """
+ import builtins
+ import os
+
+ if getattr(builtins, "_rheo_lightwheel_open_patch", False):
+ return
+
+ _orig_open = builtins.open
+
+ def _open_with_lightwheel_cache_dirs(file, *args, **kwargs):
+ if isinstance(file, (str, bytes, os.PathLike)):
+ path = os.fspath(file)
+ marker = f"{os.sep}.cache{os.sep}lightwheel_sdk{os.sep}object{os.sep}"
+ if marker in path:
+ parent = os.path.dirname(path)
+ if parent:
+ os.makedirs(parent, exist_ok=True)
+ return _orig_open(file, *args, **kwargs)
+
+ builtins.open = _open_with_lightwheel_cache_dirs
+ builtins._rheo_lightwheel_open_patch = True
+
+
def register_workflow_assets():
- """Register workflow assets into the global registries for CLI before the simulation app is started."""
- import scripts.teleop_devices # noqa: F401
- import simulation.assets.background_library # noqa: F401
- import simulation.assets.object_library # noqa: F401
- import simulation.embodiments.g1_patched # noqa: F401
+ """Register Rheo-specific assets into the global AssetRegistry.
+
+ Arena's ``@register_asset`` decorator calls ``is_registered()``, which normally
+ triggers ``ensure_assets_registered()`` and imports the full Arena library
+ (Lightwheel SDK downloads, G1/Pinocchio modules, etc.). That cascade while
+ SimulationApp is running has caused SIGSEGV in Kit Python. We register workflow
+ assets first; the full Arena registry loads later via ``get_asset_by_name()``.
+ """
+ from contextlib import contextmanager
+
+ import isaaclab_arena.assets.asset_registry as asset_registry
+
+ @contextmanager
+ def _skip_eager_arena_registry():
+ def _is_registered_local(self, name: str) -> bool:
+ return name in self._components
+
+ orig_asset = asset_registry.AssetRegistry.is_registered
+ orig_device = asset_registry.DeviceRegistry.is_registered
+ asset_registry.AssetRegistry.is_registered = _is_registered_local
+ asset_registry.DeviceRegistry.is_registered = _is_registered_local
+ try:
+ yield
+ finally:
+ asset_registry.AssetRegistry.is_registered = orig_asset
+ asset_registry.DeviceRegistry.is_registered = orig_device
+
+ with _skip_eager_arena_registry():
+ _patch_lightwheel_cache_writes()
+ import scripts.teleop_devices.motion_controllers # noqa: F401
+ import simulation.assets.background_library # noqa: F401
+ import simulation.assets.object_library # noqa: F401
+ import simulation.embodiments.g1_patched # noqa: F401
diff --git a/workflows/rheo/scripts/simulation/rl/rlinf_ext/__init__.py b/workflows/rheo/scripts/simulation/rl/rlinf_ext/__init__.py
index 7f3c22ec..e203de87 100644
--- a/workflows/rheo/scripts/simulation/rl/rlinf_ext/__init__.py
+++ b/workflows/rheo/scripts/simulation/rl/rlinf_ext/__init__.py
@@ -108,8 +108,13 @@ def __init__(self, cfg, num_envs, seed_offset, total_num_processes, worker_info)
def _make_env_function(self):
def make_env_isaaclab():
from isaaclab.app import AppLauncher
+ from scripts.utils.blackwell_render import BLACKWELL_SAFE_KIT_ARGS, blackwell_render_patch_enabled
- sim_app = AppLauncher(headless=True, enable_cameras=True).app
+ kit_args = BLACKWELL_SAFE_KIT_ARGS if blackwell_render_patch_enabled() else None
+ launcher_kwargs = {"headless": True, "enable_cameras": True}
+ if kit_args:
+ launcher_kwargs["kit_args"] = kit_args
+ sim_app = AppLauncher(**launcher_kwargs).app
import gymnasium as gym
import simulation.tasks.assemble_trocar # noqa: F401 - triggers gym.register()
from isaaclab_tasks.utils import load_cfg_from_registry
diff --git a/workflows/rheo/scripts/simulation/tasks/assemble_trocar/g1_assemble_trocar_env_cfg.py b/workflows/rheo/scripts/simulation/tasks/assemble_trocar/g1_assemble_trocar_env_cfg.py
index 4272baa7..4b31baae 100644
--- a/workflows/rheo/scripts/simulation/tasks/assemble_trocar/g1_assemble_trocar_env_cfg.py
+++ b/workflows/rheo/scripts/simulation/tasks/assemble_trocar/g1_assemble_trocar_env_cfg.py
@@ -26,6 +26,7 @@
from isaaclab.scene import InteractiveSceneCfg
from isaaclab.sim.spawners.from_files.from_files_cfg import UsdFileCfg
from isaaclab.utils import configclass
+from scripts.utils.blackwell_render import apply_blackwell_safe_render_cfg
from simulation.assets.assets import (
PUNCTURE_DEVICE_XFORM_USD,
TRAY_TROCAR_ASSEMBLY_USD,
@@ -383,12 +384,7 @@ def __post_init__(self):
self.sim.dt = 1 / 200
self.sim.render_interval = self.decimation
self.sim.physx.bounce_threshold_velocity = 0.01
- self.sim.render.enable_translucency = True
- self.sim.render.carb_settings = {
- "rtx.raytracing.fractionalCutoutOpacity": True,
- }
- self.sim.render.rendering_mode = "quality"
- self.sim.render.antialiasing_mode = "DLAA"
+ apply_blackwell_safe_render_cfg(self.sim.render)
@configclass
diff --git a/workflows/rheo/scripts/simulation/tasks/g1_observe_object_task.py b/workflows/rheo/scripts/simulation/tasks/g1_observe_object_task.py
index 33ac854a..cf3bb1d7 100644
--- a/workflows/rheo/scripts/simulation/tasks/g1_observe_object_task.py
+++ b/workflows/rheo/scripts/simulation/tasks/g1_observe_object_task.py
@@ -26,6 +26,7 @@
from isaaclab_arena.tasks.g1_locomanip_pick_and_place_task import G1LocomanipPickAndPlaceTask
from isaaclab_arena.terms.events import set_object_pose
from isaaclab_arena.utils.cameras import get_viewer_cfg_look_at_object
+from scripts.utils.blackwell_render import apply_blackwell_safe_render_cfg
class ObserveObjectTask(G1LocomanipPickAndPlaceTask):
@@ -54,21 +55,12 @@ def get_termination_cfg(self):
)
def modify_env_cfg(self, env_cfg):
- """Modify environment configuration for better rendering quality and add room camera."""
+ """Modify environment configuration and add room camera."""
# Call parent class method if it exists
if hasattr(super(), "modify_env_cfg"):
env_cfg = super().modify_env_cfg(env_cfg)
- # Set rendering configuration for better visual quality
- env_cfg.sim.render.rendering_mode = "quality"
- env_cfg.sim.render.antialiasing_mode = "DLAA"
-
- # Enable translucency rendering for transparent/glass objects
- env_cfg.sim.render.enable_translucency = True
- # Use carb_settings for fractionalCutoutOpacity (for transparent object rendering)
- if env_cfg.sim.render.carb_settings is None:
- env_cfg.sim.render.carb_settings = {}
- env_cfg.sim.render.carb_settings["rtx.raytracing.fractionalCutoutOpacity"] = True
+ apply_blackwell_safe_render_cfg(env_cfg.sim.render)
# Add room camera to scene
self._add_room_camera(env_cfg)
diff --git a/workflows/rheo/scripts/simulation/tasks/g1_push_cart_task.py b/workflows/rheo/scripts/simulation/tasks/g1_push_cart_task.py
index b9ef743d..ec4b22cd 100644
--- a/workflows/rheo/scripts/simulation/tasks/g1_push_cart_task.py
+++ b/workflows/rheo/scripts/simulation/tasks/g1_push_cart_task.py
@@ -28,6 +28,7 @@
from isaaclab_arena.utils.cameras import get_viewer_cfg_look_at_object
from isaaclab_tasks.manager_based.manipulation.stack.mdp import franka_stack_events
from simulation.tasks.terminations import object_at_destination
+from scripts.utils.blackwell_render import apply_blackwell_safe_render_cfg
class G1PushCartTask(G1LocomanipPickAndPlaceTask):
@@ -196,14 +197,12 @@ def get_termination_cfg(self):
)
def modify_env_cfg(self, env_cfg):
- """Modify environment configuration for better rendering quality."""
+ """Modify environment configuration for simulation rendering."""
if hasattr(super(), "modify_env_cfg"):
env_cfg = super().modify_env_cfg(env_cfg)
- # Set rendering configuration for better visibility
- env_cfg.sim.render.rendering_mode = "quality"
- env_cfg.sim.render.antialiasing_mode = "DLAA"
+ apply_blackwell_safe_render_cfg(env_cfg.sim.render)
return env_cfg
diff --git a/workflows/rheo/scripts/simulation/tasks/g1_tray_pick_and_place_task.py b/workflows/rheo/scripts/simulation/tasks/g1_tray_pick_and_place_task.py
index 5fccebd4..a14e9bdc 100644
--- a/workflows/rheo/scripts/simulation/tasks/g1_tray_pick_and_place_task.py
+++ b/workflows/rheo/scripts/simulation/tasks/g1_tray_pick_and_place_task.py
@@ -30,6 +30,7 @@
from isaaclab_arena.tasks.terminations import objects_in_proximity
from isaaclab_arena.utils.cameras import get_viewer_cfg_look_at_object
from isaaclab_tasks.manager_based.manipulation.stack.mdp import franka_stack_events
+from scripts.utils.blackwell_render import apply_blackwell_safe_render_cfg
class G1TrayPickPlaceTask(G1LocomanipPickAndPlaceTask):
@@ -221,14 +222,12 @@ def get_termination_cfg(self):
)
def modify_env_cfg(self, env_cfg):
- """Modify environment configuration for better rendering quality."""
+ """Modify environment configuration for simulation rendering."""
if hasattr(super(), "modify_env_cfg"):
env_cfg = super().modify_env_cfg(env_cfg)
- # Set rendering configuration for better visibility
- env_cfg.sim.render.rendering_mode = "quality"
- env_cfg.sim.render.antialiasing_mode = "DLAA"
+ apply_blackwell_safe_render_cfg(env_cfg.sim.render)
return env_cfg
diff --git a/workflows/rheo/scripts/utils/blackwell_render.py b/workflows/rheo/scripts/utils/blackwell_render.py
new file mode 100644
index 00000000..91b18691
--- /dev/null
+++ b/workflows/rheo/scripts/utils/blackwell_render.py
@@ -0,0 +1,85 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Render settings and Kit args for Blackwell GPUs (e.g. RTX 5090) on Isaac Sim 5.1."""
+
+from __future__ import annotations
+
+import os
+import sys
+from typing import Any
+
+# Kit flags applied before SimulationApp starts. NGX/DLSS and heavy RTX features can
+# segfault on RTX 50-series (Blackwell) with Isaac Sim 5.1 + Vulkan shader compilation.
+_BLACKWELL_KIT_FLAGS: tuple[str, ...] = (
+ "--/ngx/enabled=false",
+ "--/rtx-transient/dldenoiser/enabled=false",
+ "--/rtx-transient/dlssg/enabled=false",
+ "--/rtx/reflections/enabled=false",
+ "--/rtx/indirectDiffuse/enabled=false",
+ "--/rtx/translucency/enabled=false",
+ "--/rtx/ambientOcclusion/enabled=false",
+ "--/rtx/raytracing/cached/enabled=false",
+ "--/rtx/directLighting/sampledLighting/enabled=false",
+ "--/renderer/multiGpu/maxGpuCount=1",
+)
+
+BLACKWELL_SAFE_KIT_ARGS = " ".join(_BLACKWELL_KIT_FLAGS)
+
+
+def blackwell_render_patch_enabled() -> bool:
+ """Return False when the user opts out via RHEO_DISABLE_BLACKWELL_RENDER_PATCH."""
+ return os.environ.get("RHEO_DISABLE_BLACKWELL_RENDER_PATCH", "").lower() not in ("1", "true", "yes")
+
+
+def _merge_kit_arg_values(existing: str) -> str:
+ merged = (existing or "").strip()
+ for flag in _BLACKWELL_KIT_FLAGS:
+ if flag not in merged:
+ merged = f"{merged} {flag}".strip()
+ return merged
+
+
+def ensure_blackwell_safe_kit_args(argv: list[str] | None = None) -> None:
+ """Inject Kit args to stabilize rendering unless the user already set --kit_args."""
+ if not blackwell_render_patch_enabled():
+ return
+ target = sys.argv if argv is None else argv
+
+ for index, arg in enumerate(target):
+ if arg == "--kit_args":
+ if index + 1 < len(target):
+ target[index + 1] = _merge_kit_arg_values(target[index + 1])
+ else:
+ target.insert(index + 1, BLACKWELL_SAFE_KIT_ARGS)
+ return
+ if arg.startswith("--kit_args="):
+ prefix, value = arg.split("=", 1)
+ target[index] = f"{prefix}={_merge_kit_arg_values(value)}"
+ return
+
+ # Insert before env subcommands (e.g. observe_object); trailing kit_args are not seen by the parent parser.
+ target.insert(1, "--kit_args")
+ target.insert(2, BLACKWELL_SAFE_KIT_ARGS)
+
+
+def apply_blackwell_safe_render_cfg(render_cfg: Any) -> None:
+ """Tune Isaac Lab RenderCfg for stability on Blackwell GPUs."""
+ if not blackwell_render_patch_enabled():
+ return
+
+ render_cfg.rendering_mode = "performance"
+ render_cfg.antialiasing_mode = "Off"
+ render_cfg.enable_translucency = False
+ render_cfg.enable_dlssg = False
+ render_cfg.enable_dl_denoiser = False
+ render_cfg.enable_reflections = False
+ render_cfg.enable_global_illumination = False
+ render_cfg.enable_direct_lighting = False
+ render_cfg.enable_shadows = False
+ render_cfg.enable_ambient_occlusion = False
+ render_cfg.samples_per_pixel = 1
+
+ carb_settings = dict(render_cfg.carb_settings or {})
+ carb_settings.pop("rtx.raytracing.fractionalCutoutOpacity", None)
+ render_cfg.carb_settings = carb_settings or None