diff --git a/docker/Dockerfile.isaaclab_arena b/docker/Dockerfile.isaaclab_arena index 8384ee3f..43e3cb8f 100644 --- a/docker/Dockerfile.isaaclab_arena +++ b/docker/Dockerfile.isaaclab_arena @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=nvcr.io/nvidia/isaac-sim:5.0.0 +ARG BASE_IMAGE=nvcr.io/nvidia/isaac-sim:5.1.0 FROM ${BASE_IMAGE} @@ -10,6 +10,8 @@ ARG WORKDIR="/workspace" ENV WORKDIR=${WORKDIR} WORKDIR "${WORKDIR}" +USER root + # Hide conflicting Vulkan files, if needed. RUN if [ -e "/usr/share/vulkan" ] && [ -e "/etc/vulkan" ]; then \ mv /usr/share/vulkan /usr/share/vulkan_hidden; \ @@ -24,7 +26,7 @@ RUN apt-get update && apt-get install -y \ python3-pip # Update pip to the latest version -RUN pip3 install --upgrade pip +#RUN pip3 install --upgrade pip ################################ # Install Isaac Lab @@ -38,9 +40,10 @@ ENV TERM=xterm # Symlink isaac sim to IsaacLab RUN ln -s /isaac-sim/ ${WORKDIR}/submodules/IsaacLab/_isaac_sim # Install IsaacLab dependencies -RUN for DIR in ${WORKDIR}/submodules/IsaacLab/source/isaaclab*/; do pip install --no-deps -e "$DIR"; done +RUN for DIR in ${WORKDIR}/submodules/IsaacLab/source/isaaclab*/; do pip install --no-deps -e "$DIR" --break-system-packages; done # Logs and other stuff appear under dist-packages per default, so this dir has to be writeable. -RUN chmod 777 -R /isaac-sim/kit/ +# Make entire /isaac-sim directory accessible for non-root users running python commands. +RUN chmod 777 -R /isaac-sim/ # Install isaaclab RUN ${ISAACLAB_PATH}/isaaclab.sh -i @@ -57,7 +60,7 @@ RUN if python -c "import qpsolvers; print(qpsolvers.available_solvers)" | grep - # Install pip dependencies # NOTE(alexmillane, 2025-07-22): Dependencies are installed in the IsaacSim version of python. -RUN /isaac-sim/python.sh -m pip install --upgrade pip && \ +RUN /isaac-sim/python.sh -m pip install --upgrade pip --break-system-packages && \ /isaac-sim/python.sh -m pip install \ pytest \ jupyter \ @@ -67,7 +70,7 @@ RUN /isaac-sim/python.sh -m pip install --upgrade pip && \ onnxruntime # lwlabs deps -RUN /isaac-sim/python.sh -m pip install --upgrade pip && \ +RUN /isaac-sim/python.sh -m pip install --upgrade pip --break-system-packages && \ /isaac-sim/python.sh -m pip install \ vuer[all] \ lightwheel-sdk @@ -77,7 +80,7 @@ ENV LW_API_ENDPOINT="https://api-dev.lightwheel.net" # HuggingFace for downloading datasets and models. # NOTE(alexmillane, 2025-10-28): For some reason the CLI has issues when installed in the IsaacSim version of python. -RUN pip install huggingface-hub[cli] +RUN pip install huggingface-hub[cli] --break-system-packages # Create alias for hf command to use the system-installed version RUN echo "alias hf='/usr/local/bin/hf'" >> /etc/bash.bashrc @@ -132,7 +135,7 @@ RUN echo "alias pytest='/isaac-sim/python.sh -m pytest'" >> /etc/bash.bashrc # It will pause waiting for the debugger to attach. # 3) Attach to the running container with VSCode using the "Attach to debugpy session" # configuration from the Run and Debug panel. -RUN pip3 install debugpy +RUN pip3 install debugpy --break-system-packages RUN echo "alias debugpy='python -Xfrozen_modules=off -m debugpy --listen localhost:5678 --wait-for-client'" >> /etc/bash.bashrc # Change prompt so it's obvious we're inside the arena container diff --git a/docker/run_docker.sh b/docker/run_docker.sh index b4fb6cef..dc9c7a7f 100755 --- a/docker/run_docker.sh +++ b/docker/run_docker.sh @@ -7,6 +7,9 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) WORKDIR="/workspaces/isaaclab_arena" +# Default OpenXR directory shared with CloudXR runtime (lives in IsaacLab submodule) +OPENXR_HOST_DIR="./submodules/IsaacLab/openxr" + # Default mount directory on the host machine for the datasets DATASETS_HOST_MOUNT_DIRECTORY="$HOME/datasets" # Default mount directory on the host machine for the models diff --git a/isaaclab_arena/scripts/teleop.py b/isaaclab_arena/scripts/teleop.py index a3b31a57..5368f909 100644 --- a/isaaclab_arena/scripts/teleop.py +++ b/isaaclab_arena/scripts/teleop.py @@ -89,7 +89,7 @@ def main() -> None: env_name, env_cfg = arena_builder.build_registered() # modify configuration env_cfg.terminations.time_out = None - if "Lift" in args_cli.task: + if "Lift" in args_cli.example_environment: # set the resampling time range to large number to avoid resampling env_cfg.commands.object_pose.resampling_time_range = (1.0e9, 1.0e9) # add termination condition for reaching the goal otherwise the environment won't reset @@ -105,9 +105,9 @@ def main() -> None: # create environment env = gym.make(env_name, cfg=env_cfg).unwrapped # check environment name (for reach , we don't allow the gripper) - if "Reach" in args_cli.task: + if "Reach" in args_cli.example_environment: omni.log.warn( - f"The environment '{args_cli.task}' does not support gripper control. The device command will be" + f"The environment '{args_cli.example_environment}' does not support gripper control. The device command will be" " ignored." ) except Exception as e: @@ -241,6 +241,13 @@ def stop_teleoperation() -> None: # Only apply teleop commands when active if teleoperation_active: + # Pad action if teleop device outputs fewer dims than environment expects + # actions[:16] are EEF-related, actions[16:] are WBC-related (may need padding) + expected_dim = env.action_space.shape[-1] + if action.shape[0] < expected_dim: + padding = torch.zeros(expected_dim - action.shape[0], device=action.device, dtype=action.dtype) + action = torch.cat([action, padding]) + # process actions actions = action.repeat(env.num_envs, 1) # apply actions diff --git a/isaaclab_arena/teleop_devices/__init__.py b/isaaclab_arena/teleop_devices/__init__.py index 99d4b4b3..964b0db4 100644 --- a/isaaclab_arena/teleop_devices/__init__.py +++ b/isaaclab_arena/teleop_devices/__init__.py @@ -5,4 +5,5 @@ from .avp_handtracking import * from .keyboard import * +from .motion_controllers import * from .spacemouse import * diff --git a/isaaclab_arena/teleop_devices/motion_controllers.py b/isaaclab_arena/teleop_devices/motion_controllers.py new file mode 100644 index 00000000..4258c21d --- /dev/null +++ b/isaaclab_arena/teleop_devices/motion_controllers.py @@ -0,0 +1,70 @@ +# Copyright (c) 2025, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Teleop device for motion controllers (VR controllers like Quest controllers).""" + +from isaaclab.devices.device_base import DevicesCfg +from isaaclab.devices.openxr import OpenXRDeviceCfg +from isaaclab.devices.openxr.retargeters import ( + G1LowerBodyStandingMotionControllerRetargeterCfg, + G1TriHandUpperBodyMotionControllerGripperRetargeterCfg, +) +from isaaclab.devices.openxr.xr_cfg import XrAnchorRotationMode + +from isaaclab_arena.assets.register import register_device +from isaaclab_arena.teleop_devices.teleop_device_base import TeleopDeviceBase + + +@register_device +class MotionControllersTeleopDevice(TeleopDeviceBase): + """ + Teleop device for VR motion controllers (e.g., Quest controllers). + + This device uses motion controllers instead of hand tracking for teleoperation. + It's useful when you have VR controllers but not hand tracking capability. + Currently supports G1 humanoid robot with gripper control via trigger buttons. + """ + + name = "motion_controllers" + + def __init__( + self, + sim_device: str | None = None, + ): + """Initialize motion controllers teleop device. + + Args: + sim_device: The simulation device (e.g., "cuda:0"). + """ + super().__init__(sim_device=sim_device) + + def get_teleop_device_cfg(self, embodiment: object | None = None): + """Get the teleop device configuration. + + Args: + embodiment: The embodiment to use for the teleop device configuration. + + Returns: + DevicesCfg: The device configuration for motion controllers. + """ + xr_cfg = embodiment.get_xr_cfg() + xr_cfg.anchor_rotation_mode = XrAnchorRotationMode.FOLLOW_PRIM_SMOOTHED + + return DevicesCfg( + devices={ + "motion_controllers": OpenXRDeviceCfg( + retargeters=[ + G1TriHandUpperBodyMotionControllerGripperRetargeterCfg( + sim_device=self.sim_device, + ), + G1LowerBodyStandingMotionControllerRetargeterCfg( + sim_device=self.sim_device, + ), + ], + sim_device=self.sim_device, + xr_cfg=xr_cfg, + ), + } + ) diff --git a/submodules/IsaacLab b/submodules/IsaacLab index 6acdd82a..941ebdf4 160000 --- a/submodules/IsaacLab +++ b/submodules/IsaacLab @@ -1 +1 @@ -Subproject commit 6acdd82a1633732d32bb575e3d792e34fdeb437e +Subproject commit 941ebdf4ad1fbf89018777012bdfa4b5944c758f