diff --git a/apps/isaaclab.python.kit b/apps/isaaclab.python.kit index 2435019c89c..49610dd8ff9 100644 --- a/apps/isaaclab.python.kit +++ b/apps/isaaclab.python.kit @@ -38,7 +38,7 @@ keywords = ["experience", "app", "usd"] # Isaac Sim Extra "isaacsim.asset.importer.mjcf" = {} -"isaacsim.asset.importer.urdf" = {version = "2.4.31", exact = true} +"isaacsim.asset.importer.urdf" = {} "omni.physx.bundle" = {} "omni.physx.tensors" = {} "omni.replicator.core" = {} diff --git a/docs/source/experimental-features/newton-physics-integration/visualization.rst b/docs/source/experimental-features/newton-physics-integration/visualization.rst index f5443573393..b50d3467f83 100644 --- a/docs/source/experimental-features/newton-physics-integration/visualization.rst +++ b/docs/source/experimental-features/newton-physics-integration/visualization.rst @@ -5,7 +5,7 @@ Visualization Isaac Lab offers several lightweight visualizers for real-time simulation inspection and debugging. Unlike renderers that process sensor data, visualizers are meant for fast, interactive feedback. -You can use any visualizer regardless of your chosen physics engine or rendering backend. +You can launch any number of visualizers at once, and they work with any physics engine or rendering backend. Overview @@ -31,7 +31,7 @@ Isaac Lab supports three visualizer backends, each optimized for different use c - Webviewer, time scrubbing, recording export -*The following visualizers are shown training the Isaac-Velocity-Flat-Anymal-D-v0 environment.* +*The following visualizers are shown training Isaac-Velocity-Flat-Anymal-D-v0 with 4096 concurrent environments.* .. figure:: ../../_static/visualizers/ov_viz.jpg :width: 100% @@ -139,8 +139,8 @@ Omniverse Visualizer window_height=720, # Viewport height in pixels # Camera settings - camera_position=(8.0, 8.0, 3.0), # Initial camera position (x, y, z) - camera_target=(0.0, 0.0, 0.0), # Camera look-at target + camera_position=(8.0, 8.0, 3.0), # Initial camera position (x, y, z) + camera_target=(0.0, 0.0, 0.0), # Camera look-at target # Feature toggles enable_markers=True, # Enable visualization markers @@ -195,8 +195,8 @@ Newton Visualizer window_height=1080, # Window height in pixels # Camera settings - camera_position=(8.0, 8.0, 3.0), # Initial camera position (x, y, z) - camera_target=(0.0, 0.0, 0.0), # Camera look-at target + camera_position=(8.0, 8.0, 3.0), # Initial camera position (x, y, z) + camera_target=(0.0, 0.0, 0.0), # Camera look-at target # Performance tuning update_frequency=1, # Update every N frames (1=every frame) @@ -213,9 +213,9 @@ Newton Visualizer enable_wireframe=False, # Enable wireframe mode # Color customization - background_color=(0.53, 0.81, 0.92), # Sky/background color (RGB [0,1]) - ground_color=(0.18, 0.20, 0.25), # Ground plane color (RGB [0,1]) - light_color=(1.0, 1.0, 1.0), # Directional light color (RGB [0,1]) + sky_upper_color=(0.53, 0.81, 0.92), # Sky upper color (RGB [0,1]) + sky_lower_color=(0.18, 0.20, 0.25), # Sky lower color (RGB [0,1]) + light_color=(1.0, 1.0, 1.0), # Directional light color (RGB [0,1]) ) @@ -241,8 +241,8 @@ Rerun Visualizer web_port=9090, # Port for local web viewer (launched in browser) # Camera settings - camera_position=(8.0, 8.0, 3.0), # Initial camera position (x, y, z) - camera_target=(0.0, 0.0, 0.0), # Camera look-at target + camera_position=(8.0, 8.0, 3.0), # Initial camera position (x, y, z) + camera_target=(0.0, 0.0, 0.0), # Camera look-at target # History settings keep_historical_data=False, # Keep transforms for time scrubbing @@ -260,7 +260,7 @@ To reduce overhead when visualizing large-scale environments, consider: - Using Newton instead of Omniverse or Rerun - Reducing window sizes -- Higher update frequencies +- Lower update frequencies - Pausing visualizers while they are not being used diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index e986d4b664a..e1cd3e352c6 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -115,6 +115,7 @@ def __init__(self, launcher_args: argparse.Namespace | dict | None = None, **kwa self._livestream: Literal[0, 1, 2] # 0: Disabled, 1: WebRTC public, 2: WebRTC private self._offscreen_render: bool # 0: Disabled, 1: Enabled self._sim_experience_file: str # Experience file to load + self._visualizer: list[str] | None # Visualizer backends to use # Exposed to train scripts self.device_id: int # device ID for GPU simulation (defaults to 0) @@ -304,6 +305,16 @@ def add_app_launcher_args(parser: argparse.ArgumentParser) -> None: default=AppLauncher._APPLAUNCHER_CFG_INFO["device"][1], help='The device to run the simulation on. Can be "cpu", "cuda", "cuda:N", where N is the device ID', ) + arg_group.add_argument( + "--visualizer", + type=str, + nargs="+", + default=None, + help=( + "Visualizer backend(s) to use. Valid values: newton, rerun, omniverse." + " Multiple visualizers can be specified: --visualizer rerun newton" + ), + ) # Add the deprecated cpu flag to raise an error if it is used arg_group.add_argument("--cpu", action="store_true", help=argparse.SUPPRESS) arg_group.add_argument( @@ -389,6 +400,7 @@ def add_app_launcher_args(parser: argparse.ArgumentParser) -> None: "device": ([str], "cuda:0"), "experience": ([str], ""), "rendering_mode": ([str], "balanced"), + "visualizer": ([list, type(None)], None), } """A dictionary of arguments added manually by the :meth:`AppLauncher.add_app_launcher_args` method. @@ -488,6 +500,7 @@ def _config_resolution(self, launcher_args: dict): self._resolve_headless_settings(launcher_args, livestream_arg, livestream_env) self._resolve_camera_settings(launcher_args) self._resolve_xr_settings(launcher_args) + self._resolve_visualizer_settings(launcher_args) self._resolve_viewport_settings(launcher_args) # Handle device and distributed settings @@ -777,6 +790,43 @@ def _resolve_anim_recording_settings(self, launcher_args: dict): ) sys.argv += ["--enable", "omni.physx.pvd"] + def _resolve_visualizer_settings(self, launcher_args: dict) -> None: + """Resolve visualizer related settings.""" + visualizers = launcher_args.pop("visualizer", AppLauncher._APPLAUNCHER_CFG_INFO["visualizer"][1]) + valid_visualizers = {"newton", "rerun", "omniverse"} + if visualizers is not None and len(visualizers) > 0: + invalid = [v for v in visualizers if v not in valid_visualizers] + if invalid: + raise ValueError( + f"Invalid visualizer(s) specified: {invalid}. Valid options are: {sorted(valid_visualizers)}" + ) + self._visualizer = visualizers if visualizers and len(visualizers) > 0 else None + + # Auto-adjust headless based on requested visualizers (parity with feature/newton behavior). + if self._visualizer is None: + if not self._headless and self._livestream not in {1, 2}: + self._headless = True + launcher_args["headless"] = True + print( + "[INFO][AppLauncher]: No visualizers specified. " + "Automatically enabling headless mode. Use --visualizer to enable GUI." + ) + return + + if "omniverse" in self._visualizer: + if self._headless: + self._headless = False + launcher_args["headless"] = False + print("[INFO][AppLauncher]: Omniverse visualizer requested. Forcing headless=False for GUI.") + else: + if not self._headless and self._livestream not in {1, 2}: + self._headless = True + launcher_args["headless"] = True + print( + f"[INFO][AppLauncher]: Visualizer(s) {self._visualizer} requested. " + "Enabling headless mode for SimulationApp (visualizers run independently)." + ) + def _resolve_kit_args(self, launcher_args: dict): """Resolve additional arguments passed to Kit.""" # Resolve additional arguments passed to Kit @@ -867,6 +917,12 @@ def _load_extensions(self): # for example: the `Camera` sensor class carb_settings_iface.set_bool("/isaaclab/render/rtx_sensors", False) + # store visualizer selection for SimulationContext + if self._visualizer is not None: + carb_settings_iface.set_string("/isaaclab/visualizer", ",".join(self._visualizer)) + else: + carb_settings_iface.set_string("/isaaclab/visualizer", "") + # set fabric update flag to disable updating transforms when rendering is disabled carb_settings_iface.set_bool("/physics/fabricUpdateTransformations", self._rendering_enabled()) diff --git a/source/isaaclab/isaaclab/scene/interactive_scene.py b/source/isaaclab/isaaclab/scene/interactive_scene.py index 46e5895687a..228b5704272 100644 --- a/source/isaaclab/isaaclab/scene/interactive_scene.py +++ b/source/isaaclab/isaaclab/scene/interactive_scene.py @@ -124,6 +124,7 @@ def __init__(self, cfg: InteractiveSceneCfg): cfg.validate() # store inputs self.cfg = cfg + # initialize scene elements self._terrain = None self._articulations = dict() @@ -174,6 +175,7 @@ def __init__(self, cfg: InteractiveSceneCfg): ), # this won't do anything because we are not replicating physics clone_in_fabric=self.cfg.clone_in_fabric, ) + self._ensure_usd_env_clones(copy_from_source=True) self._default_env_origins = torch.tensor(env_origins, device=self.device, dtype=torch.float32) else: # otherwise, environment origins will be initialized during cloning at the end of environment creation @@ -257,6 +259,7 @@ def clone_environments(self, copy_from_source: bool = False): ), # this automatically filters collisions between environments clone_in_fabric=self.cfg.clone_in_fabric, ) + self._ensure_usd_env_clones(copy_from_source=copy_from_source) # since env_ids is only applicable when replicating physics, we have to fallback to the previous method # to filter collisions if replicate_physics is not enabled @@ -274,6 +277,36 @@ def clone_environments(self, copy_from_source: bool = False): if self._default_env_origins is None: self._default_env_origins = torch.tensor(env_origins, device=self.device, dtype=torch.float32) + def _ensure_usd_env_clones(self, copy_from_source: bool) -> None: + """Ensure USD env prims exist when cloning in fabric.""" + if not self.cfg.clone_in_fabric: + return + if get_isaac_sim_version().major < 5: + return + if not self._should_ensure_usd_env_clones(): + return + + self.cloner.clone( + source_prim_path=self.env_prim_paths[0], + prim_paths=self.env_prim_paths, + replicate_physics=False, + copy_from_source=copy_from_source, + enable_env_ids=False, + clone_in_fabric=False, + ) + + def _should_ensure_usd_env_clones(self) -> bool: + """Check if USD clones are required for current backend/visualizers.""" + sim_cfg = getattr(self.sim, "cfg", None) + if sim_cfg is None: + return True + if sim_cfg.physics_backend != "omni": + return True + + visualizer_types = self.sim.resolve_visualizer_types() + + return bool(visualizer_types) and any(viz_type != "omniverse" for viz_type in visualizer_types) + def filter_collisions(self, global_prim_paths: list[str] | None = None): """Filter environments collisions. diff --git a/source/isaaclab/isaaclab/sim/scene_data_providers/__init__.py b/source/isaaclab/isaaclab/sim/scene_data_providers/__init__.py new file mode 100644 index 00000000000..243ca11b055 --- /dev/null +++ b/source/isaaclab/isaaclab/sim/scene_data_providers/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Scene data providers for visualizers and renderers.""" + +from .newton_scene_data_provider import NewtonSceneDataProvider +from .ov_scene_data_provider import OVSceneDataProvider +from .scene_data_provider import SceneDataProvider + +__all__ = [ + "SceneDataProvider", + "NewtonSceneDataProvider", + "OVSceneDataProvider", +] diff --git a/source/isaaclab/isaaclab/sim/scene_data_providers/newton_scene_data_provider.py b/source/isaaclab/isaaclab/sim/scene_data_providers/newton_scene_data_provider.py new file mode 100644 index 00000000000..43814c48551 --- /dev/null +++ b/source/isaaclab/isaaclab/sim/scene_data_providers/newton_scene_data_provider.py @@ -0,0 +1,99 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Newton-backed scene data provider.""" + +from __future__ import annotations + +import logging +from typing import Any + +logger = logging.getLogger(__name__) + + +class NewtonSceneDataProvider: + """Scene data provider for Newton Warp physics backend. + + Native (cheap): Newton Model/State from NewtonManager + Adapted (future): USD stage (would need Newton→USD sync for OV visualizer) + """ + + def __init__(self, visualizer_cfgs: list[Any] | None) -> None: + self._has_ov_visualizer = False + self._metadata: dict[str, Any] = {} + + if visualizer_cfgs: + for cfg in visualizer_cfgs: + if getattr(cfg, "visualizer_type", None) == "omniverse": + self._has_ov_visualizer = True + + try: + from isaaclab.sim._impl.newton_manager import NewtonManager + + self._metadata = { + "physics_backend": "newton", + "num_envs": NewtonManager._num_envs if NewtonManager._num_envs is not None else 0, + "gravity_vector": NewtonManager._gravity_vector, + "clone_physics_only": NewtonManager._clone_physics_only, + } + except Exception: + self._metadata = {"physics_backend": "newton"} + + def update(self) -> None: + """No-op for Newton backend (state updated by Newton solver).""" + pass + + def get_newton_model(self) -> Any | None: + """NATIVE: Newton Model from NewtonManager.""" + try: + from isaaclab.sim._impl.newton_manager import NewtonManager + + return NewtonManager._model + except Exception: + return None + + def get_newton_state(self) -> Any | None: + """NATIVE: Newton State from NewtonManager.""" + try: + from isaaclab.sim._impl.newton_manager import NewtonManager + + return NewtonManager._state_0 + except Exception: + return None + + def get_usd_stage(self) -> None: + """UNAVAILABLE: Newton backend doesn't provide USD (future: Newton→USD sync).""" + return + + def get_metadata(self) -> dict[str, Any]: + return dict(self._metadata) + + def get_transforms(self) -> dict[str, Any] | None: + """Extract transforms from Newton state (future work).""" + return None + + def get_velocities(self) -> dict[str, Any] | None: + try: + from isaaclab.sim._impl.newton_manager import NewtonManager + + if NewtonManager._state_0 is None: + return None + return {"body_qd": NewtonManager._state_0.body_qd} + except Exception: + return None + + def get_contacts(self) -> dict[str, Any] | None: + try: + from isaaclab.sim._impl.newton_manager import NewtonManager + + if NewtonManager._contacts is None: + return None + return {"contacts": NewtonManager._contacts} + except Exception: + return None + + def get_mesh_data(self) -> dict[str, Any] | None: + """ADAPTED: Extract mesh data from Newton shapes (future work).""" + return None diff --git a/source/isaaclab/isaaclab/sim/scene_data_providers/ov_scene_data_provider.py b/source/isaaclab/isaaclab/sim/scene_data_providers/ov_scene_data_provider.py new file mode 100644 index 00000000000..100ff7f0b10 --- /dev/null +++ b/source/isaaclab/isaaclab/sim/scene_data_providers/ov_scene_data_provider.py @@ -0,0 +1,463 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""OV (Omniverse) scene data provider for Omni PhysX backend.""" + +from __future__ import annotations + +import logging +import re +from typing import Any + +logger = logging.getLogger(__name__) + + +class OVSceneDataProvider: + """Scene data provider for Omni PhysX backend.""" + + def __init__(self, visualizer_cfgs: list[Any] | None, stage, simulation_context) -> None: + from isaacsim.core.simulation_manager import SimulationManager + from pxr import UsdGeom + + self._stage = stage + self._simulation_context = simulation_context + self._physics_sim_view = SimulationManager.get_physics_sim_view() + self._rigid_body_view = None + self._articulation_view = None + self._xform_views: dict[str, Any] = {} + self._body_key_index_map: dict[str, int] = {} + self._view_body_index_map: dict[str, list[int]] = {} + + # Determine which visualizers need Newton state sync + self._has_newton_visualizer = False + self._has_rerun_visualizer = False + self._has_ov_visualizer = False + if visualizer_cfgs: + for cfg in visualizer_cfgs: + viz_type = getattr(cfg, "visualizer_type", None) + if viz_type == "newton": + self._has_newton_visualizer = True + elif viz_type == "rerun": + self._has_rerun_visualizer = True + elif viz_type == "omniverse": + self._has_ov_visualizer = True + + # Explicit mode flag for Newton synchronization + self._needs_newton_sync = self._has_newton_visualizer or self._has_rerun_visualizer + + self._metadata = { + "physics_backend": "omni", + "num_envs": self._get_num_envs(), + "gravity_vector": tuple(self._simulation_context.cfg.gravity), + "clone_physics_only": False, + } + + self._device = getattr(self._simulation_context, "device", "cuda:0") + self._newton_model = None + self._newton_state = None + self._rigid_body_paths: list[str] = [] + self._articulation_paths: list[str] = [] + self._set_body_q_kernel = None + self._up_axis = UsdGeom.GetStageUpAxis(self._stage) + + # Initialize Newton pipeline only if needed for visualization + if self._needs_newton_sync: + self._build_newton_model_from_usd() + self._setup_rigid_body_view() + self._setup_articulation_view() + else: + logger.info("[OVSceneDataProvider] OV visualizer only - skipping Newton model build") + + def _get_num_envs(self) -> int: + try: + import carb + + carb_settings_iface = carb.settings.get_settings() + num_envs = carb_settings_iface.get("/isaaclab/scene/num_envs") + if num_envs: + return int(num_envs) + except Exception: + return 0 + return 0 + + @staticmethod + def _wildcard_env_paths(paths: list[str]) -> list[str]: + wildcard_paths = [] + for path in paths: + if "/World/envs/env_0" in path: + wildcard_paths.append(path.replace("/World/envs/env_0", "/World/envs/env_*")) + return list(dict.fromkeys(wildcard_paths)) if wildcard_paths else paths + + def _refresh_newton_model_if_needed(self) -> None: + num_envs = self._get_num_envs() + if num_envs <= 0: + return + + if self._newton_model is None or self._newton_state is None: + self._build_newton_model_from_usd() + self._setup_rigid_body_view() + self._setup_articulation_view() + return + + if self._metadata.get("num_envs", 0) != num_envs: + self._build_newton_model_from_usd() + self._setup_rigid_body_view() + self._setup_articulation_view() + return + + def _build_newton_model_from_usd(self) -> None: + """Build Newton model from USD and extract scene structure.""" + try: + from newton import ModelBuilder + + builder = ModelBuilder(up_axis=self._up_axis) + builder.add_usd(self._stage) + self._newton_model = builder.finalize(device=self._device) + self._newton_state = self._newton_model.state() + + # Extract scene structure from Newton model (single source of truth) + self._rigid_body_paths = list(self._newton_model.body_key) + self._articulation_paths = list(self._newton_model.articulation_key) + + self._xform_views.clear() + self._body_key_index_map = {path: i for i, path in enumerate(self._rigid_body_paths)} + self._view_body_index_map = {} + except ModuleNotFoundError as exc: + logger.error( + "[SceneDataProvider] Newton module not available. " + "Install the Newton backend to use newton/rerun visualizers." + ) + logger.debug(f"[SceneDataProvider] Newton import error: {exc}") + except Exception as exc: + logger.error(f"[SceneDataProvider] Failed to build Newton model from USD: {exc}") + self._newton_model = None + self._newton_state = None + self._rigid_body_paths = [] + self._articulation_paths = [] + + def _setup_rigid_body_view(self) -> None: + """Create PhysX RigidBodyView from Newton's body paths. + + Uses body paths extracted from Newton model to create PhysX tensor API view + for reading rigid body transforms. + """ + if not self._rigid_body_paths: + return + try: + paths_to_use = self._wildcard_env_paths(self._rigid_body_paths) + self._rigid_body_view = self._physics_sim_view.create_rigid_body_view(paths_to_use) + self._cache_view_index_map(self._rigid_body_view, "rigid_body_view") + except Exception as exc: + logger.warning(f"[SceneDataProvider] Failed to create RigidBodyView: {exc}") + self._rigid_body_view = None + + def _setup_articulation_view(self) -> None: + """Create PhysX ArticulationView from Newton's articulation paths.""" + if not self._articulation_paths: + return + try: + paths_to_use = self._wildcard_env_paths(self._articulation_paths) + exprs = [path.replace(".*", "*") for path in paths_to_use] + self._articulation_view = self._physics_sim_view.create_articulation_view( + exprs if len(exprs) > 1 else exprs[0] + ) + self._cache_view_index_map(self._articulation_view, "articulation_view") + except Exception as exc: + logger.warning(f"[SceneDataProvider] Failed to create ArticulationView: {exc}") + self._articulation_view = None + + def _get_view_world_poses(self, view): + """Read world poses from PhysX tensor API view (ArticulationView or RigidBodyView). + + Tries multiple method names for compatibility across PhysX API versions. + Returns (positions, orientations) tuple or (None, None) if unavailable. + """ + if view is None: + return None, None + + method_names = ("get_world_poses", "get_world_transforms", "get_transforms", "get_poses") + + for name in method_names: + method = getattr(view, name, None) + if method is None: + continue + try: + result = method() + except Exception: + continue + + # Handle tuple return: (positions, orientations) + if isinstance(result, tuple) and len(result) == 2: + return result + + # Handle packed array: [..., 7] -> split into pos and quat + try: + if hasattr(result, "shape") and result.shape[-1] == 7: + positions = result[..., :3] + orientations = result[..., 3:7] + return positions, orientations + except Exception: + continue + + return None, None + + def _cache_view_index_map(self, view, key: str) -> None: + """Map PhysX view indices to Newton body_key ordering.""" + prim_paths = getattr(view, "prim_paths", None) + if not prim_paths or not self._rigid_body_paths: + return + + def split_env(path: str) -> tuple[int | None, str]: + """Extract environment ID and relative path from prim path.""" + match = re.search(r"/World/envs/env_(\d+)(/.*)", path) + return (int(match.group(1)), match.group(2)) if match else (None, path) + + # Build map: (env_id, relative_path) -> view_index + view_map: dict[tuple[int | None, str], int] = {} + for view_idx, path in enumerate(prim_paths): + env_id, rel = split_env(path) + view_map[(env_id, rel)] = view_idx + + # Build reordering: newton_body_index -> view_index + order: list[int | None] = [None] * len(self._rigid_body_paths) + for body_idx, path in enumerate(self._rigid_body_paths): + env_id, rel = split_env(path) + view_idx = view_map.get((env_id, rel)) + if view_idx is None: + view_idx = view_map.get((None, rel)) # Try without env_id + order[body_idx] = view_idx + + if all(idx is not None for idx in order): + self._view_body_index_map[key] = order # type: ignore[arg-type] + + def _get_view_velocities(self, view): + if view is None: + return None, None + + method = getattr(view, "get_velocities", None) + if method is not None: + try: + result = method() + if isinstance(result, tuple) and len(result) == 2: + return result + if hasattr(result, "shape") and result.shape[-1] == 6: + return result[..., :3], result[..., 3:6] + except Exception: + pass + + get_linear = getattr(view, "get_linear_velocities", None) + get_angular = getattr(view, "get_angular_velocities", None) + if get_linear is not None and get_angular is not None: + try: + return get_linear(), get_angular() + except Exception: + return None, None + + return None, None + + def _apply_view_poses(self, view: Any, view_key: str, positions: Any, orientations: Any, covered: Any) -> int: + """Read poses from a PhysX view and write uncovered bodies to output tensors.""" + import torch + + if view is None: + return 0 + + pos, quat = self._get_view_world_poses(view) + if pos is None or quat is None: + return 0 + + order = self._view_body_index_map.get(view_key) + if not order: + return 0 + + pos = pos.to(device=self._device, dtype=torch.float32) + quat = quat.to(device=self._device, dtype=torch.float32) + + count = 0 + for newton_idx, view_idx in enumerate(order): + if view_idx is not None and not covered[newton_idx]: + positions[newton_idx] = pos[view_idx] + orientations[newton_idx] = quat[view_idx] + covered[newton_idx] = True + count += 1 + + return count + + def _apply_xform_poses(self, positions: Any, orientations: Any, covered: Any, xform_mask: Any) -> int: + """Use XformPrimView fallback for remaining uncovered bodies.""" + import torch + + from isaaclab.sim.views import XformPrimView + + uncovered = torch.where(~covered)[0].cpu().tolist() + if not uncovered: + return 0 + + count = 0 + for idx in uncovered: + path = self._rigid_body_paths[idx] + try: + if path not in self._xform_views: + self._xform_views[path] = XformPrimView( + path, device=self._device, stage=self._stage, validate_xform_ops=False + ) + + pos, quat = self._xform_views[path].get_world_poses() + if pos is not None and quat is not None: + positions[idx] = pos.to(device=self._device, dtype=torch.float32).squeeze() + orientations[idx] = quat.to(device=self._device, dtype=torch.float32).squeeze() + covered[idx] = True + xform_mask[idx] = True + count += 1 + except Exception: + continue + + return count + + def _convert_xform_quats(self, orientations: Any, xform_mask: Any) -> Any: + """Convert XformPrimView quaternions from wxyz to xyzw where needed.""" + if not xform_mask.any(): + return orientations + + import torch + + from isaaclab.utils.math import convert_quat + + orientations_xyzw = orientations.clone() + xform_indices = torch.where(xform_mask)[0] + if len(xform_indices) > 0: + orientations_xyzw[xform_indices] = convert_quat(orientations[xform_indices], to="xyzw") + return orientations_xyzw + + def _read_poses_from_best_source(self) -> tuple[Any, Any, str, Any] | None: + """Merge pose data from ArticulationView, RigidBodyView, and XformPrimView.""" + if self._newton_state is None or not self._rigid_body_paths: + return None + + import torch + + num_bodies = len(self._rigid_body_paths) + if num_bodies != self._newton_state.body_q.shape[0]: + logger.warning(f"Body count mismatch: body_key={num_bodies}, state={self._newton_state.body_q.shape[0]}") + return None + + positions = torch.zeros((num_bodies, 3), dtype=torch.float32, device=self._device) + orientations = torch.zeros((num_bodies, 4), dtype=torch.float32, device=self._device) + covered = torch.zeros(num_bodies, dtype=torch.bool, device=self._device) + xform_mask = torch.zeros(num_bodies, dtype=torch.bool, device=self._device) + + artic = self._apply_view_poses(self._articulation_view, "articulation_view", positions, orientations, covered) + rigid = self._apply_view_poses(self._rigid_body_view, "rigid_body_view", positions, orientations, covered) + xform = self._apply_xform_poses(positions, orientations, covered, xform_mask) + + if not covered.all(): + logger.warning(f"Failed to read {(~covered).sum().item()}/{num_bodies} body poses") + return None + + active = sum([artic > 0, rigid > 0, xform > 0]) + source = ( + "merged" + if active > 1 + else ("articulation_view" if artic else "rigid_body_view" if rigid else "xform_view" if xform else "none") + ) + + return positions, orientations, source, xform_mask + + def _get_set_body_q_kernel(self): + """Get or create the Warp kernel for writing transforms to Newton state.""" + if self._set_body_q_kernel is not None: + return self._set_body_q_kernel + try: + import warp as wp + + @wp.kernel(enable_backward=False) + def _set_body_q( + positions: wp.array(dtype=wp.vec3), + orientations: wp.array(dtype=wp.quatf), + body_q: wp.array(dtype=wp.transformf), + ): + i = wp.tid() + body_q[i] = wp.transformf(positions[i], orientations[i]) + + self._set_body_q_kernel = _set_body_q + return self._set_body_q_kernel + except Exception as exc: + logger.warning(f"[SceneDataProvider] Warp unavailable for Newton state sync: {exc}") + return None + + def update(self) -> None: + """Sync PhysX transforms to Newton state for visualization.""" + if not self._needs_newton_sync or self._newton_state is None: + return + + self._refresh_newton_model_if_needed() + + try: + import warp as wp + + result = self._read_poses_from_best_source() + if result is None: + return + + positions, orientations, _, xform_mask = result + orientations_xyzw = self._convert_xform_quats(orientations.reshape(-1, 4), xform_mask) + + positions_wp = wp.from_torch(positions.reshape(-1, 3), dtype=wp.vec3) + orientations_wp = wp.from_torch(orientations_xyzw, dtype=wp.quatf) + + set_body_q = self._get_set_body_q_kernel() + if set_body_q is None or positions_wp.shape[0] != self._newton_state.body_q.shape[0]: + return + + wp.launch( + set_body_q, + dim=positions_wp.shape[0], + inputs=[positions_wp, orientations_wp, self._newton_state.body_q], + device=self._device, + ) + + except Exception as exc: + logger.debug(f"Failed to sync transforms to Newton: {exc}") + + def get_newton_model(self) -> Any | None: + return self._newton_model if self._needs_newton_sync else None + + def get_newton_state(self) -> Any | None: + return self._newton_state if self._needs_newton_sync else None + + def get_usd_stage(self) -> Any: + return self._stage + + def get_mesh_data(self) -> dict[str, Any] | None: + return None + + def get_metadata(self) -> dict[str, Any]: + return dict(self._metadata) + + def get_transforms(self) -> dict[str, Any] | None: + try: + result = self._read_poses_from_best_source() + if result is None: + return None + + positions, orientations, _, xform_mask = result + orientations_xyzw = self._convert_xform_quats(orientations, xform_mask) + return {"positions": positions, "orientations": orientations_xyzw} + except Exception: + return None + + def get_velocities(self) -> dict[str, Any] | None: + for source, view in ( + ("articulation_view", self._articulation_view), + ("rigid_body_view", self._rigid_body_view), + ): + linear, angular = self._get_view_velocities(view) + if linear is not None and angular is not None: + return {"linear": linear, "angular": angular, "source": source} + return None + + def get_contacts(self) -> dict[str, Any] | None: + """Contacts not yet supported for OV backend.""" + return None diff --git a/source/isaaclab/isaaclab/sim/scene_data_providers/scene_data_provider.py b/source/isaaclab/isaaclab/sim/scene_data_providers/scene_data_provider.py new file mode 100644 index 00000000000..14708219c2d --- /dev/null +++ b/source/isaaclab/isaaclab/sim/scene_data_providers/scene_data_provider.py @@ -0,0 +1,86 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Scene data provider for visualizers and renderers.""" + +from __future__ import annotations + +import logging +from typing import Any + +logger = logging.getLogger(__name__) + + +class SceneDataProvider: + """Creates appropriate data provider based on physics backend.""" + + def __init__( + self, + backend: str, + visualizer_cfgs: list[Any] | None, + stage=None, + simulation_context=None, + ) -> None: + self._backend = backend + self._provider = None + + if backend == "newton": + from .newton_scene_data_provider import NewtonSceneDataProvider + + self._provider = NewtonSceneDataProvider(visualizer_cfgs) + elif backend == "omni": + if stage is None or simulation_context is None: + logger.warning("OV scene data provider requires stage and simulation context.") + self._provider = None + else: + from .ov_scene_data_provider import OVSceneDataProvider + + self._provider = OVSceneDataProvider(visualizer_cfgs, stage, simulation_context) + else: + logger.warning(f"Unknown physics backend '{backend}'.") + + def update(self) -> None: + if self._provider is not None: + self._provider.update() + + def get_newton_model(self) -> Any | None: + if self._provider is None: + return None + return self._provider.get_newton_model() + + def get_newton_state(self) -> Any | None: + if self._provider is None: + return None + return self._provider.get_newton_state() + + def get_usd_stage(self) -> Any | None: + if self._provider is None: + return None + return self._provider.get_usd_stage() + + def get_metadata(self) -> dict[str, Any]: + if self._provider is None: + return {} + return self._provider.get_metadata() + + def get_transforms(self) -> dict[str, Any] | None: + if self._provider is None: + return None + return self._provider.get_transforms() + + def get_velocities(self) -> dict[str, Any] | None: + if self._provider is None: + return None + return self._provider.get_velocities() + + def get_contacts(self) -> dict[str, Any] | None: + if self._provider is None: + return None + return self._provider.get_contacts() + + def get_mesh_data(self) -> dict[str, Any] | None: + if self._provider is None: + return None + return self._provider.get_mesh_data() diff --git a/source/isaaclab/isaaclab/sim/simulation_cfg.py b/source/isaaclab/isaaclab/sim/simulation_cfg.py index 06ed4826af6..2ccb0176bdf 100644 --- a/source/isaaclab/isaaclab/sim/simulation_cfg.py +++ b/source/isaaclab/isaaclab/sim/simulation_cfg.py @@ -12,6 +12,7 @@ from typing import Any, Literal from isaaclab.utils import configclass +from isaaclab.visualizers import VisualizerCfg from .spawners.materials import RigidBodyMaterialCfg @@ -412,6 +413,9 @@ class SimulationCfg: physx: PhysxCfg = PhysxCfg() """PhysX solver settings. Default is PhysxCfg().""" + physics_backend: Literal["omni", "newton"] = "omni" + """Physics backend to use for scene data providers and visualizers.""" + physics_material: RigidBodyMaterialCfg = RigidBodyMaterialCfg() """Default physics material settings for rigid bodies. Default is RigidBodyMaterialCfg(). @@ -424,6 +428,18 @@ class SimulationCfg: render: RenderCfg = RenderCfg() """Render settings. Default is RenderCfg().""" + visualizer_cfgs: list[VisualizerCfg] | VisualizerCfg | None = None + """Visualizer settings. Default is no visualizer. + + Visualizers are separate from Renderers and intended for light-weight monitoring and debugging. + + This field can support multiple visualizer backends. It accepts: + + * A single VisualizerCfg: One visualizer will be created + * A list of VisualizerCfg: Multiple visualizers will be created + * None or empty list: No visualizers will be created + """ + create_stage_in_memory: bool = False """If stage is first created in memory. Default is False. diff --git a/source/isaaclab/isaaclab/sim/simulation_context.py b/source/isaaclab/isaaclab/sim/simulation_context.py index d022c2d32a7..f5af18b624c 100644 --- a/source/isaaclab/isaaclab/sim/simulation_context.py +++ b/source/isaaclab/isaaclab/sim/simulation_context.py @@ -33,7 +33,9 @@ import isaaclab.sim as sim_utils from isaaclab.utils.logger import configure_logging from isaaclab.utils.version import get_isaac_sim_version +from isaaclab.visualizers import NewtonVisualizerCfg, OVVisualizerCfg, RerunVisualizerCfg, Visualizer +from .scene_data_providers import SceneDataProvider from .simulation_cfg import SimulationCfg from .spawners import DomeLightCfg, GroundPlaneCfg from .utils import bind_physics_material @@ -256,6 +258,11 @@ def __init__(self, cfg: SimulationCfg | None = None): self._app_control_on_stop_handle = None self._disable_app_control_on_stop_handle = False + # initialize visualizers and scene data provider + self._visualizers: list[Visualizer] = [] + self._visualizer_step_counter = 0 + self._scene_data_provider: SceneDataProvider | None = None + # flatten out the simulation dictionary sim_params = self.cfg.to_dict() if sim_params is not None: @@ -505,6 +512,122 @@ def get_initial_stage(self) -> Usd.Stage: """ return self._initial_stage + """ + Visualizers. + """ + + def _create_default_visualizer_configs(self, requested_visualizers: list[str]) -> list: + """Create default visualizer configs for requested types.""" + default_configs = [] + for viz_type in requested_visualizers: + try: + if viz_type == "newton": + default_configs.append(NewtonVisualizerCfg()) + elif viz_type == "rerun": + default_configs.append(RerunVisualizerCfg()) + elif viz_type == "omniverse": + default_configs.append(OVVisualizerCfg()) + else: + logger.warning( + f"[SimulationContext] Unknown visualizer type '{viz_type}' requested. " + "Valid types: 'newton', 'rerun', 'omniverse'. Skipping." + ) + except Exception as exc: + logger.error(f"[SimulationContext] Failed to create default config for visualizer '{viz_type}': {exc}") + return default_configs + + def resolve_visualizer_types(self) -> list[str]: + """Resolve visualizer types from config or CLI settings.""" + visualizer_cfgs = self.cfg.visualizer_cfgs + if visualizer_cfgs is None: + requested = self.get_setting("/isaaclab/visualizer") + return [v.strip() for v in requested.split(",") if v.strip()] if requested else [] + + if not isinstance(visualizer_cfgs, list): + visualizer_cfgs = [visualizer_cfgs] + return [cfg.visualizer_type for cfg in visualizer_cfgs if getattr(cfg, "visualizer_type", None)] + + def initialize_visualizers(self) -> None: + """Initialize visualizers from SimulationCfg.visualizer_cfgs.""" + visualizer_cfgs = self.cfg.visualizer_cfgs + if visualizer_cfgs is None: + requested_visualizers = self.resolve_visualizer_types() + if not requested_visualizers: + return + visualizer_cfgs = self._create_default_visualizer_configs(requested_visualizers) + elif not isinstance(visualizer_cfgs, list): + visualizer_cfgs = [visualizer_cfgs] + + self._scene_data_provider = SceneDataProvider( + backend=self.cfg.physics_backend, + visualizer_cfgs=visualizer_cfgs, + stage=self.stage, + simulation_context=self, + ) + + for viz_cfg in visualizer_cfgs: + try: + visualizer = viz_cfg.create_visualizer() + scene_data: dict[str, Any] = {"scene_data_provider": self._scene_data_provider} + + # OV visualizer gets USD stage + if viz_cfg.visualizer_type == "omniverse": + if self._scene_data_provider: + scene_data["usd_stage"] = self._scene_data_provider.get_usd_stage() + else: + scene_data["usd_stage"] = self.stage + + visualizer.initialize(scene_data) + self._visualizers.append(visualizer) + logger.info(f"Initialized visualizer: {type(visualizer).__name__} (type: {viz_cfg.visualizer_type})") + except Exception as exc: + logger.error( + f"Failed to initialize visualizer '{viz_cfg.visualizer_type}' ({type(viz_cfg).__name__}): {exc}" + ) + + def step_visualizers(self, dt: float) -> None: + """Update all active visualizers.""" + if not self._visualizers: + return + + self._visualizer_step_counter += 1 + if self._scene_data_provider: + self._scene_data_provider.update() + + visualizers_to_remove = [] + for visualizer in self._visualizers: + try: + if not visualizer.is_running(): + visualizers_to_remove.append(visualizer) + continue + + while visualizer.is_training_paused() and visualizer.is_running(): + visualizer.step(0.0, state=None) + + visualizer.step(dt, state=None) + except Exception as exc: + logger.error(f"Error stepping visualizer '{type(visualizer).__name__}': {exc}") + visualizers_to_remove.append(visualizer) + + for visualizer in visualizers_to_remove: + try: + visualizer.close() + self._visualizers.remove(visualizer) + logger.info(f"Removed visualizer: {type(visualizer).__name__}") + except Exception as exc: + logger.error(f"Error closing visualizer: {exc}") + + def close_visualizers(self) -> None: + """Close all active visualizers and clean up resources.""" + for visualizer in self._visualizers: + try: + visualizer.close() + except Exception as exc: + logger.error(f"Error closing visualizer '{type(visualizer).__name__}': {exc}") + + self._visualizers.clear() + logger.info("All visualizers closed") + """ Operations - Override (standalone) """ @@ -528,6 +651,8 @@ def reset(self, soft: bool = False): if not soft: for _ in range(2): self.render() + if not soft and not self._visualizers: + self.initialize_visualizers() self._disable_app_control_on_stop_handle = False def forward(self) -> None: @@ -578,6 +703,9 @@ def step(self, render: bool = True): # step the simulation super().step(render=render) + # Update visualizers after stepping + self.step_visualizers(self.cfg.dt) + # app.update() may be changing the cuda device in step, so we force it back to our desired device here if "cuda" in self.device: torch.cuda.set_device(self.device) @@ -676,6 +804,8 @@ def clear_instance(cls): if cls._instance._app_control_on_stop_handle is not None: cls._instance._app_control_on_stop_handle.unsubscribe() cls._instance._app_control_on_stop_handle = None + if hasattr(cls._instance, "_visualizers"): + cls._instance.close_visualizers() # call parent to clear the instance super().clear_instance() diff --git a/source/isaaclab/isaaclab/sim/spawners/materials/visual_materials.py b/source/isaaclab/isaaclab/sim/spawners/materials/visual_materials.py index 074d6ac0e43..27cc41c7725 100644 --- a/source/isaaclab/isaaclab/sim/spawners/materials/visual_materials.py +++ b/source/isaaclab/isaaclab/sim/spawners/materials/visual_materials.py @@ -60,30 +60,32 @@ def spawn_preview_surface(prim_path: str, cfg: visual_materials_cfg.PreviewSurfa # in that case is always the one from USD Context which makes it difficult to # handle scene creation on a custom stage. material_prim = UsdShade.Material.Define(stage, prim_path) - if material_prim: - shader_prim = CreateShaderPrimFromSdrCommand( - parent_path=prim_path, - identifier="UsdPreviewSurface", - stage_or_context=stage, - name="Shader", - ).do() - # bind the shader graph to the material - if shader_prim: - surface_out = shader_prim.GetOutput("surface") - if surface_out: - material_prim.CreateSurfaceOutput().ConnectToSource(surface_out) - - displacement_out = shader_prim.GetOutput("displacement") - if displacement_out: - material_prim.CreateDisplacementOutput().ConnectToSource(displacement_out) - else: + if not material_prim: raise ValueError(f"Failed to create preview surface shader at path: '{prim_path}'.") + + shader_prim = CreateShaderPrimFromSdrCommand( + parent_path=prim_path, + identifier="UsdPreviewSurface", + stage_or_context=stage, + ).do() + + if not shader_prim: + raise ValueError(f"Failed to create shader prim at path: '{prim_path}'.") + + # The command returns a Shader object directly, not a path + if shader_prim: + surface_out = shader_prim.GetOutput("surface") + if surface_out: + material_prim.CreateSurfaceOutput().ConnectToSource(surface_out) + + displacement_out = shader_prim.GetOutput("displacement") + if displacement_out: + material_prim.CreateDisplacementOutput().ConnectToSource(displacement_out) else: raise ValueError(f"A prim already exists at path: '{prim_path}'.") - # obtain prim - prim = stage.GetPrimAtPath(f"{prim_path}/Shader") - # check prim is valid + # Get the underlying prim from the shader + prim = shader_prim.GetPrim() if not prim.IsValid(): raise ValueError(f"Failed to create preview surface material at path: '{prim_path}'.") # apply properties diff --git a/source/isaaclab/isaaclab/visualizers/__init__.py b/source/isaaclab/isaaclab/visualizers/__init__.py new file mode 100644 index 00000000000..df9f3b60ab8 --- /dev/null +++ b/source/isaaclab/isaaclab/visualizers/__init__.py @@ -0,0 +1,63 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Sub-package for visualizer configurations and implementations.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from .newton_visualizer_cfg import NewtonVisualizerCfg +from .ov_visualizer_cfg import OVVisualizerCfg +from .rerun_visualizer_cfg import RerunVisualizerCfg +from .visualizer import Visualizer +from .visualizer_cfg import VisualizerCfg + +if TYPE_CHECKING: + from typing import Type + + from .newton_visualizer import NewtonVisualizer + from .ov_visualizer import OVVisualizer + from .rerun_visualizer import RerunVisualizer + +_VISUALIZER_REGISTRY: dict[str, Any] = {} + +__all__ = [ + "Visualizer", + "VisualizerCfg", + "NewtonVisualizerCfg", + "OVVisualizerCfg", + "RerunVisualizerCfg", + "get_visualizer_class", +] + + +def get_visualizer_class(name: str) -> type[Visualizer] | None: + """Get a visualizer class by name (lazy-loaded).""" + if name in _VISUALIZER_REGISTRY: + return _VISUALIZER_REGISTRY[name] + + try: + if name == "newton": + from .newton_visualizer import NewtonVisualizer + + _VISUALIZER_REGISTRY["newton"] = NewtonVisualizer + return NewtonVisualizer + if name == "omniverse": + from .ov_visualizer import OVVisualizer + + _VISUALIZER_REGISTRY["omniverse"] = OVVisualizer + return OVVisualizer + if name == "rerun": + from .rerun_visualizer import RerunVisualizer + + _VISUALIZER_REGISTRY["rerun"] = RerunVisualizer + return RerunVisualizer + return None + except ImportError as exc: + import warnings + + warnings.warn(f"Failed to load visualizer '{name}': {exc}", ImportWarning) + return None diff --git a/source/isaaclab/isaaclab/visualizers/newton_visualizer.py b/source/isaaclab/isaaclab/visualizers/newton_visualizer.py new file mode 100644 index 00000000000..a2c50b8d640 --- /dev/null +++ b/source/isaaclab/isaaclab/visualizers/newton_visualizer.py @@ -0,0 +1,323 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Newton OpenGL Visualizer implementation.""" + +from __future__ import annotations + +import contextlib +import logging +from typing import Any + +import numpy as np +import warp as wp +from newton.viewer import ViewerGL + +from .newton_visualizer_cfg import NewtonVisualizerCfg +from .visualizer import Visualizer + +logger = logging.getLogger(__name__) + + +class NewtonViewerGL(ViewerGL): + """Wrapper around Newton's ViewerGL with training/rendering pause controls.""" + + def __init__(self, *args, metadata: dict | None = None, update_frequency: int = 1, **kwargs): + super().__init__(*args, **kwargs) + self._paused_training = False + self._paused_rendering = False + self._metadata = metadata or {} + self._fallback_draw_controls = False + self._update_frequency = update_frequency + + try: + self.register_ui_callback(self._render_training_controls, position="side") + except AttributeError: + self._fallback_draw_controls = True + + def is_training_paused(self) -> bool: + return self._paused_training + + def is_rendering_paused(self) -> bool: + return self._paused_rendering + + def _render_training_controls(self, imgui): + imgui.separator() + imgui.text("IsaacLab Controls") + + pause_label = "Resume Training" if self._paused_training else "Pause Training" + if imgui.button(pause_label): + self._paused_training = not self._paused_training + + rendering_label = "Resume Rendering" if self._paused_rendering else "Pause Rendering" + if imgui.button(rendering_label): + self._paused_rendering = not self._paused_rendering + self._paused = self._paused_rendering + + imgui.text("Visualizer Update Frequency") + current_frequency = self._update_frequency + changed, new_frequency = imgui.slider_int( + "##VisualizerUpdateFreq", current_frequency, 1, 20, f"Every {current_frequency} frames" + ) + if changed: + self._update_frequency = new_frequency + + if imgui.is_item_hovered(): + imgui.set_tooltip( + "Controls visualizer update frequency\nlower values -> more responsive visualizer but slower" + " training\nhigher values -> less responsive visualizer but faster training" + ) + + def on_key_press(self, symbol, modifiers): + if self.ui.is_capturing(): + return + + try: + import pyglet # noqa: PLC0415 + except Exception: + return + + if symbol == pyglet.window.key.SPACE: + self._paused_rendering = not self._paused_rendering + self._paused = self._paused_rendering + return + + super().on_key_press(symbol, modifiers) + + def _render_ui(self): + if not self._fallback_draw_controls: + return super()._render_ui() + + super()._render_ui() + imgui = self.ui.imgui + from contextlib import suppress + + with suppress(Exception): + imgui.set_next_window_pos(imgui.ImVec2(320, 10)) + + flags = 0 + if imgui.begin("Training Controls", flags=flags): + self._render_training_controls(imgui) + imgui.end() + return None + + def _render_left_panel(self): + """Override the left panel to remove the base pause checkbox.""" + import newton as nt + + imgui = self.ui.imgui + nav_highlight_color = self.ui.get_theme_color(imgui.Col_.nav_cursor, (1.0, 1.0, 1.0, 1.0)) + + io = self.ui.io + imgui.set_next_window_pos(imgui.ImVec2(10, 10)) + imgui.set_next_window_size(imgui.ImVec2(300, io.display_size[1] - 20)) + + flags = imgui.WindowFlags_.no_resize.value + + if imgui.begin(f"Newton Viewer v{nt.__version__}", flags=flags): + imgui.separator() + + header_flags = 0 + + imgui.set_next_item_open(True, imgui.Cond_.appearing) + if imgui.collapsing_header("IsaacLab Options"): + for callback in self._ui_callbacks["side"]: + callback(self.ui.imgui) + + if self.model is not None: + imgui.set_next_item_open(True, imgui.Cond_.appearing) + if imgui.collapsing_header("Model Information", flags=header_flags): + imgui.separator() + num_envs = self._metadata.get("num_envs", 0) + imgui.text(f"Environments: {num_envs}") + axis_names = ["X", "Y", "Z"] + imgui.text(f"Up Axis: {axis_names[self.model.up_axis]}") + gravity = wp.to_torch(self.model.gravity)[0] + gravity_text = f"Gravity: ({gravity[0]:.2f}, {gravity[1]:.2f}, {gravity[2]:.2f})" + imgui.text(gravity_text) + + imgui.set_next_item_open(True, imgui.Cond_.appearing) + if imgui.collapsing_header("Visualization", flags=header_flags): + imgui.separator() + + show_joints = self.show_joints + changed, self.show_joints = imgui.checkbox("Show Joints", show_joints) + + show_contacts = self.show_contacts + changed, self.show_contacts = imgui.checkbox("Show Contacts", show_contacts) + + show_springs = self.show_springs + changed, self.show_springs = imgui.checkbox("Show Springs", show_springs) + + show_com = self.show_com + changed, self.show_com = imgui.checkbox("Show Center of Mass", show_com) + + imgui.set_next_item_open(True, imgui.Cond_.appearing) + if imgui.collapsing_header("Rendering Options"): + imgui.separator() + + changed, self.renderer.draw_sky = imgui.checkbox("Sky", self.renderer.draw_sky) + changed, self.renderer.draw_shadows = imgui.checkbox("Shadows", self.renderer.draw_shadows) + changed, self.renderer.draw_wireframe = imgui.checkbox("Wireframe", self.renderer.draw_wireframe) + + changed, self.renderer._light_color = imgui.color_edit3("Light Color", self.renderer._light_color) + changed, self.renderer.sky_upper = imgui.color_edit3("Upper Sky Color", self.renderer.sky_upper) + changed, self.renderer.sky_lower = imgui.color_edit3("Lower Sky Color", self.renderer.sky_lower) + + imgui.set_next_item_open(True, imgui.Cond_.appearing) + if imgui.collapsing_header("Camera"): + imgui.separator() + + pos = self.camera.pos + pos_text = f"Position: ({pos[0]:.2f}, {pos[1]:.2f}, {pos[2]:.2f})" + imgui.text(pos_text) + imgui.text(f"FOV: {self.camera.fov:.1f}°") + imgui.text(f"Yaw: {self.camera.yaw:.1f}°") + imgui.text(f"Pitch: {self.camera.pitch:.1f}°") + + imgui.separator() + imgui.push_style_color(imgui.Col_.text, imgui.ImVec4(*nav_highlight_color)) + imgui.text("Controls:") + imgui.pop_style_color() + imgui.text("WASD - Forward/Left/Back/Right") + imgui.text("QE - Down/Up") + imgui.text("Left Click - Look around") + imgui.text("Scroll - Zoom") + imgui.text("Space - Pause/Resume Rendering") + imgui.text("H - Toggle UI") + imgui.text("ESC - Exit") + + imgui.end() + return + + +class NewtonVisualizer(Visualizer): + """Newton OpenGL visualizer for Isaac Lab.""" + + def __init__(self, cfg: NewtonVisualizerCfg): + super().__init__(cfg) + self.cfg: NewtonVisualizerCfg = cfg + self._viewer: NewtonViewerGL | None = None + self._sim_time = 0.0 + self._step_counter = 0 + self._model = None + self._state = None + self._update_frequency = cfg.update_frequency + self._scene_data_provider = None + + def initialize(self, scene_data: dict[str, Any] | None = None) -> None: + if self._is_initialized: + return + + if not scene_data or "scene_data_provider" not in scene_data: + raise RuntimeError("Newton visualizer requires scene_data_provider.") + + self._scene_data_provider = scene_data["scene_data_provider"] + self._model = self._scene_data_provider.get_newton_model() + self._state = self._scene_data_provider.get_newton_state() + metadata = self._scene_data_provider.get_metadata() + + self._viewer = NewtonViewerGL( + width=self.cfg.window_width, + height=self.cfg.window_height, + metadata=metadata, + update_frequency=self.cfg.update_frequency, + ) + + self._viewer.set_model(self._model) + self._viewer.set_world_offsets((0.0, 0.0, 0.0)) + self._viewer.camera.pos = wp.vec3(*self.cfg.camera_position) + self._viewer.up_axis = 2 # Z-up + + cam_pos = np.array(self.cfg.camera_position, dtype=np.float32) + cam_target = np.array(self.cfg.camera_target, dtype=np.float32) + direction = cam_target - cam_pos + yaw = np.degrees(np.arctan2(direction[1], direction[0])) + horizontal_dist = np.sqrt(direction[0] ** 2 + direction[1] ** 2) + pitch = np.degrees(np.arctan2(direction[2], horizontal_dist)) + + self._viewer.camera.yaw = float(yaw) + self._viewer.camera.pitch = float(pitch) + + self._viewer.scaling = 1.0 + self._viewer._paused = False + + self._viewer.show_joints = self.cfg.show_joints + self._viewer.show_contacts = self.cfg.show_contacts + self._viewer.show_springs = self.cfg.show_springs + self._viewer.show_com = self.cfg.show_com + + self._viewer.renderer.draw_shadows = self.cfg.enable_shadows + self._viewer.renderer.draw_sky = self.cfg.enable_sky + self._viewer.renderer.draw_wireframe = self.cfg.enable_wireframe + + self._viewer.renderer.sky_upper = self.cfg.sky_upper_color + self._viewer.renderer.sky_lower = self.cfg.sky_lower_color + self._viewer.renderer._light_color = self.cfg.light_color + + self._is_initialized = True + + def step(self, dt: float, state: Any | None = None) -> None: + if not self._is_initialized or self._is_closed or self._viewer is None: + return + + self._sim_time += dt + self._step_counter += 1 + + self._state = self._scene_data_provider.get_newton_state() + + contacts = None + if self._viewer.show_contacts: + contacts_data = self._scene_data_provider.get_contacts() + if isinstance(contacts_data, dict): + contacts = contacts_data.get("contacts", contacts_data) + else: + contacts = contacts_data + + update_frequency = self._viewer._update_frequency if self._viewer else self._update_frequency + if self._step_counter % update_frequency != 0: + return + + with contextlib.suppress(Exception): + if not self._viewer.is_paused(): + self._viewer.begin_frame(self._sim_time) + if self._state is not None: + self._viewer.log_state(self._state) + if contacts is not None and hasattr(self._viewer, "log_contacts"): + try: + self._viewer.log_contacts(contacts, self._state) + except Exception as exc: + logger.debug(f"[NewtonVisualizer] Failed to log contacts: {exc}") + self._viewer.end_frame() + else: + self._viewer._update() + + def close(self) -> None: + if self._is_closed: + return + if self._viewer is not None: + self._viewer = None + self._is_closed = True + + def is_running(self) -> bool: + if not self._is_initialized or self._is_closed or self._viewer is None: + return False + return self._viewer.is_running() + + def supports_markers(self) -> bool: + return False + + def supports_live_plots(self) -> bool: + return False + + def is_training_paused(self) -> bool: + if not self._is_initialized or self._viewer is None: + return False + return self._viewer.is_training_paused() + + def is_rendering_paused(self) -> bool: + if not self._is_initialized or self._viewer is None: + return False + return self._viewer.is_rendering_paused() diff --git a/source/isaaclab/isaaclab/visualizers/newton_visualizer_cfg.py b/source/isaaclab/isaaclab/visualizers/newton_visualizer_cfg.py new file mode 100644 index 00000000000..0d0439843b7 --- /dev/null +++ b/source/isaaclab/isaaclab/visualizers/newton_visualizer_cfg.py @@ -0,0 +1,57 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Configuration for Newton OpenGL Visualizer.""" + +from isaaclab.utils import configclass + +from .visualizer_cfg import VisualizerCfg + + +@configclass +class NewtonVisualizerCfg(VisualizerCfg): + """Configuration for Newton OpenGL visualizer.""" + + visualizer_type: str = "newton" + """Type identifier for Newton visualizer.""" + + window_width: int = 1920 + """Window width in pixels.""" + + window_height: int = 1080 + """Window height in pixels.""" + + update_frequency: int = 1 + """Visualizer update frequency (updates every N frames).""" + + show_joints: bool = False + """Show joint visualization.""" + + show_contacts: bool = False + """Show contact visualization.""" + + show_springs: bool = False + """Show spring visualization.""" + + show_com: bool = False + """Show center of mass visualization.""" + + enable_shadows: bool = True + """Enable shadow rendering.""" + + enable_sky: bool = True + """Enable sky rendering.""" + + enable_wireframe: bool = False + """Enable wireframe rendering.""" + + sky_upper_color: tuple[float, float, float] = (0.2, 0.4, 0.6) + """Sky upper color RGB [0,1].""" + + sky_lower_color: tuple[float, float, float] = (0.5, 0.6, 0.7) + """Sky lower color RGB [0,1].""" + + light_color: tuple[float, float, float] = (1.0, 1.0, 1.0) + """Light color RGB [0,1].""" diff --git a/source/isaaclab/isaaclab/visualizers/ov_visualizer.py b/source/isaaclab/isaaclab/visualizers/ov_visualizer.py new file mode 100644 index 00000000000..d3e2c8ea822 --- /dev/null +++ b/source/isaaclab/isaaclab/visualizers/ov_visualizer.py @@ -0,0 +1,200 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Omniverse-based visualizer using Isaac Sim viewport.""" + +from __future__ import annotations + +import asyncio +import logging +from typing import Any + +from pxr import UsdGeom + +from .ov_visualizer_cfg import OVVisualizerCfg +from .visualizer import Visualizer + +logger = logging.getLogger(__name__) + + +class OVVisualizer(Visualizer): + """Omniverse visualizer using Isaac Sim viewport.""" + + def __init__(self, cfg: OVVisualizerCfg): + super().__init__(cfg) + self.cfg: OVVisualizerCfg = cfg + + self._simulation_app = None + self._viewport_window = None + self._viewport_api = None + self._is_initialized = False + self._sim_time = 0.0 + self._step_counter = 0 + + def initialize(self, scene_data: dict[str, Any] | None = None) -> None: + if self._is_initialized: + logger.warning("[OVVisualizer] Already initialized.") + return + + usd_stage = scene_data["usd_stage"] + scene_data_provider = scene_data["scene_data_provider"] + metadata = scene_data_provider.get_metadata() + + self._ensure_simulation_app() + self._setup_viewport(usd_stage, metadata) + + num_envs = metadata.get("num_envs", 0) + physics_backend = metadata.get("physics_backend", "unknown") + logger.info(f"[OVVisualizer] Initialized ({num_envs} envs, {physics_backend} physics)") + + self._is_initialized = True + + def step(self, dt: float, state: Any | None = None) -> None: + if not self._is_initialized: + return + self._sim_time += dt + self._step_counter += 1 + + def close(self) -> None: + if not self._is_initialized: + return + self._simulation_app = None + self._viewport_window = None + self._viewport_api = None + self._is_initialized = False + + def is_running(self) -> bool: + if self._simulation_app is None: + return False + return self._simulation_app.is_running() + + def is_training_paused(self) -> bool: + return False + + def supports_markers(self) -> bool: + return True + + def supports_live_plots(self) -> bool: + return True + + def set_camera_view( + self, eye: tuple[float, float, float] | list[float], target: tuple[float, float, float] | list[float] + ) -> None: + if not self._is_initialized: + logger.warning("[OVVisualizer] Cannot set camera view - visualizer not initialized.") + return + self._set_viewport_camera(tuple(eye), tuple(target)) + + def _ensure_simulation_app(self) -> None: + import omni.kit.app + + app = omni.kit.app.get_app() + if app is None or not app.is_running(): + raise RuntimeError("[OVVisualizer] Isaac Sim app is not running.") + + try: + from isaacsim import SimulationApp + + sim_app = None + if hasattr(SimulationApp, "_instance") and SimulationApp._instance is not None: + sim_app = SimulationApp._instance + elif hasattr(SimulationApp, "instance") and callable(SimulationApp.instance): + sim_app = SimulationApp.instance() + + if sim_app is not None: + self._simulation_app = sim_app + if self._simulation_app.config.get("headless", False): + logger.warning("[OVVisualizer] Running in headless mode. Viewport may not display.") + except ImportError: + pass + + def _setup_viewport(self, usd_stage, metadata: dict) -> None: + import omni.kit.viewport.utility as vp_utils + from omni.ui import DockPosition + + if self.cfg.create_viewport and self.cfg.viewport_name: + dock_position_map = { + "LEFT": DockPosition.LEFT, + "RIGHT": DockPosition.RIGHT, + "BOTTOM": DockPosition.BOTTOM, + "SAME": DockPosition.SAME, + } + dock_pos = dock_position_map.get(self.cfg.dock_position.upper(), DockPosition.SAME) + + self._viewport_window = vp_utils.create_viewport_window( + name=self.cfg.viewport_name, + width=self.cfg.window_width, + height=self.cfg.window_height, + position_x=50, + position_y=50, + docked=True, + ) + + logger.info(f"[OVVisualizer] Created viewport '{self.cfg.viewport_name}'") + asyncio.ensure_future(self._dock_viewport_async(self.cfg.viewport_name, dock_pos)) + self._create_and_assign_camera(usd_stage) + else: + if self.cfg.viewport_name: + self._viewport_window = vp_utils.get_viewport_window_by_name(self.cfg.viewport_name) + if self._viewport_window is None: + logger.warning(f"[OVVisualizer] Viewport '{self.cfg.viewport_name}' not found. Using active.") + self._viewport_window = vp_utils.get_active_viewport_window() + else: + self._viewport_window = vp_utils.get_active_viewport_window() + + self._viewport_api = self._viewport_window.viewport_api + self._set_viewport_camera(self.cfg.camera_position, self.cfg.camera_target) + + async def _dock_viewport_async(self, viewport_name: str, dock_position) -> None: + import omni.kit.app + import omni.ui + + viewport_window = None + for _ in range(10): + viewport_window = omni.ui.Workspace.get_window(viewport_name) + if viewport_window: + break + await omni.kit.app.get_app().next_update_async() + + if not viewport_window: + logger.warning(f"[OVVisualizer] Could not find viewport window '{viewport_name}'.") + return + + main_viewport = omni.ui.Workspace.get_window("Viewport") + if not main_viewport: + for alt_name in ["/OmniverseKit/Viewport", "Viewport Next"]: + main_viewport = omni.ui.Workspace.get_window(alt_name) + if main_viewport: + break + + if main_viewport and main_viewport != viewport_window: + viewport_window.dock_in(main_viewport, dock_position, 0.5) + await omni.kit.app.get_app().next_update_async() + viewport_window.focus() + viewport_window.visible = True + await omni.kit.app.get_app().next_update_async() + viewport_window.focus() + + def _create_and_assign_camera(self, usd_stage) -> None: + # Create camera prim path based on viewport name (sanitize to enure valid USD path) 1 + camera_path = f"/World/Cameras/{self.cfg.viewport_name}_Camera".replace(" ", "_") + + camera_prim = usd_stage.GetPrimAtPath(camera_path) + if not camera_prim.IsValid(): + UsdGeom.Camera.Define(usd_stage, camera_path) + + if self._viewport_api: + self._viewport_api.set_active_camera(camera_path) + + def _set_viewport_camera(self, position: tuple[float, float, float], target: tuple[float, float, float]) -> None: + import isaacsim.core.utils.viewports as vp_utils + + camera_path = self._viewport_api.get_active_camera() + if not camera_path: + camera_path = "/OmniverseKit_Persp" + + vp_utils.set_camera_view( + eye=list(position), target=list(target), camera_prim_path=camera_path, viewport_api=self._viewport_api + ) diff --git a/source/isaaclab/isaaclab/visualizers/ov_visualizer_cfg.py b/source/isaaclab/isaaclab/visualizers/ov_visualizer_cfg.py new file mode 100644 index 00000000000..33099ef34f8 --- /dev/null +++ b/source/isaaclab/isaaclab/visualizers/ov_visualizer_cfg.py @@ -0,0 +1,33 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Configuration for Omniverse-based visualizer.""" + +from isaaclab.utils import configclass + +from .visualizer_cfg import VisualizerCfg + + +@configclass +class OVVisualizerCfg(VisualizerCfg): + """Configuration for Omniverse visualizer using Isaac Sim viewport.""" + + visualizer_type: str = "omniverse" + """Type identifier for Omniverse visualizer.""" + + viewport_name: str | None = "Visualizer Viewport" + """Viewport name to use. If None, uses active viewport.""" + + create_viewport: bool = True + """Create new viewport with specified name and camera pose.""" + + dock_position: str = "SAME" + """Dock position for new viewport. Options: 'LEFT', 'RIGHT', 'BOTTOM', 'SAME'.""" + + window_width: int = 1280 + """Viewport width in pixels.""" + + window_height: int = 720 + """Viewport height in pixels.""" diff --git a/source/isaaclab/isaaclab/visualizers/rerun_visualizer.py b/source/isaaclab/isaaclab/visualizers/rerun_visualizer.py new file mode 100644 index 00000000000..d73e313cfdd --- /dev/null +++ b/source/isaaclab/isaaclab/visualizers/rerun_visualizer.py @@ -0,0 +1,176 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Rerun-based visualizer using rerun-sdk.""" + +from __future__ import annotations + +import logging +from typing import Any + +import rerun as rr +import rerun.blueprint as rrb +from newton.viewer import ViewerRerun + +from .rerun_visualizer_cfg import RerunVisualizerCfg +from .visualizer import Visualizer + +logger = logging.getLogger(__name__) + + +class NewtonViewerRerun(ViewerRerun): + """Isaac Lab wrapper for Newton's ViewerRerun.""" + + def __init__( + self, + app_id: str | None = None, + web_port: int | None = None, + keep_historical_data: bool = False, + keep_scalar_history: bool = True, + record_to_rrd: str | None = None, + metadata: dict | None = None, + ): + super().__init__( + app_id=app_id, + web_port=web_port, + serve_web_viewer=True, + keep_historical_data=keep_historical_data, + keep_scalar_history=keep_scalar_history, + record_to_rrd=record_to_rrd, + ) + + self._metadata = metadata or {} + self._log_metadata() + + def _log_metadata(self) -> None: + metadata_text = "# Isaac Lab Scene Metadata\n\n" + physics_backend = self._metadata.get("physics_backend", "unknown") + metadata_text += f"**Physics Backend:** {physics_backend}\n" + num_envs = self._metadata.get("num_envs", 0) + metadata_text += f"**Total Environments:** {num_envs}\n" + + for key, value in self._metadata.items(): + if key not in ["physics_backend", "num_envs"]: + metadata_text += f"**{key}:** {value}\n" + + rr.log("metadata", rr.TextDocument(metadata_text, media_type=rr.MediaType.MARKDOWN)) + + +class RerunVisualizer(Visualizer): + """Rerun web-based visualizer with time scrubbing, recording, and data inspection.""" + + def __init__(self, cfg: RerunVisualizerCfg): + super().__init__(cfg) + self.cfg: RerunVisualizerCfg = cfg + self._viewer: NewtonViewerRerun | None = None + self._model = None + self._state = None + self._is_initialized = False + self._sim_time = 0.0 + self._scene_data_provider = None + + def initialize(self, scene_data: dict[str, Any] | None = None) -> None: + if self._is_initialized: + logger.warning("[RerunVisualizer] Already initialized.") + return + + if not scene_data or "scene_data_provider" not in scene_data: + raise RuntimeError("Rerun visualizer requires scene_data_provider.") + + self._scene_data_provider = scene_data["scene_data_provider"] + self._model = self._scene_data_provider.get_newton_model() + self._state = self._scene_data_provider.get_newton_state() + metadata = self._scene_data_provider.get_metadata() + + try: + if self.cfg.record_to_rrd: + logger.info(f"[RerunVisualizer] Recording enabled to: {self.cfg.record_to_rrd}") + + self._viewer = NewtonViewerRerun( + app_id=self.cfg.app_id, + web_port=self.cfg.web_port, + keep_historical_data=self.cfg.keep_historical_data, + keep_scalar_history=self.cfg.keep_scalar_history, + record_to_rrd=self.cfg.record_to_rrd, + metadata=metadata, + ) + + self._viewer.set_model(self._model) + self._viewer.set_world_offsets((0.0, 0.0, 0.0)) + + try: + cam_pos = self.cfg.camera_position + cam_target = self.cfg.camera_target + + blueprint = rrb.Blueprint( + rrb.Spatial3DView( + name="3D View", + origin="/", + eye_controls=rrb.EyeControls3D( + position=cam_pos, + look_target=cam_target, + ), + ), + collapse_panels=True, + ) + rr.send_blueprint(blueprint) + + logger.info(f"[RerunVisualizer] Set initial camera view: position={cam_pos}, target={cam_target}") + except Exception as exc: + logger.warning(f"[RerunVisualizer] Could not set initial camera view: {exc}") + + num_envs = metadata.get("num_envs", 0) + physics_backend = metadata.get("physics_backend", "unknown") + logger.info(f"[RerunVisualizer] Initialized with {num_envs} environments (physics: {physics_backend})") + + self._is_initialized = True + except Exception as exc: + logger.error(f"[RerunVisualizer] Failed to initialize viewer: {exc}") + raise + + def step(self, dt: float, state: Any | None = None) -> None: + self._state = self._scene_data_provider.get_newton_state() + self._sim_time += dt + + self._viewer.begin_frame(self._sim_time) + self._viewer.log_state(self._state) + self._viewer.end_frame() + + def close(self) -> None: + if not self._is_initialized or self._viewer is None: + return + + try: + if self.cfg.record_to_rrd: + logger.info(f"[RerunVisualizer] Finalizing recording to: {self.cfg.record_to_rrd}") + self._viewer.close() + logger.info("[RerunVisualizer] Closed successfully.") + if self.cfg.record_to_rrd: + import os + + if os.path.exists(self.cfg.record_to_rrd): + size = os.path.getsize(self.cfg.record_to_rrd) + logger.info(f"[RerunVisualizer] Recording saved: {self.cfg.record_to_rrd} ({size} bytes)") + else: + logger.warning(f"[RerunVisualizer] Recording file not found: {self.cfg.record_to_rrd}") + except Exception as exc: + logger.warning(f"[RerunVisualizer] Error during close: {exc}") + + self._viewer = None + self._is_initialized = False + + def is_running(self) -> bool: + if self._viewer is None: + return False + return self._viewer.is_running() + + def is_training_paused(self) -> bool: + return False + + def supports_markers(self) -> bool: + return False + + def supports_live_plots(self) -> bool: + return False diff --git a/source/isaaclab/isaaclab/visualizers/rerun_visualizer_cfg.py b/source/isaaclab/isaaclab/visualizers/rerun_visualizer_cfg.py new file mode 100644 index 00000000000..8b1e32d7ded --- /dev/null +++ b/source/isaaclab/isaaclab/visualizers/rerun_visualizer_cfg.py @@ -0,0 +1,35 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Configuration for the Rerun visualizer.""" + +from __future__ import annotations + +from isaaclab.utils import configclass + +from .visualizer_cfg import VisualizerCfg + + +@configclass +class RerunVisualizerCfg(VisualizerCfg): + """Configuration for Rerun visualizer (web-based visualization).""" + + visualizer_type: str = "rerun" + """Type identifier for Rerun visualizer.""" + + app_id: str = "isaaclab-simulation" + """Application identifier shown in viewer title.""" + + web_port: int = 9090 + """Port of the local rerun web viewer which is launched in the browser.""" + + keep_historical_data: bool = False + """Keep transform history for time scrubbing (False = constant memory for training).""" + + keep_scalar_history: bool = False + """Keep scalar/plot history in timeline.""" + + record_to_rrd: str | None = None + """Path to save .rrd recording file. None = no recording.""" diff --git a/source/isaaclab/isaaclab/visualizers/visualizer.py b/source/isaaclab/isaaclab/visualizers/visualizer.py new file mode 100644 index 00000000000..f3991113d3b --- /dev/null +++ b/source/isaaclab/isaaclab/visualizers/visualizer.py @@ -0,0 +1,78 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Base class for visualizers.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .visualizer_cfg import VisualizerCfg + + +class Visualizer(ABC): + """Base class for all visualizer backends. + + Lifecycle: __init__() -> initialize() -> step() (repeated) -> close() + """ + + def __init__(self, cfg: VisualizerCfg): + """Initialize visualizer with config.""" + self.cfg = cfg + self._is_initialized = False + self._is_closed = False + + @abstractmethod + def initialize(self, scene_data: dict[str, Any] | None = None) -> None: + """Initialize visualizer with scene data (model, state, usd_stage, etc.).""" + raise NotImplementedError + + @abstractmethod + def step(self, dt: float, state: Any | None = None) -> None: + """Update visualization for one step. + + Args: + dt: Time step in seconds. + state: Updated physics state (e.g., newton.State). + """ + raise NotImplementedError + + @abstractmethod + def close(self) -> None: + """Clean up resources.""" + raise NotImplementedError + + @abstractmethod + def is_running(self) -> bool: + """Check if visualizer is still running (e.g., window not closed).""" + raise NotImplementedError + + def is_training_paused(self) -> bool: + """Check if training is paused by visualizer controls.""" + return False + + def is_rendering_paused(self) -> bool: + """Check if rendering is paused by visualizer controls.""" + return False + + @property + def is_initialized(self) -> bool: + """Check if initialize() has been called.""" + return self._is_initialized + + @property + def is_closed(self) -> bool: + """Check if close() has been called.""" + return self._is_closed + + def supports_markers(self) -> bool: + """Check if visualizer supports VisualizationMarkers.""" + return False + + def supports_live_plots(self) -> bool: + """Check if visualizer supports LivePlots.""" + return False diff --git a/source/isaaclab/isaaclab/visualizers/visualizer_cfg.py b/source/isaaclab/isaaclab/visualizers/visualizer_cfg.py new file mode 100644 index 00000000000..d86c10504f3 --- /dev/null +++ b/source/isaaclab/isaaclab/visualizers/visualizer_cfg.py @@ -0,0 +1,55 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Base configuration for visualizers.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from isaaclab.utils import configclass + +if TYPE_CHECKING: + from .visualizer import Visualizer + + +@configclass +class VisualizerCfg: + """Base configuration for all visualizer backends.""" + + visualizer_type: str | None = None + """Type identifier (e.g., 'newton', 'rerun', 'omniverse').""" + + enable_markers: bool = True + """Enable visualization markers (debug drawing).""" + + enable_live_plots: bool = True + """Enable live plotting of data.""" + + camera_position: tuple[float, float, float] = (8.0, 8.0, 3.0) + """Initial camera position (x, y, z) in world coordinates.""" + + camera_target: tuple[float, float, float] = (0.0, 0.0, 0.0) + """Initial camera target/look-at point (x, y, z) in world coordinates.""" + + def get_visualizer_type(self) -> str | None: + """Get the visualizer type identifier.""" + return self.visualizer_type + + def create_visualizer(self) -> Visualizer: + """Create visualizer instance from this config using factory pattern.""" + from . import get_visualizer_class + + if self.visualizer_type is None: + raise ValueError( + "Cannot create visualizer from base VisualizerCfg class. " + "Use a specific visualizer config: NewtonVisualizerCfg, RerunVisualizerCfg, or OVVisualizerCfg." + ) + + visualizer_class = get_visualizer_class(self.visualizer_type) + if visualizer_class is None: + raise ValueError(f"Visualizer type '{self.visualizer_type}' is not available.") + + return visualizer_class(self) diff --git a/source/isaaclab/setup.py b/source/isaaclab/setup.py index 939ee294d8d..ef43d1c289f 100644 --- a/source/isaaclab/setup.py +++ b/source/isaaclab/setup.py @@ -29,11 +29,19 @@ "gymnasium==1.2.1", # procedural-generation "trimesh", - "pyglet<2", + "pyglet>=2.1.6", # image processing "transformers==4.57.6", "einops", # needed for transformers, doesn't always auto-install - "warp-lang", + "warp-lang>=1.11.0.dev20251205", + # newton visualizers / backend dependencies + "mujoco>=3.4.0.dev839962392", + "mujoco-warp>=0.0.1", + "cbor2>=5.7.0", + "newton>=0.2.1", + "imgui_bundle>=1.92.0", + "PyOpenGL-accelerate==3.1.10", + "rerun-sdk>=0.27.1", # make sure this is consistent with isaac sim version "pillow==11.3.0", # livestream @@ -56,6 +64,8 @@ f"daqp==0.7.2 ; platform_system == 'Linux' and ({SUPPORTED_ARCHS_ARM})", # required by isaaclab.devices.openxr.retargeters.humanoid.fourier.gr1_t2_dex_retargeting_utils f"dex-retargeting==0.4.6 ; platform_system == 'Linux' and ({SUPPORTED_ARCHS})", + f"usd-core==25.05.0 ; platform_system == 'Linux' and ({SUPPORTED_ARCHS})", + f"usd-exchange>=2.1 ; platform_system == 'Linux' and ({SUPPORTED_ARCHS_ARM})", ] PYTORCH_INDEX_URL = ["https://download.pytorch.org/whl/cu128"] diff --git a/source/isaaclab_tasks/test/benchmarking/configs_custom_reward_thresholds.yaml b/source/isaaclab_tasks/test/benchmarking/configs_custom_reward_thresholds.yaml new file mode 100644 index 00000000000..cb899cd591b --- /dev/null +++ b/source/isaaclab_tasks/test/benchmarking/configs_custom_reward_thresholds.yaml @@ -0,0 +1,9 @@ +custom_rewards: + rl_games:Isaac-Dexsuite-Kuka-Allegro-Lift-v0: + max_iterations: 2000 + upper_thresholds: + duration: 999999 + sb3:Isaac-Lift-Cube-Franka-v0: + max_iterations: 200 + upper_thresholds: + duration: 999999