[Task Clean-up] Dexterous Part 4/8: Fix MARL-to-single-agent training and enable handover Direct RSL-RL

source/isaaclab/changelog.d/dexterous-env-convergence.rst

@@ -0,0 +1,7 @@
+Fixed
+^^^^^
+
+* Fixed :meth:`~isaaclab.envs.DirectRLEnv.reset` to store the observation buffer
+  like :meth:`~isaaclab.envs.DirectRLEnv.step` already does, and exposed the
+  latest observations on the multi-agent-to-single-agent adapter through the
+  same public buffer.

source/isaaclab/isaaclab/envs/direct_rl_env.py

@@ -382,7 +382,9 @@ def reset(self, seed: int | None = None, options: dict[str, Any] | None = None)
                     self.sim.render()
 
         # return observations
-        return self._get_observations(), self.extras
+        # store the buffer like step() does, so consumers can read the latest observations
+        self.obs_buf = self._get_observations()
+        return self.obs_buf, self.extras
 
     def step(self, action: torch.Tensor) -> VecEnvStepReturn:
         """Execute one time-step of the environment's dynamics.

source/isaaclab/isaaclab/envs/utils/marl.py

@@ -81,22 +81,34 @@ def __init__(self, env: DirectMARLEnv) -> None:
             )
             self.action_space = gym.vector.utils.batch_space(self.single_action_space, self.num_envs)
 
-        def reset(self, seed: int | None = None, options: dict[str, Any] | None = None) -> tuple[VecEnvObs, dict]:
-            obs, extras = self.env.reset(seed, options)
+        @property
+        def episode_length_buf(self) -> torch.Tensor:
+            """Episode lengths from the wrapped multi-agent environment."""
+            return self.env.episode_length_buf
 
-            # use environment state as observation
-            if self._state_as_observation:
-                obs = {"policy": self.env.state()}
-            # concatenate agents' observations
+        @episode_length_buf.setter
+        def episode_length_buf(self, value: torch.Tensor) -> None:
+            self.env.episode_length_buf = value
+
+        @property
+        def obs_buf(self) -> VecEnvObs:
+            """Latest observations from the wrapped multi-agent environment."""
+            return self._convert_observations(self.env.obs_dict)
+
+        def _convert_observations(self, obs: dict[AgentID, ObsType]) -> VecEnvObs:
+            """Convert multi-agent observations to the single-agent policy observation."""
             # FIXME: This implementation assumes the spaces are fundamental ones. Fix it to support composite spaces
-            else:
-                obs = {
-                    "policy": torch.cat(
-                        [obs[agent].reshape(self.num_envs, -1) for agent in self.env.possible_agents], dim=-1
-                    )
-                }
+            if self._state_as_observation:
+                return {"policy": self.env.state()}
+            return {
+                "policy": torch.cat(
+                    [obs[agent].reshape(self.num_envs, -1) for agent in self.env.possible_agents], dim=-1
+                )
+            }
 
-            return obs, extras
+        def reset(self, seed: int | None = None, options: dict[str, Any] | None = None) -> tuple[VecEnvObs, dict]:
+            obs, extras = self.env.reset(seed, options)
+            return self._convert_observations(obs), extras
 
         def step(self, action: torch.Tensor) -> VecEnvStepReturn:
             # split single-agent actions to build the multi-agent ones
@@ -111,17 +123,7 @@ def step(self, action: torch.Tensor) -> VecEnvStepReturn:
             # step the environment
             obs, rewards, terminated, time_outs, extras = self.env.step(_actions)
 
-            # use environment state as observation
-            if self._state_as_observation:
-                obs = {"policy": self.env.state()}
-            # concatenate agents' observations
-            # FIXME: This implementation assumes the spaces are fundamental ones. Fix it to support composite spaces
-            else:
-                obs = {
-                    "policy": torch.cat(
-                        [obs[agent].reshape(self.num_envs, -1) for agent in self.env.possible_agents], dim=-1
-                    )
-                }
+            obs = self._convert_observations(obs)
 
             # process environment outputs to return single-agent data
             rewards = sum(rewards.values())

source/isaaclab/test/envs/test_marl_utils.py

@@ -0,0 +1,82 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+from types import SimpleNamespace
+
+import gymnasium as gym
+import torch
+
+from isaaclab.envs.utils.marl import multi_agent_to_single_agent
+
+
+class _FakeMultiAgentEnv:
+    possible_agents = ["agent_0", "agent_1"]
+    observation_spaces = {
+        "agent_0": gym.spaces.Box(low=-1.0, high=1.0, shape=(2,)),
+        "agent_1": gym.spaces.Box(low=-1.0, high=1.0, shape=(1,)),
+    }
+    action_spaces = {
+        "agent_0": gym.spaces.Box(low=-1.0, high=1.0, shape=(1,)),
+        "agent_1": gym.spaces.Box(low=-1.0, high=1.0, shape=(1,)),
+    }
+    render_mode = None
+
+    def __init__(self):
+        self.unwrapped = self
+        self.cfg = SimpleNamespace(state_space=2)
+        self.state_space = gym.spaces.Box(low=-1.0, high=1.0, shape=(2,))
+        self.sim = object()
+        self.scene = SimpleNamespace(num_envs=2)
+        self.episode_length_buf = torch.tensor([1, 2])
+        self.obs_dict = {
+            "agent_0": torch.tensor([[1.0, 2.0], [3.0, 4.0]]),
+            "agent_1": torch.tensor([[5.0], [6.0]]),
+        }
+
+    def reset(self, seed=None, options=None):
+        return self.obs_dict, {}
+
+    def state(self):
+        return torch.tensor([[7.0, 8.0], [9.0, 10.0]])
+
+    def close(self):
+        pass
+
+
+def test_multi_agent_to_single_agent_reset_concatenates_agents():
+    """The adapter reset should concatenate the agents' observations."""
+    env = multi_agent_to_single_agent(_FakeMultiAgentEnv())
+
+    observations, _ = env.reset()
+
+    torch.testing.assert_close(observations["policy"], torch.tensor([[1.0, 2.0, 5.0], [3.0, 4.0, 6.0]]))
+
+
+def test_multi_agent_to_single_agent_reset_can_use_state():
+    """The adapter reset should support the state-as-observation mode."""
+    env = multi_agent_to_single_agent(_FakeMultiAgentEnv(), state_as_observation=True)
+
+    observations, _ = env.reset()
+
+    torch.testing.assert_close(observations["policy"], torch.tensor([[7.0, 8.0], [9.0, 10.0]]))
+
+
+def test_multi_agent_to_single_agent_forwards_episode_lengths():
+    """RSL-RL episode randomization should update the wrapped environment buffer."""
+    source_env = _FakeMultiAgentEnv()
+    env = multi_agent_to_single_agent(source_env)
+    episode_lengths = torch.tensor([3, 4])
+
+    env.episode_length_buf = episode_lengths
+
+    assert env.episode_length_buf is episode_lengths
+    assert source_env.episode_length_buf is episode_lengths
+
+
+def test_multi_agent_to_single_agent_exposes_latest_observations():
+    """The public observation buffer should reflect the wrapped environment's buffer."""
+    env = multi_agent_to_single_agent(_FakeMultiAgentEnv())
+
+    torch.testing.assert_close(env.obs_buf["policy"], torch.tensor([[1.0, 2.0, 5.0], [3.0, 4.0, 6.0]]))

source/isaaclab_experimental/changelog.d/dexterous-env-convergence.rst

@@ -0,0 +1,7 @@
+Fixed
+^^^^^
+
+* Fixed :meth:`~isaaclab_experimental.envs.DirectRLEnvWarp.step` and
+  :meth:`~isaaclab_experimental.envs.DirectRLEnvWarp.reset` to store the returned
+  observation dictionary in ``obs_buf``, which
+  :meth:`~isaaclab_rl.rsl_rl.RslRlVecEnvWrapper.get_observations` now reads.

source/isaaclab_experimental/isaaclab_experimental/envs/direct_rl_env_warp.py

@@ -377,7 +377,9 @@ def reset(self, seed: int | None = None, options: dict[str, Any] | None = None)
 
         # return observations
         self._get_observations()
-        return {"policy": self.torch_obs_buf.clone()}, self.extras
+        # store the returned buffer so RslRlVecEnvWrapper.get_observations() can read env.obs_buf
+        self.obs_buf = {"policy": self.torch_obs_buf.clone()}
+        return self.obs_buf, self.extras
 
     @Timer(name="env_step", msg="Step took:", enable=DEBUG_TIMER_STEP or DEBUG_TIMERS)
     def step(self, action: torch.Tensor) -> VecEnvStepReturn:
@@ -460,8 +462,10 @@ def step(self, action: torch.Tensor) -> VecEnvStepReturn:
             self._post_step_visualize()
 
         # return observations, rewards, resets and extras
+        # store the returned buffer so RslRlVecEnvWrapper.get_observations() can read env.obs_buf
+        self.obs_buf = {"policy": self.torch_obs_buf.clone()}
         return (
-            {"policy": self.torch_obs_buf.clone()},
+            self.obs_buf,
             self.torch_reward_buf,
             self.torch_reset_terminated,
             self.torch_reset_time_outs,

source/isaaclab_rl/changelog.d/dexterous-env-convergence.minor.rst

@@ -0,0 +1,10 @@
+Changed
+^^^^^^^
+
+* Changed :meth:`~isaaclab_rl.rsl_rl.RslRlVecEnvWrapper.get_observations` to read
+  the environment-owned observation buffer instead of calling private environment
+  methods. The returned observations now match the latest reset/step returns,
+  including observation-noise corruption that the private path skipped, and
+  multi-agent environments converted with
+  :func:`~isaaclab.envs.utils.multi_agent_to_single_agent` train with RSL-RL
+  without environment-side accommodations.

source/isaaclab_rl/isaaclab_rl/rsl_rl/vecenv_wrapper.py

@@ -171,11 +171,7 @@ def reset(self) -> tuple[TensorDict, dict]:  # noqa: D102
 
     def get_observations(self) -> TensorDict:
         """Returns the current observations of the environment."""
-        if hasattr(self.unwrapped, "observation_manager"):
-            obs_dict = self.unwrapped.observation_manager.compute()
-        else:
-            obs_dict = self.unwrapped._get_observations()
-        return TensorDict(obs_dict, batch_size=[self.num_envs])
+        return TensorDict(self.unwrapped.obs_buf, batch_size=[self.num_envs])
 
     def step(self, actions: torch.Tensor) -> tuple[TensorDict, torch.Tensor, torch.Tensor, dict]:
         # clip actions

source/isaaclab_tasks/changelog.d/task-cleanup-dex-part03.minor.rst

@@ -0,0 +1,13 @@
+Added
+^^^^^
+
+* Added behavioral-success metrics and threshold-independent episode-error
+  diagnostics to the dexterous reorientation environments.
+
+Fixed
+^^^^^
+
+* Fixed dexterous hand resets that could initialize joints below their lower
+  position limits. Reset joint positions now sample uniformly across the full
+  joint range; previously the distribution was biased toward the lower half of
+  the range.

source/isaaclab_tasks/changelog.d/task-cleanup-dex-part04.minor.rst

@@ -0,0 +1,25 @@
+Added
+^^^^^
+
+* Added an RSL-RL training configuration and behavioral-success metrics to the
+  Shadow handover Direct task.
+* Added renderer presets and configuration validation to the Shadow camera
+  Direct task, including an RGB-depth preset for training with the Newton Warp
+  renderer.
+* Added OVPhysX physics presets to the handover and camera Direct
+  environments.
+
+Deprecated
+^^^^^^^^^^
+
+* Deprecated ``shadow_hand_camera_env.compute_keypoints`` in favor of
+  :func:`~isaaclab_tasks.core.reorient.mdp.observations.compute_cube_keypoints`.
+* Deprecated the ``Isaac-Reorient-Cube-Shadow-Camera-Benchmark-Direct``
+  registration in favor of the regular camera task with the
+  ``env.feature_extractor.enabled=False`` override.
+
+Fixed
+^^^^^
+
+* Fixed handover construction on Newton, broken by renamed distal joints in
+  the current Shadow Newton asset.

source/isaaclab_tasks/isaaclab_tasks/core/handover/__init__.py

@@ -21,6 +21,7 @@
     disable_env_checker=True,
     kwargs={
         "env_cfg_entry_point": f"{__name__}.handover_env_cfg:HandoverEnvCfg",
+        "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:HandoverPPORunnerCfg",
         "rl_games_cfg_entry_point": f"{agents.__name__}:rl_games_ppo_cfg.yaml",
         "skrl_cfg_entry_point": f"{agents.__name__}:skrl_ppo_cfg.yaml",
         "skrl_ippo_cfg_entry_point": f"{agents.__name__}:skrl_ippo_cfg.yaml",

source/isaaclab_tasks/isaaclab_tasks/core/handover/agents/rsl_rl_ppo_cfg.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+from isaaclab.utils.configclass import configclass
+
+from isaaclab_rl.rsl_rl import RslRlMLPModelCfg, RslRlOnPolicyRunnerCfg, RslRlPpoAlgorithmCfg
+
+
+@configclass
+class HandoverPPORunnerCfg(RslRlOnPolicyRunnerCfg):
+    """RSL-RL PPO configuration for the single-agent view of Shadow Hand handover."""
+
+    num_steps_per_env = 16
+    max_iterations = 5000
+    save_interval = 250
+    experiment_name = "handover"
+    obs_groups = {"actor": ["policy"], "critic": ["policy"]}
+    actor = RslRlMLPModelCfg(
+        hidden_dims=[512, 512, 256, 128],
+        activation="elu",
+        obs_normalization=True,
+        distribution_cfg=RslRlMLPModelCfg.GaussianDistributionCfg(init_std=1.0),
+    )
+    critic = RslRlMLPModelCfg(
+        hidden_dims=[512, 512, 256, 128],
+        activation="elu",
+        obs_normalization=True,
+    )
+    algorithm = RslRlPpoAlgorithmCfg(
+        value_loss_coef=1.0,
+        use_clipped_value_loss=True,
+        clip_param=0.2,
+        entropy_coef=0.005,
+        num_learning_epochs=5,
+        num_mini_batches=4,
+        learning_rate=5.0e-4,
+        schedule="adaptive",
+        gamma=0.99,
+        lam=0.95,
+        desired_kl=0.016,
+        max_grad_norm=1.0,
+    )