Fix reward weights and update README (#4)

Author: xzhu0428

README.md

@@ -10,7 +10,7 @@
 
 This repository contains the official implementation for the paper "Versatile Loco-Manipulation through Flexible Interlimb Coordination", accepted to CoRL 2025.
 
-[**Project Website**](https://relic-locoman.rai-inst.com/) | [**arXiv Paper**](https://arxiv.org/abs/2506.07876)
+[**Project Website**](https://relic-locoman.rai-inst.com/) | [**arXiv Paper**](https://arxiv.org/abs/2506.07876) | [**Blog Post**](https://rai-inst.com/resources/blog/reinforcement-learning-for-flexible-loco-manipulation/) | [**X**](https://x.com/rai_inst/status/1971590845817045218) | [**Threads**](https://www.threads.com/@robotics_and_ai_institute/post/DPEdj0UFEnm?xmt=AQF0fkVTtJFB-UTiEjOQATobby44KSl_DdfDymPMpr42Vg)
 
 ---
 *Our ReLIC policy enables a quadruped robot to walk with three legs and manipulate with the arm and one leg.*
@@ -44,4 +44,22 @@ This project uses [Pixi](https://pixi.sh/latest/installation/) to manage depende
 
 Alternatively, you can install the project without Pixi by following the standard installation guides for [IsaacLab](https://isaac-sim.github.io/IsaacLab/v2.1.0/source/setup/installation/isaaclab_pip_installation.html) and its [extensions](https://github.com/isaac-sim/IsaacLabExtensionTemplate/tree/main?tab=readme-ov-file#installation).
 
-## Training
+## Training and Eval
+
+```bash
+python scripts/rsl_rl/train.py --task Isaac-Spot-Interlimb-Phase-1-v0 --headless
+python scripts/rsl_rl/play.py --task Isaac-Spot-Interlimb-Play-v0 --center
+```
+
+To achieve optimal deployment results, we implemented a weight curriculum with multiple training phases. Users can fine-tune the models from `Phase-2` to `Phase-4` to reproduce the results presented in our paper. The pre-trained weights can be found in `relic/source/relic/relic/assets/spot/pretrained`.
+
+## Deployment
+
+Please find deployment code with Boston Dynamics's Spot robot at our deployment branch.
+
+## Adapted Code
+
+We use RSL_RL for RL training and adapt the following scripts from [IsaacLabExtensionTemplate](https://github.com/isaac-sim/IsaacLabExtensionTemplate/tree/main?tab=readme-ov-file#installation)
+- `scripts/rsl_rl/*`
+- `source/relic/pyproject.toml`
+- `source/relic/setup.py`

scripts/rsl_rl/play.py

@@ -35,6 +35,9 @@
     "--num_envs", type=int, default=None, help="Number of environments to simulate."
 )
 parser.add_argument("--task", type=str, default=None, help="Name of the task.")
+parser.add_argument(
+    "--center", action="store_true", default=False, help="Look at the robot."
+)
 # append RSL-RL cli arguments
 cli_args.add_rsl_rl_args(parser)
 # append AppLauncher cli args
@@ -83,6 +86,13 @@ def main():
         args_cli.task, args_cli
     )
 
+    if args_cli.center:
+        env_cfg.viewer.origin_type = "asset_root"
+        env_cfg.viewer.asset_name = "robot"
+        env_cfg.viewer.env_index = 10
+        env_cfg.viewer.eye = (3.0, 3.0, 3.0)
+        env_cfg.viewer.resolution = (4096, 2160)
+
     # specify directory for logging experiments
     log_root_path = os.path.join("logs", "rsl_rl", agent_cfg.experiment_name)
     log_root_path = os.path.abspath(log_root_path)
@@ -125,19 +135,30 @@ def main():
     # obtain the trained policy for inference
     policy = ppo_runner.get_inference_policy(device=env.unwrapped.device)
 
+    # extract the neural network module
+    # we do this in a try-except to maintain backwards compatibility.
+    try:
+        # version 2.3 onwards
+        policy_nn = ppo_runner.alg.policy
+    except AttributeError:
+        # version 2.2 and below
+        policy_nn = ppo_runner.alg.actor_critic
+
+    # extract the normalizer
+    if hasattr(policy_nn, "actor_obs_normalizer"):
+        normalizer = policy_nn.actor_obs_normalizer
+    elif hasattr(policy_nn, "student_obs_normalizer"):
+        normalizer = policy_nn.student_obs_normalizer
+    else:
+        normalizer = None
+
     # export policy to onnx/jit
     export_model_dir = os.path.join(os.path.dirname(resume_path), "exported")
     export_policy_as_jit(
-        ppo_runner.alg.actor_critic,
-        ppo_runner.obs_normalizer,
-        path=export_model_dir,
-        filename="policy.pt",
+        policy_nn, normalizer=normalizer, path=export_model_dir, filename="policy.pt"
     )
     export_policy_as_onnx(
-        ppo_runner.alg.actor_critic,
-        normalizer=ppo_runner.obs_normalizer,
-        path=export_model_dir,
-        filename="policy.onnx",
+        policy_nn, normalizer=normalizer, path=export_model_dir, filename="policy.onnx"
     )
 
     # reset environment

source/relic/relic/assets/spot/pretrained/policy.onnx

@@ -65,17 +65,17 @@
             effort_limit=HIP_EFFORT_LIMIT,
             stiffness=HIP_STIFFNESS,
             damping=HIP_DAMPING,
-            min_delay=1,  # physics time steps (min: 5.0*1=5.0ms)
-            max_delay=2,  # physics time steps (max: 5.0*2=10.0ms)
+            min_delay=0,  # physics time steps (min: 5.0*1=5.0ms)
+            max_delay=3,  # physics time steps (max: 5.0*2=10.0ms)
         ),
         "spot_knee": SpotKneeActuatorCfg(
             joint_names_expr=[".*_kn"],
             joint_parameter_lookup=JOINT_PARAMETER_LOOKUP_TABLE,
             effort_limit=None,  # torque limits are handled based experimental data
             stiffness=KNEE_STIFFNESS,
             damping=KNEE_DAMPING,
-            min_delay=1,  # physics time steps (min: 5.0*1=5.0ms)
-            max_delay=2,  # physics time steps (max: 5.0*2=10.0ms)
+            min_delay=0,  # physics time steps (min: 5.0*1=5.0ms)
+            max_delay=3,  # physics time steps (max: 5.0*2=10.0ms)
             enable_torque_speed_limit=True,
         ),
         "spot_arm_sh0": DelayedPDActuatorCfg(
@@ -84,62 +84,62 @@
             stiffness=ARM_STIFFNESS[0],
             damping=ARM_DAMPING[0],
             armature=ARM_ARMATURE[0],
-            min_delay=1,  # physics time steps (min: 5.0*1=5.0ms)
-            max_delay=2,  # physics time steps (max: 5.0*2=10.0ms)
+            min_delay=0,  # physics time steps (min: 5.0*1=5.0ms)
+            max_delay=3,  # physics time steps (max: 5.0*2=10.0ms)
         ),
         "spot_arm_sh1": DelayedPDActuatorCfg(
             joint_names_expr=["arm_sh1"],
             effort_limit=ARM_EFFORT_LIMIT[1],
             stiffness=ARM_STIFFNESS[1],
             damping=ARM_DAMPING[1],
             armature=ARM_ARMATURE[1],
-            min_delay=1,  # physics time steps (min: 5.0*1=5.0ms)
-            max_delay=2,  # physics time steps (max: 5.0*2=10.0ms)
+            min_delay=0,  # physics time steps (min: 5.0*1=5.0ms)
+            max_delay=3,  # physics time steps (max: 5.0*2=10.0ms)
         ),
         "spot_arm_el0": DelayedPDActuatorCfg(
             joint_names_expr=["arm_el0"],
             effort_limit=ARM_EFFORT_LIMIT[2],
             stiffness=ARM_STIFFNESS[2],
             damping=ARM_DAMPING[2],
             armature=ARM_ARMATURE[2],
-            min_delay=1,  # physics time steps (min: 5.0*1=5.0ms)
-            max_delay=2,  # physics time steps (max: 5.0*2=10.0ms)
+            min_delay=0,  # physics time steps (min: 5.0*1=5.0ms)
+            max_delay=3,  # physics time steps (max: 5.0*2=10.0ms)
         ),
         "spot_arm_el1": DelayedPDActuatorCfg(
             joint_names_expr=["arm_el1"],
             effort_limit=ARM_EFFORT_LIMIT[3],
             stiffness=ARM_STIFFNESS[3],
             damping=ARM_DAMPING[3],
             armature=ARM_ARMATURE[3],
-            min_delay=1,  # physics time steps (min: 5.0*1=5.0ms)
-            max_delay=2,  # physics time steps (max: 5.0*2=10.0ms)
+            min_delay=0,  # physics time steps (min: 5.0*1=5.0ms)
+            max_delay=3,  # physics time steps (max: 5.0*2=10.0ms)
         ),
         "spot_arm_wr0": DelayedPDActuatorCfg(
             joint_names_expr=["arm_wr0"],
             effort_limit=ARM_EFFORT_LIMIT[4],
             stiffness=ARM_STIFFNESS[4],
             damping=ARM_DAMPING[4],
             armature=ARM_ARMATURE[4],
-            min_delay=1,  # physics time steps (min: 5.0*1=5.0ms)
-            max_delay=2,  # physics time steps (max: 5.0*2=10.0ms)
+            min_delay=0,  # physics time steps (min: 5.0*1=5.0ms)
+            max_delay=3,  # physics time steps (max: 5.0*2=10.0ms)
         ),
         "spot_arm_wr1": DelayedPDActuatorCfg(
             joint_names_expr=["arm_wr1"],
             effort_limit=ARM_EFFORT_LIMIT[5],
             stiffness=ARM_STIFFNESS[5],
             damping=ARM_DAMPING[5],
             armature=ARM_ARMATURE[5],
-            min_delay=1,  # physics time steps (min: 5.0*1=5.0ms)
-            max_delay=2,  # physics time steps (max: 5.0*2=10.0ms)
+            min_delay=0,  # physics time steps (min: 5.0*1=5.0ms)
+            max_delay=3,  # physics time steps (max: 5.0*2=10.0ms)
         ),
         "spot_arm_f1x": DelayedPDActuatorCfg(
             joint_names_expr=["arm_f1x"],
             effort_limit=ARM_EFFORT_LIMIT[6],
             stiffness=ARM_STIFFNESS[6],
             damping=ARM_DAMPING[6],
             armature=ARM_ARMATURE[6],
-            min_delay=1,  # physics time steps (min: 5.0*1=5.0ms)
-            max_delay=2,  # physics time steps (max: 5.0*2=10.0ms)
+            min_delay=0,  # physics time steps (min: 5.0*1=5.0ms)
+            max_delay=3,  # physics time steps (max: 5.0*2=10.0ms)
         ),
     },
 )

source/relic/relic/assets/spot/pretrained/policy.pt

@@ -38,6 +38,16 @@
     },
 )
 
+gym.register(
+    id="Isaac-Spot-Interlimb-Phase-4-v0",
+    entry_point="isaaclab.envs:ManagerBasedRLEnv",
+    disable_env_checker=True,
+    kwargs={
+        "env_cfg_entry_point": spot_env_cfg.SpotInterlimbEnvCfg_Phase_4,
+        "rsl_rl_cfg_entry_point": agents.rsl_rl_cfg.SpotInterlimbPPORunnerCfg,
+    },
+)
+
 gym.register(
     id="Isaac-Spot-Interlimb-Play-v0",
     entry_point="isaaclab.envs:ManagerBasedRLEnv",

source/relic/relic/assets/spot/spot.py

@@ -19,7 +19,22 @@ def __post_init__(self):
         # switch robot to spot-arm
         self.scene.robot = SPOT_ARM_CFG.replace(prim_path="{ENV_REGEX_NS}/Robot")
         self.scene.robot.spawn.joint_drive.gains.stiffness = None
-        # self.rewards.foot_clearance = None
+
+        # Weights
+        self.rewards.track_lin_vel_xy_exp.weight = 7.0
+        self.rewards.track_ang_vel_z_exp.weight = 3.5
+        self.rewards.track_base_orientation_l2.weight = -7.0
+        self.rewards.track_base_height_l2.weight = -3.5
+        self.rewards.dof_torques_l2.weight = -1.0e-05
+        self.rewards.dof_acc_l2.weight = -2.5e-7
+        self.rewards.action_rate_l2.weight = -0.01
+        self.rewards.feet_air_time.weight = 0.01
+        self.rewards.joint_energy.weight = 2.0
+        self.rewards.air_time_variance.weight = -10.0
+        self.rewards.flight_penalty.weight = -5.0
+        self.rewards.foot_impact.weight = -1.0
+        self.rewards.foot_slip.weight = -0.5
+        self.rewards.foot_clearance.weight = 0.0
 
 
 @configclass
@@ -31,25 +46,29 @@ def __post_init__(self):
         super().__post_init__()
         # switch robot to spot-arm
         self.scene.robot = SPOT_ARM_CFG.replace(prim_path="{ENV_REGEX_NS}/Robot")
+        self.scene.robot.spawn.joint_drive.gains.stiffness = None
 
-        # change event and termination terms
-        self.events.physics_material.params["static_friction_range"] = (0.8, 0.8)
-        self.events.physics_material.params["dynamic_friction_range"] = (0.6, 0.6)
+        # relax contact events
         self.terminations.undesired_ground_contact.params["sensor_cfg"].body_names = [
             ".*uleg"
         ]
 
-        # change the reward weights
-        self.rewards.dof_torques_l2.weight = (-1.0e-4,)  # -1.0e-05
-        self.rewards.dof_acc_l2.weight = (-1.0e-6,)  # -2.5e-7
-        self.rewards.action_rate_l2.weight = -0.1  # -0.01
+        # Weights
+        self.rewards.track_lin_vel_xy_exp.weight = 7.0
+        self.rewards.track_ang_vel_z_exp.weight = 3.5
         self.rewards.track_base_orientation_l2.weight = -7.0
         self.rewards.track_base_height_l2.weight = -30.0
+        self.rewards.dof_torques_l2.weight = -0.0001
+        self.rewards.dof_acc_l2.weight = -1.0e-06
+        self.rewards.action_rate_l2.weight = -0.1
+        self.rewards.feet_air_time.weight = 0.01
+        self.rewards.joint_energy.weight = 2.0
+        self.rewards.air_time_variance.weight = -10.0
         self.rewards.flight_penalty.weight = -10.0
+        self.rewards.foot_impact.weight = -1.0
+        self.rewards.foot_slip.weight = -0.5
         self.rewards.foot_clearance.weight = 1.0
 
-        self.rewards.foot_slip.weight = -1.0
-
 
 @configclass
 class SpotInterlimbEnvCfg_Phase_3(InterlimbEnvCfg):
@@ -60,31 +79,65 @@ def __post_init__(self):
         super().__post_init__()
         # switch robot to spot-arm
         self.scene.robot = SPOT_ARM_CFG.replace(prim_path="{ENV_REGEX_NS}/Robot")
+        self.scene.robot.spawn.joint_drive.gains.stiffness = None
 
         # change event and termination terms
-        self.events.physics_material.params["static_friction_range"] = (0.3, 1.0)
-        self.events.physics_material.params["dynamic_friction_range"] = (0.3, 0.8)
         self.terminations.undesired_ground_contact.params["sensor_cfg"].body_names = [
             ".*uleg"
         ]
 
-        # change the reward weights
-        # self.rewards.dof_torques_l2.weight = -5.0e-4 # -1.0e-4, -1.0e-05
-        # self.rewards.dof_acc_l2.weight = -2.0e-6 # -1.0e-6, -2.5e-7
-        self.rewards.dof_torques_l2.params["leg_ctl_weight"] = -2.0e-4
-        self.rewards.dof_acc_l2.params["leg_ctl_weight"] = -1.0e-6
-
-        self.rewards.action_rate_l2.weight = -0.1
+        # Weights
+        self.rewards.track_lin_vel_xy_exp.weight = 7.0
+        self.rewards.track_ang_vel_z_exp.weight = 3.5
         self.rewards.track_base_orientation_l2.weight = -45.0
         self.rewards.track_base_height_l2.weight = -120.0
+        self.rewards.dof_torques_l2.weight = -0.0001
+        self.rewards.dof_acc_l2.weight = -1.0e-06
+        self.rewards.action_rate_l2.weight = -0.1
+        self.rewards.feet_air_time.weight = 0.01
+        self.rewards.joint_energy.weight = 2.0
+        self.rewards.air_time_variance.weight = -10.0
         self.rewards.flight_penalty.weight = -10.0
+        self.rewards.foot_impact.weight = -1.0
+        self.rewards.foot_slip.weight = -1.0
         self.rewards.foot_clearance.weight = 1.0
-        self.rewards.foot_slip.weight = -2.0
 
-        self.rewards.feet_air_time_target.weight = 0.5
+
+@configclass
+class SpotInterlimbEnvCfg_Phase_4(InterlimbEnvCfg):
+    """Configuration for the phase 4 environment."""
+
+    def __post_init__(self):
+        # post init of parent
+        super().__post_init__()
+        # switch robot to spot-arm
+        self.scene.robot = SPOT_ARM_CFG.replace(prim_path="{ENV_REGEX_NS}/Robot")
+        self.scene.robot.spawn.joint_drive.gains.stiffness = None
+
+        # change event and termination terms
+        self.terminations.undesired_ground_contact.params["sensor_cfg"].body_names = [
+            ".*uleg"
+        ]
+
+        # Weights
+        self.rewards.track_lin_vel_xy_exp.weight = 7.0
+        self.rewards.track_ang_vel_z_exp.weight = 3.5
+        self.rewards.track_base_orientation_l2.weight = -45.0
+        self.rewards.track_base_height_l2.weight = -120.0
+        self.rewards.dof_torques_l2.weight = -0.0001
+        self.rewards.dof_acc_l2.weight = -1.0e-06
+        self.rewards.action_rate_l2.weight = -0.1
+        self.rewards.feet_air_time.weight = 0.01
+        self.rewards.joint_energy.weight = 2.0
         self.rewards.air_time_variance.weight = 0.0
-        self.rewards.lin_vel_z_l2.weight = -0.0
-        self.rewards.ang_vel_xy_l2.weight = -0.0
+        self.rewards.flight_penalty.weight = -10.0
+        self.rewards.foot_impact.weight = -1.0
+        self.rewards.foot_slip.weight = -2.0
+        self.rewards.foot_clearance.weight = 1.0
+        self.rewards.dof_torque_limits_l2.weight = -1.0
+        self.rewards.gait.weight = 5.0
+        self.rewards.three_leg_gait.weight = 5.0
+        self.rewards.feet_air_time_target.weight = 0.5
 
 
 @configclass