Lift Task Kinova Drifting Away from Task

[alex-newhouse]

I am training the Kinova Gen3 n7 robot to grab and lift the dex cube but am noticing around iteration 500 the robot approaches the dex cube but then begins retracting in the opposite direction. Can someone help me understand why this occurs?

IsaacSim: 5.1.0
IsaacLab: 2.3.1

I also notice that the grippers don't move that much during the training, especially as the training progresses, the grippers appear to stay open majority of the time, whereas in the begin they will open/close more.

I have tried changing the damping and stiffness values for the joints, the solver_position_iteration_count and solver_velocity_iteration_count for the robot and cube, velocity_limit_sim for the joints and gripper in joint_pos_env_cfg.py.

What else could I change so that the robot no longer retreats mid-training and picks up the dex cube instead?

This is my lift_env_cfg.py

@configclass
class ObjectTableSceneCfg(InteractiveSceneCfg):
    # robots: will be populated by agent env cfg
    robot: ArticulationCfg = MISSING
    # end-effector sensor: will be populated by agent env cfg
    ee_frame: FrameTransformerCfg = MISSING
    # target object: will be populated by agent env cfg
    object: RigidObjectCfg | DeformableObjectCfg = MISSING

    # Table
    table = AssetBaseCfg(
        prim_path="{ENV_REGEX_NS}/Table",
        init_state=AssetBaseCfg.InitialStateCfg(pos=[0.5, 0, 0], rot=[0.707, 0, 0, 0.707]),
        spawn=UsdFileCfg(usd_path="/home/dgx_adi/isaacsim_assets/Assets/Isaac/5.1/Isaac/Props/Mounts/SeattleLabTable/table_instanceable.usd"),
    )

    # plane
    plane = AssetBaseCfg(
        prim_path="/World/GroundPlane",
        init_state=AssetBaseCfg.InitialStateCfg(pos=[0, 0, -1.05]),
        spawn=GroundPlaneCfg(),
    )

    # lights
    light = AssetBaseCfg(
        prim_path="/World/light",
        spawn=sim_utils.DomeLightCfg(color=(0.75, 0.75, 0.75), intensity=3000.0),
    )


##
# MDP settings
##


@configclass
class CommandsCfg:
    """Command terms for the MDP."""

    object_pose = mdp.UniformPoseCommandCfg(
        asset_name="robot",
        # body_name=MISSING,  # will be set by agent env cfg
        body_name="end_effector_link", 
        resampling_time_range=(6.0, 8.0),
        debug_vis=True,
        ranges=mdp.UniformPoseCommandCfg.Ranges(
            pos_x=(0.48, 0.52),           # narrower approach window
            pos_y=(-0.03, 0.03),          # keep near midline
            pos_z=(0.07, 0.10),           # slightly above cube for pre-grasp
            roll=(0.0, 0.0),
            pitch=(1.57079632679, 1.57079632679),  # tool vertical (90°)
            yaw=(-0.10, 0.10)
        ),

    )

@configclass
class ActionsCfg:
    """Action specifications for the MDP."""

    # will be set by agent env cfg
    arm_action: mdp.JointPositionActionCfg | mdp.DifferentialInverseKinematicsActionCfg = MISSING
    gripper_action: mdp.BinaryJointPositionActionCfg = MISSING


@configclass
class ObservationsCfg:
    """Observation specifications for the MDP."""

    @configclass
    class PolicyCfg(ObsGroup):
        """Observations for policy group."""

        joint_pos = ObsTerm(func=mdp.joint_pos_rel)
        joint_vel = ObsTerm(func=mdp.joint_vel_rel)
        object_position = ObsTerm(func=mdp.object_position_in_robot_root_frame)
        target_object_position = ObsTerm(func=mdp.generated_commands, params={"command_name": "object_pose"})
        actions = ObsTerm(func=mdp.last_action)

        def __post_init__(self):
            self.enable_corruption = True
            self.concatenate_terms = True

    # observation groups
    policy: PolicyCfg = PolicyCfg()


@configclass
class EventCfg:
    """Configuration for events."""

    reset_all = EventTerm(func=mdp.reset_scene_to_default, mode="reset")

    reset_object_position = EventTerm(
        func=mdp.reset_root_state_uniform,
        mode="reset",
        params={
            "pose_range": {"x": (-0.1, 0.1), "y": (-0.25, 0.25), "z": (0.0, 0.0)},
            "velocity_range": {},
            "asset_cfg": SceneEntityCfg("object", body_names="Object"),
        },
    )


@configclass
class RewardsCfg:
    """Reward terms for the MDP."""

    reaching_object = RewTerm(func=mdp.object_ee_distance, params={"std": 0.1}, weight=1.0)

    lifting_object = RewTerm(func=mdp.object_is_lifted, params={"minimal_height": 0.04}, weight=15.0)

    object_goal_tracking = RewTerm(
        func=mdp.object_goal_distance,
        params={"std": 0.3, "minimal_height": 0.04, "command_name": "object_pose"},
        weight=16.0,
    )

    object_goal_tracking_fine_grained = RewTerm(
        func=mdp.object_goal_distance,
        params={"std": 0.05, "minimal_height": 0.04, "command_name": "object_pose"},
        weight=5.0,
    )

    # action penalty
    action_rate = RewTerm(func=mdp.action_rate_l2, weight=-1e-4)

    joint_vel = RewTerm(
        func=mdp.joint_vel_l2,
        weight=-1e-4,
        params={"asset_cfg": SceneEntityCfg("robot")},
    )

@configclass
class TerminationsCfg:
    """Termination terms for the MDP."""

    time_out = DoneTerm(func=mdp.time_out, time_out=True)

    object_dropping = DoneTerm(
        func=mdp.root_height_below_minimum, params={"minimum_height": -0.05, "asset_cfg": SceneEntityCfg("object")}
    )


@configclass
class CurriculumCfg:
    """Curriculum terms for the MDP."""

    action_rate = CurrTerm(
        func=mdp.modify_reward_weight, params={"term_name": "action_rate", "weight": -1e-1, "num_steps": 10000}
    )

    joint_vel = CurrTerm(
        func=mdp.modify_reward_weight, params={"term_name": "joint_vel", "weight": -1e-1, "num_steps": 10000}
    )


##
# Environment configuration
##


@configclass
class LiftEnvCfg(ManagerBasedRLEnvCfg):
    """Configuration for the lifting environment."""

    # Scene settings
    scene: ObjectTableSceneCfg = ObjectTableSceneCfg(num_envs=4096, env_spacing=2.5)
    # Basic settings
    observations: ObservationsCfg = ObservationsCfg()
    actions: ActionsCfg = ActionsCfg()
    commands: CommandsCfg = CommandsCfg()
    # MDP settings
    rewards: RewardsCfg = RewardsCfg()
    terminations: TerminationsCfg = TerminationsCfg()
    events: EventCfg = EventCfg()
    curriculum: CurriculumCfg = CurriculumCfg()

    def __post_init__(self):
        """Post initialization."""
        # general settings
        self.decimation = 2
        self.episode_length_s = 5.0
        # simulation settings
        self.sim.dt = 0.01  # 100Hz
        self.sim.render_interval = self.decimation

        self.sim.physx.bounce_threshold_velocity = 0.2
        self.sim.physx.bounce_threshold_velocity = 0.01
        self.sim.physx.gpu_found_lost_aggregate_pairs_capacity = 1024 * 1024 * 4
        self.sim.physx.gpu_total_aggregate_pairs_capacity = 16 * 1024
        self.sim.physx.friction_correlation_distance = 0.00625

This is my joint_pos_env_cfg.py:

@configclass
class KinovaArucoLiftEnvCfg(LiftEnvCfg):
    def __post_init__(self):
        
        # post init of parent
        super().__post_init__()

        # Set actions for the specific robot type (Kinova gen3n7) 
        # NOTE: This might conflict with cfg_kinova
        self.actions.arm_action = mdp.JointPositionActionCfg(
            asset_name="robot", joint_names=["joint_.*"], scale=0.2, use_default_offset=True
        )

        # Gripper action to control the Robotiq gripper (same path as actuator)
        self.actions.gripper_action = mdp.BinaryJointPositionActionCfg(
            asset_name="robot",
            joint_names=["finger_joint"],
            open_command_expr={"finger_joint": 0.785},
            close_command_expr={"finger_joint": 0.0}
        )

        # Set the body name for the end effector
        self.commands.object_pose.body_name = "end_effector_link"

        # Kinova robot cfg
        cfg_kinova = ArticulationCfg(
            spawn=sim_utils.UsdFileCfg(
                usd_path="/home/dgx_adi/Desktop/robotics_projects/sim_scenes/kinova_with_gripper.usd",
                rigid_props=sim_utils.RigidBodyPropertiesCfg(disable_gravity=False),
                articulation_props=sim_utils.ArticulationRootPropertiesCfg(
                    solver_position_iteration_count=8,
                    solver_velocity_iteration_count=2,
                    fix_root_link=True,  # Fix root to world (arm mounted on table)
                    enabled_self_collisions=True
                ),
            ),
            init_state=ArticulationCfg.InitialStateCfg(
                joint_pos={
                    # Home pose (degrees -> radians):
                    # joint_1: 0°, joint_2: 15°, joint_3: 180°, joint_4: -130°, joint_5: 0°, joint_6: 55°, joint_7: 90°
                    # Converted to radians below
                    "joint_1": 0.0,
                    "joint_2": 0.2617993877991494,   # 15°
                    "joint_3": 3.141592653589793,    # 180°
                    "joint_4": -2.2689280275926285,  # -130°
                    "joint_5": 0.0,
                    "joint_6": 0.9599310885968813,   # 55°
                    "joint_7": 3.141592653589793,   # 90°
                    "finger_joint": 0.0,
                },
                pos=(0.0, 0.0, 0.0),
            ),

            actuators={
                "shoulder": ImplicitActuatorCfg(
                    joint_names_expr=["joint_1", "joint_2"],
                    stiffness=5000.0,
                    damping=220.0,
                    friction=0.0,
                    armature=0.0,
                    velocity_limit_sim=0.5
                ),
                "elbow": ImplicitActuatorCfg(
                    joint_names_expr=["joint_3","joint_4", "joint_5", "joint_6", "joint_7"],
                    stiffness=2500.0,
                    damping=160.00,
                    friction=0.0,
                    armature=0.0,
                    velocity_limit_sim=0.5
                ),
                # Gripper actuators - using scoped names (now at same level as gen3n7_instanceable)
                "gripper": ImplicitActuatorCfg(
                    joint_names_expr=["finger_joint"],
                    stiffness=11000,
                    damping=300,
                    velocity_limit_sim=10.0
                ),
            }
        )

        # Use the Kinova + Robotiq gripper configuration
        self.scene.robot = cfg_kinova.replace(prim_path="{ENV_REGEX_NS}/Robot")

        # Set Cube as object
        self.scene.object = RigidObjectCfg(
            prim_path="{ENV_REGEX_NS}/Object",
            init_state=RigidObjectCfg.InitialStateCfg(pos=[0.5, 0, 0.055], rot=[1, 0, 0, 0]),
            spawn=UsdFileCfg(
                usd_path="/home/dgx_adi/IsaacLab/source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/kinova_pick_place_aruco/assets/DexCube/dex_cube_instanceable.usd",
                scale=(0.8, 0.8, 0.8),
                rigid_props=RigidBodyPropertiesCfg(
                    solver_position_iteration_count=16,
                    solver_velocity_iteration_count=3,
                    max_angular_velocity=1000.0,
                    max_linear_velocity=1000.0,
                    max_depenetration_velocity=2.0,
                    disable_gravity=False,
                ),
            ),
        )

        # Listens to the required transforms
        marker_cfg = FRAME_MARKER_CFG.copy()
        marker_cfg.markers["frame"].scale = (0.1, 0.1, 0.1)
        marker_cfg.prim_path = "/Visuals/FrameTransformer"
        self.scene.ee_frame = FrameTransformerCfg(
            prim_path="{ENV_REGEX_NS}/Robot/gen3n7_instanceable/base_link",
            debug_vis=False,
            visualizer_cfg=marker_cfg,
            target_frames=[
                FrameTransformerCfg.FrameCfg(
                    prim_path="{ENV_REGEX_NS}/Robot/gen3n7_instanceable/end_effector_link",
                    name="end_effector",
                    offset=OffsetCfg(
                        pos=[0.0, 0.0, 0.1034],
                    ),
                ),
            ],
        )

@configclass
class KinovaArucoLiftEnvCfg_PLAY(KinovaArucoLiftEnvCfg):
    def __post_init__(self):
        # post init of parent
        super().__post_init__()
        # make a smaller scene for play

        # Set events
        self.events = KinovaArucoLiftEnvCfg()

        self.scene.num_envs = 50
        self.scene.env_spacing = 2.5
        # disable randomization for play
        self.observations.policy.enable_corruption = False

Very Beginning of Training:
Screencast from 2025-12-16 22-09-06.webm

Early Training Around Iteration 120:
Screencast from 2025-12-16 22-14-53.webm

Mid-Training Around Iteration 600:
Screencast from 2025-12-16 22-07-02.webm

[RandomOakForest]

This behavior is almost always a reward / observation / curriculum issue: once the policy has partially learned to approach, the learning signal starts favoring “getting away” over actually closing and lifting, so PPO, for example, improves a policy that walks away from the cube instead of finishing the grasp.

The main issue seems to be that your reward and command setup is asking the robot to “move the cube to a desired end-effector pose above the table”, not to “grasp where the cube currently is”, and your regularization curriculum rapidly makes it cheaper to stay away than to explore a grasp once the policy can roughly approach. This combination very plausibly produces “move toward cube, then retract” behavior around the point where PPO stops exploring aggressively, for example.