[Question] Using Issac ROS repo ros2 topics in IssacLab

[shiven2001]

Question

Hello, I am trying to use Issac ROS modules in the RL training env in IssacLab for the Nova Carter robot.

I am using the isaac_ros_detectnet:

https://nvidia-isaac-ros.github.io/repositories_and_packages/isaac_ros_object_detection/isaac_ros_detectnet/index.html#quickstart

I am confused how it is supposed to be integrated when running multiple environments as when I use 4 envs for training and go to ros2 topic list, there is only one topic there instead of 4 topics. I am guessing it is not meant for muitple envs but if I was suppose to do that, how can I achieve it?

Basically, I am trying to use the action graphs provided with the nova carter robot and use the isaac_ros_detectnet to get the inference and use it as an observation for training.

This is my cfg file:

`

Scene cfg

@configclass
class WheeledSceneCfg(InteractiveSceneCfg):
"""Configuration for a Wheeled robot scene."""

# dome light
dome_light = AssetBaseCfg(
    prim_path="/World/DomeLight",
    spawn=sim_utils.DomeLightCfg(color=(0.9, 0.9, 0.9), intensity=500.0),
) 

# distant lights
distant_light = AssetBaseCfg(
    prim_path="/World/DistantLight",
    spawn=sim_utils.DistantLightCfg(color=(0.9, 0.9, 0.9), intensity=2500.0),
    init_state=AssetBaseCfg.InitialStateCfg(rot=(0.738, 0.477, 0.477, 0.0)),
)

# ground plane
ground = AssetBaseCfg(
    prim_path="/World/ground",
    spawn=sim_utils.UsdFileCfg(usd_path=ground_usd_path),
)
person = AssetBaseCfg(
    prim_path="{ENV_REGEX_NS}/Person",
    spawn=sim_utils.UsdFileCfg(
        usd_path=person_usd_path,
        rigid_props=sim_utils.RigidBodyPropertiesCfg(),  # Add physics
        collision_props=sim_utils.CollisionPropertiesCfg()  # Add collision
    ),
    init_state=AssetBaseCfg.InitialStateCfg(
        pos=(5, 0, 0),
        rot=(0.7071, 0, 0, -0.7071)  # -90° Z rotation
    )
)
    
# quadruped robot
robot: ArticulationCfg = NOVA_CFG.replace(prim_path="{ENV_REGEX_NS}/Robot")

def __post_init__(self):
    """Ensure no global /World/Person prim exists."""
    from omni.isaac.core.utils.prims import delete_prim
    try:
        delete_prim("/World/Person")
    except:
        pass  # Ignore if prim doesn't exist

import dependencies for ros2

import rclpy
from rclpy.node import Node
from geometry_msgs.msg import Twist
import random
import sys
sys.path.append(os.path.join(os.environ["HOME"], "niris-isaaclab/IsaacLab/source/isaaclab_tasks/isaaclab_tasks/manager_based/classic/nova_carter_conf/"))
import mdp

Actions cfg

@configclass
class ActionsCfg:
"""Action specifications for the environment."""
joint_velocities = mdp.JointVelocityActionCfg(asset_name="robot", joint_names=["joint_wheel_left", "joint_wheel_right"], scale=5.0)
joint_positions = mdp.JointPositionActionCfg(asset_name="robot", joint_names=["joint_swing_left", "joint_swing_right"], scale=5.0)

Observations cfg

@configclass
class ObservationsCfg:
"""Observation specifications for the MDP."""

@configclass
class PolicyCfg(ObsGroup):
    """Observations for policy group."""

    # observation terms (order preserved)
    actions = ObsTerm(func=mdp.last_action)
    obj_detectnet = ObsTerm(func=mdp.get_confidence)
  
    def __post_init__(self) -> None:
        self.enable_corruption = False
        self.concatenate_terms = False

# observation groups
policy: PolicyCfg = PolicyCfg()

Event cfg

@configclass
class EventCfg:
"""Configuration for events with fixed orientation."""
reset_scene = EventTerm(
func=mdp.reset_root_state_uniform,
params={
"pose_range": {
"x": (-2.0, 2.0), # Random x position
"y": (-2.0, 2.0), # Random y position
"z": (0.05, 0.05), # Fixed z at 0.05
"roll": (0.0, 0.0), # No rotation
"pitch": (0.0, 0.0), # No rotation
"yaw": (0.0, 0.0) # No rotation
},
"velocity_range":{
"x": (0.0, 0.0), # No linear velocity
"y": (0.0, 0.0),
"z": (0.0, 0.0),
"roll": (0.0, 0.0), # No angular velocity
"pitch": (0.0, 0.0),
"yaw": (0.0, 0.0)
},
"asset_cfg":SceneEntityCfg(name="robot")
},
mode="reset")
# reset_scene = EventTerm(func=mdp.reset_scene_to_default, mode="reset")

@configclass
class RewardsCfg:
"""Reward terms for the MDP."""
# (1) Constant running reward
alive = RewTerm(func=mdp.is_alive, weight=1.0)

Terminations cfg

@configclass
class TerminationsCfg:
"""Termination terms for the MDP."""

# (1) Time out
time_out = DoneTerm(func=mdp.time_out, time_out=True)

@configclass
class CommandsCfg:
"""Command terms for the MDP."""

# no commands for this MDP
null = mdp.NullCommandCfg()

@configclass
class CurriculumCfg:
"""Configuration for the curriculum."""
pass

@configclass
class WheeledEnvCfg(ManagerBasedRLEnvCfg):
"""Configuration for the wheeled environment."""
# Scene settings
scene: WheeledSceneCfg = WheeledSceneCfg(num_envs=4096, env_spacing=40.0)
# Basic settings
observations: ObservationsCfg = ObservationsCfg()
actions: ActionsCfg = ActionsCfg()
events: EventCfg = EventCfg()
# MDP settings
curriculum: CurriculumCfg = CurriculumCfg()
rewards: RewardsCfg = RewardsCfg()
terminations: TerminationsCfg = TerminationsCfg()
# No command generator
commands: CommandsCfg = CommandsCfg()

# Post initialization
def __post_init__(self) -> None:
    """Post initialization."""
    self.sim.device = "cpu"
    # general settings
    self.decimation = 2
    # simulation settings
    self.sim.dt = 0.01  # simulation timestep
    self.episode_length_s = 10
    # viewer settings
    self.viewer.eye = (8.0, 0.0, 5.0)
    # simulation settings
    self.sim.render_interval = self.decimation

`
and this is my custom nova_carter_ros2.py file in mdp folder:

`
class DetectNetDetections:
def init(self, node, device='cuda:0'):
self.node = node
self.detections_data = []
self.device = device
self.lock = Lock() # Add lock for thread safety
self.class_id_mapping = {"person": 0, "car": 1, "bicycle": 2}
self.detections_sub = self.node.create_subscription(
Detection2DArray, "/detectnet/detections", self._detections_callback, 10
)

def _detections_callback(self, msg):
    with self.lock:
        logger.info(f"ROS2 Received detection message: {msg.detections}")
        self.detections_data = []
        if msg.detections:
            for detection in msg.detections:
                if detection.results:
                    class_id = detection.results[0].hypothesis.class_id
                    class_id_num = self.class_id_mapping.get(class_id, -1)
                    logger.info(f"Class ID: {class_id}, Mapped: {class_id_num}")
                    detection_info = {
                        "class_id": class_id_num,
                        "score": detection.results[0].hypothesis.score,
                        "bbox_center": [detection.bbox.center.position.x, detection.bbox.center.position.y],
                        "bbox_size": [detection.bbox.size_x, detection.bbox.size_y],
                    }
                    self.detections_data.append(detection_info)
        logger.info(f"Detections Data: {self.detections_data}")

def get_confidence(env: ManagerBasedEnv, asset_cfg: SceneEntityCfg = SceneEntityCfg("robot")) -> torch.Tensor:
asset: Articulation = env.scene[asset_cfg.name]
num_envs = asset.data.root_state_w.shape[0]

if not hasattr(env, "detectnet_detections_instance"):
    logger.info("Confidence Data: No detection instance available")
    return torch.zeros((num_envs, 1), device=env.device)

with env.detectnet_detections_instance.lock:
    if not env.detectnet_detections_instance.detections_data:
        logger.info("Confidence Data: No detection data available yet")
        return torch.zeros((num_envs, 1), device=env.device)

    confidences = [float(detection["score"]) for detection in env.detectnet_detections_instance.detections_data]
    max_confidence = max(confidences)
    logger.info(f"Confidence Data: Max confidence = {max_confidence}")

    confidence_tensor = torch.full((num_envs, 1), max_confidence, device=env.device)
    logger.info(f"FROM NOVA CARTER ROS2 FILE: Confidence Tensor: {confidence_tensor}")
    return confidence_tensor

as you can see, I create env.detectnet_detections_instance when i run my custom training file:
# create isaac environment
env = gym.make(args_cli.task, cfg=env_cfg, render_mode="rgb_array" if args_cli.video else None)

# ROS2 setup
detectnet_node = Node("detectnet_node")
detectnet_detections = mdp.DetectNetDetections(detectnet_node)  # No device param yet

# Store the unwrapped environment
unwrapped_env = env.unwrapped

# convert to single-agent instance if required by the RL algorithm
if isinstance(env.unwrapped, DirectMARLEnv):
    env = multi_agent_to_single_agent(env)

# wrap for video recording
if args_cli.video:
    video_kwargs = {
        "video_folder": os.path.join(log_dir, "videos", "train"),
        "step_trigger": lambda step: step % args_cli.video_interval == 0,
        "video_length": args_cli.video_length,
        "disable_logger": True,
    }
    print("[INFO] Recording videos during training.")
    print_dict(video_kwargs, nesting=4)
    env = gym.wrappers.RecordVideo(env, **video_kwargs)

# wrap around environment for stable baselines
env = Sb3VecEnvWrapper(env)

if "normalize_input" in agent_cfg:
    env = VecNormalize(
        env,
        training=True,
        norm_obs="normalize_input" in agent_cfg and agent_cfg.pop("normalize_input"),
        norm_reward="normalize_value" in agent_cfg and agent_cfg.pop("normalize_value"),
        clip_obs="clip_obs" in agent_cfg and agent_cfg.pop("clip_obs"),
        gamma=agent_cfg["gamma"],
        clip_reward=np.inf,
    )

unwrapped_env.detectnet_detections_instance = detectnet_detections
unwrapped_env.prev_confidence = None
logger.info("Attached DetectNetDetections to env.detectnet_detections_instance")

def spin_node():
    try:
        logger.info("Starting ROS2 spin for detectnet_node")
        rclpy.spin(detectnet_node)
    except Exception as e:
        logger.error(f"ROS2 spin failed: {e}")

spin_thread = threading.Thread(target=spin_node, daemon=True)
spin_thread.start()

`
But I think it is just giving me the last spawned or first spawned envs inference data and not each envs individual data. How to fix that?

[RandomOakForest]

Thank you for posting this. It is not clear what exactly is what you are trying to train. It seems detectnet is just an intermediate process that would give you a form of ground-truth. Could you elaborate and clarify what is your training task? I will move this post to our discussions for the team to follow up.

[shiven2001]

Hi, let me elaborate. I want to use an object detection algo like DetectNet to get inference. I want to feed that inference data as an observation in my IssacLab environment. My training task goal is that the robot should move around to increase the confidence of the detected object or detect more objects in the environment.

So far, by code is working fine for a single environment. However, it fails in multiple envs.

What I did was I made a custom function in class Observation Cfg:
obj_detectnet = ObsTerm(func=mdp.get_confidence)

To make this work, I created a class called DetectNetDetections which subscribes to the /detectnet/detections topic and stores the detection data in self.detections_data:

  class DetectNetDetections:
      def __init__(self, node, device='cuda:0'):
          self.node = node
          self.detections_data = []
          self.device = device
          self.lock = Lock()  # Add lock for thread safety
          self.class_id_mapping = {"person": 0, "car": 1, "bicycle": 2}
          self.detections_sub = self.node.create_subscription(
              Detection2DArray, "/detectnet/detections", self._detections_callback, 10
          )

      def _detections_callback(self, msg):
          with self.lock:
              logger.info(f"ROS2 Received detection message: {msg.detections}")
              self.detections_data = []
              if msg.detections:
                  for detection in msg.detections:
                      if detection.results:
                          class_id = detection.results[0].hypothesis.class_id
                          class_id_num = self.class_id_mapping.get(class_id, -1)
                          logger.info(f"Class ID: {class_id}, Mapped: {class_id_num}")
                          detection_info = {
                              "class_id": class_id_num,
                              "score": detection.results[0].hypothesis.score,
                              "bbox_center": [detection.bbox.center.position.x, detection.bbox.center.position.y],
                              "bbox_size": [detection.bbox.size_x, detection.bbox.size_y],
                          }
                          self.detections_data.append(detection_info)
              logger.info(f"Detections Data: {self.detections_data}")

Then I customized the train.py file for sb3 adding the ROS2 initialization and creating a unwrapped_env.detectnet_detections_instance which contains the data from detections_data array:

# create isaac environment
env = gym.make(args_cli.task, cfg=env_cfg, render_mode="rgb_array" if args_cli.video else None)

# ROS2 setup
detectnet_node = Node("detectnet_node")
detectnet_detections = mdp.DetectNetDetections(detectnet_node)  # No device param yet

# Store the unwrapped environment
unwrapped_env = env.unwrapped

# convert to single-agent instance if required by the RL algorithm
if isinstance(env.unwrapped, DirectMARLEnv):
    env = multi_agent_to_single_agent(env)

# wrap around environment for stable baselines
env = Sb3VecEnvWrapper(env)

unwrapped_env.detectnet_detections_instance = detectnet_detections
unwrapped_env.prev_confidence = None
logger.info("Attached DetectNetDetections to env.detectnet_detections_instance")

def spin_node():
    try:
        logger.info("Starting ROS2 spin for detectnet_node")
        rclpy.spin(detectnet_node)
    except Exception as e:
        logger.error(f"ROS2 spin failed: {e}")

spin_thread = threading.Thread(target=spin_node, daemon=True)
spin_thread.start()

# Give ROS2 time to start
from time import sleep
sleep(1)  # Wait for initial messages
logger.info(f"Spin thread alive: {spin_thread.is_alive()}")

Then I use my custom observation function to get the detectnet_data from detectnet_detections_instance (mainly the confidence scores) which is now being stored in unwrapped_env:

  def get_confidence(env: ManagerBasedEnv, asset_cfg: SceneEntityCfg = SceneEntityCfg("robot")) -> torch.Tensor:
      asset: Articulation = env.scene[asset_cfg.name]
      num_envs = asset.data.root_state_w.shape[0]
  
      if not hasattr(env, "detectnet_detections_instance"):
          logger.info("Confidence Data: No detection instance available")
          return torch.zeros((num_envs, 1), device=env.device)
  
      with env.detectnet_detections_instance.lock:
          if not env.detectnet_detections_instance.detections_data:
              logger.info("Confidence Data: No detection data available yet")
              return torch.zeros((num_envs, 1), device=env.device)
  
          confidences = [float(detection["score"]) for detection in env.detectnet_detections_instance.detections_data]
          max_confidence = max(confidences)
          logger.info(f"Confidence Data: Max confidence = {max_confidence}")
  
          confidence_tensor = torch.full((num_envs, 1), max_confidence, device=env.device)
          logger.info(f"FROM NOVA CARTER ROS2 FILE: Confidence Tensor: {confidence_tensor}")
          return confidence_tensor

My issue is that this works ok for a single environment. I use
args_cli.experience = "isaacsim.exp.full.kit" # Set the experience here

to use the full Issac Sim instead of the bare one so that I can use the action graphs that were premade for the Issac Nova Carter ROS usd file. Hence, when I launch this in IssacLab, the topics like /front_stereo_camera/left/image_rect_color are being published. But when I run 2 envs in IssacLab as such, there are two Nova Carter robots so each should publish their own topics right? But in ros2 topic list, there is only 1 /front_stereo_camera/left/image_rect_color. And when I use the /detectnet/detections using the commands mentioned in Issac ROS (https://nvidia-isaac-ros.github.io/repositories_and_packages/isaac_ros_object_detection/isaac_ros_detectnet/index.html#quickstart), it also publishes 1 /detectnet/detections topic instead of 2 for each individual robot (even though each robot is seeing different things).

So how do I fix this?