some changes

Author: brianzheng206

deep_object_detection/README.md

@@ -19,11 +19,7 @@ deep_object_detection_node:
       nms_iou_threshold: 0.45
       max_detections: 300
       score_activation: "sigmoid"
-      enable_nms: true
       use_multi_output: false
-      output_boxes_idx: 0
-      output_scores_idx: 1
-      output_classes_idx: 2
 
       class_score_mode: "all_classes"
       class_score_start_idx: -1
@@ -41,20 +37,19 @@ deep_object_detection_node:
         score_idx: 4
         class_idx: 5
 
-    use_camera_sync: true
-    camera_sync_topic: "/multi_camera_sync/multi_image_compressed"
-    input_qos_reliability: "best_effort"
+    input_topic: "/multi_camera_sync/multi_image_compressed"
     output_detections_topic: "/detections"
 
-    min_batch_size: 1
     max_batch_size: 3
-    max_batch_latency_ms: 0
     queue_size: 10
-    queue_overflow_policy: "drop_oldest"
-    decode_failure_policy: "drop"
 
     preferred_provider: "tensorrt"
     device_id: 0
-    warmup_tensor_shapes: true
     enable_trt_engine_cache: true
     trt_engine_cache_path: "/tmp/deep_ros_ort_trt_cache"
+
+    Backend:
+      execution_provider: "tensorrt"
+      device_id: 0
+      trt_engine_cache_enable: true
+      trt_engine_cache_path: "/tmp/deep_ros_ort_trt_cache"

deep_object_detection/config/generic_model_params.yaml

@@ -14,13 +14,13 @@
 
 #pragma once
 
-#include <atomic>
 #include <deque>
 #include <memory>
 #include <mutex>
 #include <string>
 #include <vector>
 
+#include <deep_msgs/msg/multi_image.hpp>
 #include <opencv2/core/mat.hpp>
 #include <rclcpp/node_options.hpp>
 #include <rclcpp/rclcpp.hpp>
@@ -29,8 +29,6 @@
 #include <sensor_msgs/msg/compressed_image.hpp>
 #include <std_msgs/msg/header.hpp>
 
-#include <deep_msgs/msg/multi_image.hpp>
-
 #include "deep_object_detection/backend_manager.hpp"
 #include "deep_object_detection/detection_types.hpp"
 #include "deep_object_detection/generic_postprocessor.hpp"
@@ -42,9 +40,9 @@ namespace deep_object_detection
 /**
  * @brief ROS2 lifecycle node for object detection using ONNX models
  *
- * This node performs object detection on images from cameras or synchronized multi-camera streams.
+ * This node performs object detection on synchronized multi-camera streams via MultiImage messages.
  * It supports:
- * - Multiple input modes: individual camera topics or synchronized MultiImage messages
+ * - MultiImage input: synchronized compressed images from multiple cameras
  * - Batch processing: groups images for efficient inference
  * - Multiple backends: CPU, CUDA, or TensorRT execution providers
  * - Configurable preprocessing: resizing, normalization, color format conversion
@@ -114,143 +112,40 @@ class DeepObjectDetectionNode : public rclcpp_lifecycle::LifecycleNode
     const rclcpp_lifecycle::State &) override;
 
 private:
-  /**
-   * @brief Declare and read all ROS2 parameters
-   *
-   * Reads model configuration, preprocessing/postprocessing parameters, camera topics,
-   * batch settings, and backend provider settings from ROS2 parameters.
-   */
   void declareAndReadParameters();
-
-  /**
-   * @brief Setup subscriptions to individual camera compressed image topics
-   *
-   * Creates one subscription per camera topic in params_.camera_topics.
-   * Each subscription calls handleCompressedImage() with the camera index.
-   */
-  void setupMultiCameraSubscriptions();
-
-  /**
-   * @brief Setup subscription to synchronized MultiImage topic
-   *
-   * Creates a single subscription to camera_sync_topic_ that receives MultiImage
-   * messages containing synchronized compressed images from multiple cameras.
-   */
-  void setupCameraSyncSubscription();
-
-  /**
-   * @brief Handle incoming MultiImage message with synchronized images
-   * @param msg MultiImage message containing multiple compressed images
-   *
-   * Extracts each compressed image from the MultiImage and processes them
-   * through handleCompressedImage() with sequential camera IDs.
-   */
+  void setupSubscription();
   void onMultiImage(const deep_msgs::msg::MultiImage::ConstSharedPtr & msg);
-
-  /**
-   * @brief Handle incoming compressed image from a camera
-   * @param msg Compressed image message
-   * @param camera_id Camera identifier (index for multi-camera, or from MultiImage)
-   *
-   * Decodes the compressed image, enqueues it for batch processing.
-   * Handles decode failures according to decode_failure_policy.
-   */
-  void handleCompressedImage(const sensor_msgs::msg::CompressedImage & msg, int camera_id);
-
-  /**
-   * @brief Add image to processing queue
-   * @param image Decoded BGR image (OpenCV Mat)
-   * @param header ROS message header with timestamp and frame_id
-   *
-   * Thread-safe enqueueing. Applies queue_overflow_policy if queue is full.
-   * Tracks first image timestamp for batch timeout calculation.
-   */
+  void handleCompressedImage(const sensor_msgs::msg::CompressedImage & msg);
   void enqueueImage(cv::Mat image, const std_msgs::msg::Header & header);
-
-  /**
-   * @brief Format tensor shape vector as string for logging
-   * @param shape Vector of dimension sizes
-   * @return Comma-separated string representation (e.g., "1, 3, 640, 640")
-   */
-  std::string formatShape(const std::vector<size_t> & shape) const;
-
-  /**
-   * @brief Timer callback for batch processing
-   *
-   * Called periodically (every 5ms) to check if batch should be processed.
-   * Processes batch if:
-   * - Queue size >= min_batch_size, OR
-   * - max_batch_latency_ms exceeded and queue not empty
-   * Extracts up to max_batch_size images and calls processBatch().
-   */
   void onBatchTimer();
-
-  /**
-   * @brief Process a batch of images through inference pipeline
-   * @param batch Vector of queued images to process
-   *
-   * For each image: preprocess -> inference -> postprocess -> publish detections.
-   * Handles multi-output models if configured. Publishes Detection2DArray messages.
-   */
   void processBatch(const std::vector<QueuedImage> & batch);
-
-  /**
-   * @brief Publish detection results for a batch
-   * @param batch_detections Detections for each image in batch
-   * @param headers Message headers for each image (for frame_id and timestamp)
-   * @param metas Image metadata for coordinate transformation
-   *
-   * Creates and publishes Detection2DArray message for each image with its detections.
-   */
   void publishDetections(
     const std::vector<std::vector<SimpleDetection>> & batch_detections,
     const std::vector<std_msgs::msg::Header> & headers,
     const std::vector<ImageMeta> & metas);
-
-  /**
-   * @brief Load class names from file
-   *
-   * Reads class names from params_.model_metadata.class_names_file (one per line).
-   * Stores in params_.class_names for use in postprocessing and message publishing.
-   */
   void loadClassNames();
-
-  /**
-   * @brief Stop all subscriptions and cancel batch timer
-   *
-   * Clears all camera subscriptions, resets MultiImage subscription,
-   * cancels batch timer, and clears image queue. Used in deactivate/cleanup/shutdown.
-   */
+  void cleanupPartialConfiguration();
+  void cleanupAllResources();
   void stopSubscriptionsAndTimer();
 
-  DetectionParams params_;  ///< All node configuration parameters
+  DetectionParams params_;
+
+  rclcpp::Subscription<deep_msgs::msg::MultiImage>::SharedPtr multi_image_sub_;
+  std::string input_topic_;
+  rclcpp_lifecycle::LifecyclePublisher<Detection2DArrayMsg>::SharedPtr detection_pub_;
+  rclcpp::TimerBase::SharedPtr batch_timer_;
 
-  std::vector<rclcpp::Subscription<sensor_msgs::msg::CompressedImage>::SharedPtr>
-    multi_camera_subscriptions_;  ///< Subscriptions for individual camera topics
-  rclcpp::Subscription<deep_msgs::msg::MultiImage>::SharedPtr
-    multi_image_sub_;  ///< Subscription for synchronized MultiImage messages
-  bool use_camera_sync_{false};  ///< Whether to use MultiImage sync mode or individual topics
-  std::string camera_sync_topic_;  ///< Topic name for MultiImage messages
-  rclcpp::Publisher<Detection2DArrayMsg>::SharedPtr detection_pub_;  ///< Publisher for detection results
-  rclcpp::TimerBase::SharedPtr batch_timer_;  ///< Timer for periodic batch processing checks
+  std::deque<QueuedImage> image_queue_;
+  std::mutex queue_mutex_;
+  rclcpp::CallbackGroup::SharedPtr callback_group_;
 
-  std::deque<QueuedImage> image_queue_;  ///< Queue of images waiting for batch processing
-  std::mutex queue_mutex_;  ///< Mutex protecting image_queue_ and first_image_timestamp_
-  std::atomic<bool> processing_{false};  ///< Flag to prevent concurrent batch processing
-  rclcpp::Time first_image_timestamp_;  ///< Timestamp of oldest image in queue (for batch timeout)
+  size_t dropped_images_count_;
 
-  std::unique_ptr<ImagePreprocessor> preprocessor_;  ///< Image preprocessing (resize, normalize, etc.)
-  std::unique_ptr<GenericPostprocessor> postprocessor_;  ///< Detection postprocessing (NMS, decode, etc.)
-  std::unique_ptr<BackendManager> backend_manager_;  ///< Backend plugin manager (CPU/CUDA/TensorRT)
+  std::unique_ptr<ImagePreprocessor> preprocessor_;
+  std::unique_ptr<GenericPostprocessor> postprocessor_;
+  std::unique_ptr<BackendManager> backend_manager_;
 };
 
-/**
- * @brief Factory function to create DeepObjectDetectionNode instance
- * @param options Node options for ROS2 configuration
- * @return Shared pointer to lifecycle node
- *
- * Used by rclcpp_components for component loading.
- */
 std::shared_ptr<rclcpp_lifecycle::LifecycleNode> createDeepObjectDetectionNode(
   const rclcpp::NodeOptions & options = rclcpp::NodeOptions());
 

deep_object_detection/include/deep_object_detection/deep_object_detection_node.hpp

@@ -44,8 +44,7 @@ using Detection2DArrayMsg = vision_msgs::msg::Detection2DArray;
 namespace deep_object_detection
 {
 
-// Image processing constants
-constexpr size_t RGB_CHANNELS = 3;  // Number of channels in RGB/BGR images
+constexpr size_t RGB_CHANNELS = 3;
 
 enum class Provider
 {
@@ -126,8 +125,8 @@ struct ImageMeta
 
 struct QueuedImage
 {
-  cv::Mat bgr;
-  std_msgs::msg::Header header;
+  cv::Mat bgr;  ///< Decoded BGR image (OpenCV Mat)
+  std_msgs::msg::Header header;  ///< ROS message header with timestamp and frame_id
 };
 
 struct PackedInput
@@ -191,12 +190,9 @@ struct DetectionParams
   ModelMetadata model_metadata;
   PreprocessingConfig preprocessing;
   PostprocessingConfig postprocessing;
-  std::vector<std::string> camera_topics;
   std::string input_qos_reliability{"best_effort"};
   std::string output_detections_topic{"/detections"};
-  int min_batch_size{1};
   int max_batch_size{3};
-  int max_batch_latency_ms{0};  // 0 means no timeout (wait for min_batch_size)
   int queue_size{10};
   QueueOverflowPolicy queue_overflow_policy{QueueOverflowPolicy::DROP_OLDEST};
   DecodeFailurePolicy decode_failure_policy{DecodeFailurePolicy::DROP};

deep_object_detection/include/deep_object_detection/detection_types.hpp

@@ -54,6 +54,17 @@ class GenericPostprocessor
 
   static OutputLayout detectLayout(const std::vector<size_t> & output_shape);
 
+  /**
+   * @brief Auto-configure output layout based on config and optional output shape
+   * @param output_shape Model output shape (can be empty for deferred detection)
+   * @param layout_config Layout configuration from parameters
+   * @return Configured OutputLayout
+   *
+   * Handles both manual and auto-detection modes. If auto_detect is true and output_shape
+   * is available, automatically detects layout. Otherwise uses manual config or defers detection.
+   */
+  static OutputLayout autoConfigure(const std::vector<size_t> & output_shape, const OutputLayoutConfig & layout_config);
+
   std::vector<std::vector<SimpleDetection>> decode(
     const deep_ros::Tensor & output, const std::vector<ImageMeta> & metas) const;
 

deep_object_detection/include/deep_object_detection/generic_postprocessor.hpp

@@ -93,7 +93,6 @@ Provider BackendManager::parseProvider(const std::string & provider_str) const
 
 void BackendManager::initializeBackend()
 {
-  // Check CUDA availability if needed
   if ((provider_ == Provider::TENSORRT || provider_ == Provider::CUDA) && !isCudaRuntimeAvailable()) {
     std::string error = "Provider " + providerToString(provider_) +
                         " requires CUDA runtime libraries (libcudart/libcuda) which are not available";
@@ -105,11 +104,32 @@ void BackendManager::initializeBackend()
     throw std::runtime_error("No plugin name for provider: " + providerToString(provider_));
   }
 
-  // Update Backend.execution_provider parameter to match the actual provider
   const auto provider_name = providerToString(provider_);
-  node_.set_parameters({rclcpp::Parameter("Backend.execution_provider", provider_name)});
 
-  // Pass the main node directly to the plugin (plugin will read Backend.* parameters from it)
+  if (!node_.has_parameter("Backend.execution_provider")) {
+    node_.declare_parameter<std::string>("Backend.execution_provider", provider_name);
+  } else {
+    node_.set_parameters({rclcpp::Parameter("Backend.execution_provider", provider_name)});
+  }
+
+  if (!node_.has_parameter("Backend.device_id")) {
+    node_.declare_parameter<int>("Backend.device_id", params_.device_id);
+  } else {
+    node_.set_parameters({rclcpp::Parameter("Backend.device_id", params_.device_id)});
+  }
+
+  if (!node_.has_parameter("Backend.trt_engine_cache_enable")) {
+    node_.declare_parameter<bool>("Backend.trt_engine_cache_enable", params_.enable_trt_engine_cache);
+  } else {
+    node_.set_parameters({rclcpp::Parameter("Backend.trt_engine_cache_enable", params_.enable_trt_engine_cache)});
+  }
+
+  if (!node_.has_parameter("Backend.trt_engine_cache_path")) {
+    node_.declare_parameter<std::string>("Backend.trt_engine_cache_path", params_.trt_engine_cache_path);
+  } else {
+    node_.set_parameters({rclcpp::Parameter("Backend.trt_engine_cache_path", params_.trt_engine_cache_path)});
+  }
+
   auto node_ptr = node_.shared_from_this();
   plugin_holder_ = plugin_loader_->createUniqueInstance(plugin_name);
   plugin_holder_->initialize(node_ptr);
@@ -195,7 +215,6 @@ std::string BackendManager::providerToString(Provider provider) const
   }
 }
 
-
 void BackendManager::declareActiveProviderParameter(const std::string & value)
 {
   rcl_interfaces::msg::ParameterDescriptor desc;