loading backend at runtime

Author: brianzheng206

deep_yolo_inference/CMakeLists.txt

@@ -34,10 +34,7 @@ find_package(message_filters REQUIRED)
 find_package(vision_msgs REQUIRED)
 find_package(deep_core REQUIRED)
 find_package(deep_msgs REQUIRED)
-find_package(deep_ort_backend_plugin REQUIRED)
-find_package(deep_ort_gpu_backend_plugin REQUIRED)
-find_package(onnxruntime_vendor REQUIRED)
-find_package(onnxruntime_gpu_vendor REQUIRED)
+find_package(pluginlib REQUIRED)
 find_package(rcl_interfaces REQUIRED)
 find_package(yaml-cpp REQUIRED)
 find_package(OpenCV REQUIRED COMPONENTS core imgproc)
@@ -68,19 +65,14 @@ ament_target_dependencies(yolo_inference_lib
   vision_msgs
   deep_core
   deep_msgs
-  deep_ort_backend_plugin
-  deep_ort_gpu_backend_plugin
-  onnxruntime_vendor
-  onnxruntime_gpu_vendor
+  pluginlib
   rcl_interfaces
 )
 
 target_link_libraries(yolo_inference_lib
+  pluginlib::pluginlib
+  rclcpp::rclcpp
   deep_core::deep_core_lib
-  deep_ort_backend_plugin::deep_ort_backend_plugin_lib
-  deep_ort_gpu_backend_plugin::deep_ort_gpu_backend_plugin_lib
-  onnxruntime_vendor::onnxruntime_lib
-  onnxruntime_gpu_vendor::onnxruntime_gpu_lib
   ${OpenCV_LIBRARIES}
   yaml-cpp
 )
@@ -123,8 +115,6 @@ if(BUILD_TESTING)
   target_link_libraries(test_parameters
       yolo_inference_lib
       deep_core::deep_core_lib
-      deep_ort_backend_plugin::deep_ort_backend_plugin_lib
-      deep_ort_gpu_backend_plugin::deep_ort_gpu_backend_plugin_lib
       ${OpenCV_LIBRARIES}
     )
     target_include_directories(test_parameters PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)
@@ -138,10 +128,7 @@ if(BUILD_TESTING)
       vision_msgs
       deep_core
       deep_msgs
-      deep_ort_backend_plugin
-      deep_ort_gpu_backend_plugin
-      onnxruntime_vendor
-      onnxruntime_gpu_vendor
+      pluginlib
       rcl_interfaces
     )
   else()
@@ -153,8 +140,6 @@ if(BUILD_TESTING)
     target_link_libraries(test_fallback_selection
       yolo_inference_lib
       deep_core::deep_core_lib
-      deep_ort_backend_plugin::deep_ort_backend_plugin_lib
-      deep_ort_gpu_backend_plugin::deep_ort_gpu_backend_plugin_lib
       ${OpenCV_LIBRARIES}
       yaml-cpp
     )
@@ -169,10 +154,7 @@ if(BUILD_TESTING)
       vision_msgs
       deep_core
       deep_msgs
-      deep_ort_backend_plugin
-      deep_ort_gpu_backend_plugin
-      onnxruntime_vendor
-      onnxruntime_gpu_vendor
+      pluginlib
       rcl_interfaces
     )
   else()

deep_yolo_inference/include/deep_yolo_inference/backend_manager.hpp

@@ -18,7 +18,9 @@
 #include <string>
 #include <vector>
 
+#include <deep_core/plugin_interfaces/deep_backend_plugin.hpp>
 #include <deep_core/types/tensor.hpp>
+#include <pluginlib/class_loader.hpp>
 #include <rclcpp/rclcpp.hpp>
 #include <rclcpp_lifecycle/lifecycle_node.hpp>
 
@@ -28,7 +30,6 @@ namespace deep_ros
 {
 class BackendInferenceExecutor;
 class BackendMemoryAllocator;
-class DeepBackendPlugin;
 }  // namespace deep_ros
 
 namespace deep_yolo_inference
@@ -38,6 +39,7 @@ class BackendManager
 {
 public:
   BackendManager(rclcpp_lifecycle::LifecycleNode & node, const YoloParams & params);
+  ~BackendManager();
 
   void buildProviderOrder();
   bool initialize(size_t start_index = 0);
@@ -76,7 +78,12 @@ class BackendManager
   std::string active_provider_{"unknown"};
   std::shared_ptr<deep_ros::BackendInferenceExecutor> executor_;
   std::shared_ptr<deep_ros::BackendMemoryAllocator> allocator_;
-  std::shared_ptr<deep_ros::DeepBackendPlugin> plugin_holder_;
+  // plugin_loader_ must be declared before plugin_holder_ so it's destroyed after
+  // (members are destroyed in reverse declaration order)
+  std::unique_ptr<pluginlib::ClassLoader<deep_ros::DeepBackendPlugin>> plugin_loader_;
+  pluginlib::UniquePtr<deep_ros::DeepBackendPlugin> plugin_holder_;
+  
+  std::string providerToPluginName(Provider provider) const;
 };
 
 }  // namespace deep_yolo_inference

deep_yolo_inference/src/backend_manager.cpp

@@ -23,8 +23,7 @@
 #include <utility>
 
 #include <deep_core/plugin_interfaces/deep_backend_plugin.hpp>
-#include <deep_ort_backend_plugin/ort_backend_plugin.hpp>
-#include <deep_ort_gpu_backend_plugin/ort_gpu_backend_plugin.hpp>
+#include <pluginlib/class_loader.hpp>
 #include <rcl_interfaces/msg/parameter_descriptor.hpp>
 
 namespace deep_yolo_inference
@@ -33,8 +32,18 @@ namespace deep_yolo_inference
 BackendManager::BackendManager(rclcpp_lifecycle::LifecycleNode & node, const YoloParams & params)
 : node_(node)
 , params_(params)
+, plugin_loader_(std::make_unique<pluginlib::ClassLoader<deep_ros::DeepBackendPlugin>>("deep_core", "deep_ros::DeepBackendPlugin"))
 {}
 
+BackendManager::~BackendManager()
+{
+  // Reset plugin holder before plugin loader is destroyed
+  // This ensures proper cleanup order to avoid class_loader warnings
+  plugin_holder_.reset();
+  executor_.reset();
+  allocator_.reset();
+}
+
 void BackendManager::buildProviderOrder()
 {
   auto normalize_pref = params_.preferred_provider;
@@ -121,34 +130,32 @@ bool BackendManager::initializeBackend(size_t start_index)
     }
 
     try {
-      switch (provider) {
-        case Provider::TENSORRT:
-        case Provider::CUDA: {
-          const auto provider_name = providerToString(provider);
-          auto overrides = std::vector<rclcpp::Parameter>{
-            rclcpp::Parameter("Backend.device_id", params_.device_id),
-            rclcpp::Parameter("Backend.execution_provider", provider_name),
-            rclcpp::Parameter("Backend.trt_engine_cache_enable", params_.enable_trt_engine_cache),
-            rclcpp::Parameter("Backend.trt_engine_cache_path", params_.trt_engine_cache_path)};
-          auto backend_node = createBackendConfigNode(provider_name, std::move(overrides));
-          auto plugin = std::make_shared<deep_ort_gpu_backend::OrtGpuBackendPlugin>();
-          plugin->initialize(backend_node);
-          allocator_ = plugin->get_allocator();
-          executor_ = plugin->get_inference_executor();
-          plugin_holder_ = plugin;
-          break;
-        }
-        case Provider::CPU: {
-          auto backend_node = createBackendConfigNode("cpu");
-          auto plugin = std::make_shared<deep_ort_backend::OrtBackendPlugin>();
-          plugin->initialize(backend_node);
-          allocator_ = plugin->get_allocator();
-          executor_ = plugin->get_inference_executor();
-          plugin_holder_ = plugin;
-          break;
-        }
+      const std::string plugin_name = providerToPluginName(provider);
+      if (plugin_name.empty()) {
+        RCLCPP_WARN(node_.get_logger(), "No plugin name for provider: %s", providerToString(provider).c_str());
+        continue;
+      }
+
+      // Create backend config node with provider-specific parameters
+      rclcpp_lifecycle::LifecycleNode::SharedPtr backend_node;
+      if (provider == Provider::TENSORRT || provider == Provider::CUDA) {
+        const auto provider_name = providerToString(provider);
+        auto overrides = std::vector<rclcpp::Parameter>{
+          rclcpp::Parameter("Backend.device_id", params_.device_id),
+          rclcpp::Parameter("Backend.execution_provider", provider_name),
+          rclcpp::Parameter("Backend.trt_engine_cache_enable", params_.enable_trt_engine_cache),
+          rclcpp::Parameter("Backend.trt_engine_cache_path", params_.trt_engine_cache_path)};
+        backend_node = createBackendConfigNode(provider_name, std::move(overrides));
+      } else {
+        backend_node = createBackendConfigNode("cpu");
       }
 
+      // Load plugin using pluginlib
+      plugin_holder_ = plugin_loader_->createUniqueInstance(plugin_name);
+      plugin_holder_->initialize(backend_node);
+      allocator_ = plugin_holder_->get_allocator();
+      executor_ = plugin_holder_->get_inference_executor();
+
       if (!executor_ || !allocator_) {
         throw std::runtime_error("Executor or allocator is null");
       }
@@ -281,4 +288,17 @@ deep_ros::Tensor BackendManager::buildInputTensor(const PackedInput & packed) co
   return tensor;
 }
 
+std::string BackendManager::providerToPluginName(Provider provider) const
+{
+  switch (provider) {
+    case Provider::TENSORRT:
+    case Provider::CUDA:
+      return "onnxruntime_gpu";
+    case Provider::CPU:
+      return "onnxruntime_cpu";
+    default:
+      return "";
+  }
+}
+
 }  // namespace deep_yolo_inference