Add Onnxruntime (CPU-only) backend plugin (#17)

Author: Edwardius

DEVELOPING.md

@@ -31,6 +31,9 @@ This project includes VS Code dev container configurations for easy ROS2 develop
 Inside the container:
 
 ```bash
+# Update apt
+sudo apt update
+
 # Update rosdep
 rosdep update
 

deep_core/include/deep_core/deep_node_base.hpp

@@ -135,10 +135,10 @@ class DeepNodeBase : public rclcpp_lifecycle::LifecycleNode
 
   /**
    * @brief Run inference on input tensor
-   * @param inputs Input tensor for inference
+   * @param inputs Input tensor for inference (note: some backends may require mutable access for zero-copy operations)
    * @return Output tensor from inference
    */
-  Tensor run_inference(const Tensor & inputs);
+  Tensor run_inference(Tensor & inputs);
 
   /**
    * @brief Check if a backend plugin is loaded

deep_core/include/deep_core/plugin_interfaces/backend_inference_executor.hpp

@@ -45,12 +45,12 @@ class BackendInferenceExecutor
 
   /**
    * @brief Run inference on input tensor
-   * @param input Input tensor
+   * @param input Input tensor (note: some backends may require mutable access for zero-copy operations)
    * @return Output tensor
    * @throws std::invalid_argument if input tensor is invalid
    * @throws std::runtime_error if no model is loaded
    */
-  Tensor run_inference(const Tensor & input);
+  Tensor run_inference(Tensor & input);
 
   /**
    * @brief Unload the currently loaded model
@@ -80,8 +80,9 @@ class BackendInferenceExecutor
 
   /**
    * @brief Implementation of run_inference (to be overridden by backends)
+   * @param input Input tensor (note: some backends may require mutable access)
    */
-  virtual Tensor run_inference_impl(const Tensor & input) = 0;
+  virtual Tensor run_inference_impl(Tensor & input) = 0;
 
   /**
    * @brief Implementation of unload_model (to be overridden by backends)

deep_core/package.xml

@@ -13,7 +13,6 @@
   <depend>pluginlib</depend>
   <depend>bondcpp</depend>
 
-
   <export>
     <build_type>ament_cmake</build_type>
   </export>

deep_core/src/backend_inference_executor.cpp

@@ -34,7 +34,7 @@ bool BackendInferenceExecutor::load_model(const std::filesystem::path & model_pa
   return success;
 }
 
-Tensor BackendInferenceExecutor::run_inference(const Tensor & input)
+Tensor BackendInferenceExecutor::run_inference(Tensor & input)
 {
   // Validate input tensor
   if (input.data() == nullptr) {

deep_core/src/deep_node_base.cpp

@@ -191,7 +191,7 @@ void DeepNodeBase::unload_model()
   }
 }
 
-Tensor DeepNodeBase::run_inference(const Tensor & inputs)
+Tensor DeepNodeBase::run_inference(Tensor & inputs)
 {
   if (!plugin_) {
     throw std::runtime_error("No plugin loaded");

deep_ort_backend_plugin/CMakeLists.txt

@@ -0,0 +1,96 @@
+# Copyright (c) 2025-present WATonomous. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cmake_minimum_required(VERSION 3.22)
+project(deep_ort_backend_plugin)
+
+if(NOT CMAKE_CXX_STANDARD)
+  set(CMAKE_CXX_STANDARD 17)
+endif()
+
+if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+  add_compile_options(-Wall -Wextra -Wpedantic)
+  add_link_options(-Wl,-no-undefined)
+endif()
+
+find_package(ament_cmake REQUIRED)
+find_package(deep_core REQUIRED)
+find_package(onnxruntime_vendor REQUIRED)
+find_package(pluginlib REQUIRED)
+find_package(rclcpp REQUIRED)
+find_package(rclcpp_lifecycle REQUIRED)
+
+set(include_dir ${CMAKE_CURRENT_SOURCE_DIR}/include)
+
+# deep_ort_backend_plugin library
+set(DEEP_ORT_LIB ${PROJECT_NAME}_lib)
+add_library(${DEEP_ORT_LIB} SHARED
+  src/ort_cpu_memory_allocator.cpp
+  src/ort_backend_executor.cpp
+  src/ort_backend_plugin.cpp
+)
+
+target_include_directories(${DEEP_ORT_LIB} PUBLIC
+  $<BUILD_INTERFACE:${include_dir}>
+  $<INSTALL_INTERFACE:include>
+)
+
+target_link_libraries(${DEEP_ORT_LIB}
+  PUBLIC
+    pluginlib::pluginlib
+    rclcpp::rclcpp
+  PRIVATE
+    deep_core::deep_core_lib
+    onnxruntime_vendor::onnxruntime_lib
+)
+
+install(TARGETS
+  ${DEEP_ORT_LIB}
+  EXPORT ${PROJECT_NAME}Targets
+  ARCHIVE DESTINATION lib
+  LIBRARY DESTINATION lib
+  RUNTIME DESTINATION bin
+)
+
+install(EXPORT ${PROJECT_NAME}Targets
+  NAMESPACE ${PROJECT_NAME}::
+  DESTINATION share/${PROJECT_NAME}/cmake
+)
+
+install(DIRECTORY include/
+  DESTINATION include
+)
+
+install(FILES plugins.xml
+  DESTINATION share/${PROJECT_NAME}
+)
+
+# Export plugin description file to ament index
+pluginlib_export_plugin_description_file(deep_core plugins.xml)
+
+if(BUILD_TESTING)
+  find_package(deep_test REQUIRED)
+
+  add_deep_test(test_ort_backend test/test_ort_backend.cpp
+    LIBRARIES
+      ${DEEP_ORT_LIB}
+      deep_core::deep_core_lib
+      onnxruntime_vendor::onnxruntime_lib
+  )
+
+endif()
+
+ament_export_targets(${PROJECT_NAME}Targets HAS_LIBRARY_TARGET)
+ament_export_libraries(${DEEP_ORT_LIB})
+ament_package()

deep_ort_backend_plugin/README.md

@@ -0,0 +1,40 @@
+# deep_ort_backend_plugin
+
+ONNX Runtime CPU backend plugin for deep_core.
+
+## Overview
+
+Provides:
+- CPU inference executor using ONNX Runtime
+- 64-byte aligned memory allocator for optimal SIMD performance
+- Zero-copy inference with IO binding
+
+## Plugin Name
+
+`onnxruntime_cpu`
+
+## Supported Formats
+
+ONNX models (.onnx files)
+
+## Usage
+
+Add to your `package.xml`:
+
+```xml
+<exec_depend>deep_ort_backend_plugin</exec_depend>
+```
+
+Configure your inference nodes to use this plugin:
+
+```yaml
+inference_node:
+  ros__parameters:
+    Backend.plugin: "onnxruntime_cpu"
+    model_path: "/path/to/model.onnx"
+```
+
+## Dependencies
+
+- deep_core
+- onnxruntime_vendor

deep_ort_backend_plugin/include/deep_ort_backend_plugin/ort_backend_executor.hpp

@@ -0,0 +1,106 @@
+// Copyright (c) 2025-present WATonomous. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <onnxruntime_cxx_api.h>
+
+#include <filesystem>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <deep_core/plugin_interfaces/backend_inference_executor.hpp>
+#include <deep_core/types/tensor.hpp>
+
+namespace deep_ort_backend
+{
+
+/**
+ * @brief ONNX Runtime backend inference executor
+ *
+ * Provides inference execution using ONNX Runtime with CPU optimization.
+ * Uses zero-copy IO binding for efficient tensor operations.
+ */
+class OrtBackendExecutor : public deep_ros::BackendInferenceExecutor
+{
+public:
+  /**
+   * @brief Constructor - initializes ONNX Runtime environment
+   */
+  OrtBackendExecutor();
+
+  /**
+   * @brief Destructor
+   */
+  ~OrtBackendExecutor() override = default;
+
+  /**
+   * @brief Get supported model formats
+   * @return Vector containing "onnx"
+   */
+  std::vector<std::string> supported_model_formats() const override;
+
+protected:
+  /**
+   * @brief Load an ONNX model from file
+   * @param model_path Path to the .onnx model file
+   * @return true if successful, false otherwise
+   */
+  bool load_model_impl(const std::filesystem::path & model_path) override;
+
+  /**
+   * @brief Run inference using zero-copy IO binding
+   * @param input Input tensor (must be compatible with model input)
+   * @return Output tensor with inference results
+   * @throws std::runtime_error if inference fails or no model loaded
+   */
+  deep_ros::Tensor run_inference_impl(deep_ros::Tensor & input) override;
+
+  /**
+   * @brief Unload the currently loaded model
+   */
+  void unload_model_impl() override;
+
+private:
+  std::filesystem::path model_path_;
+
+  std::unique_ptr<Ort::Env> env_;
+  std::unique_ptr<Ort::Session> session_;
+  Ort::MemoryInfo memory_info_;
+
+  /**
+   * @brief Convert deep_ros DataType to ONNX tensor element type
+   * @param dtype deep_ros data type
+   * @return ONNX tensor element data type
+   */
+  ONNXTensorElementDataType convert_to_onnx_type(deep_ros::DataType dtype) const;
+
+  /**
+   * @brief Get model output shape based on input shape
+   * @param input_shape Input tensor shape
+   * @return Expected output tensor shape
+   * @throws std::runtime_error if model not loaded or shape inference fails
+   */
+  std::vector<size_t> get_output_shape(const std::vector<size_t> & input_shape) const;
+
+  /**
+   * @brief Get element size in bytes for a data type
+   * @param dtype Data type
+   * @return Size in bytes per element
+   */
+  size_t get_element_size(deep_ros::DataType dtype) const;
+};
+
+}  // namespace deep_ort_backend

deep_ort_backend_plugin/include/deep_ort_backend_plugin/ort_backend_plugin.hpp

@@ -0,0 +1,69 @@
+// Copyright (c) 2025-present WATonomous. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include <deep_core/plugin_interfaces/backend_inference_executor.hpp>
+#include <deep_core/plugin_interfaces/backend_memory_allocator.hpp>
+#include <deep_core/plugin_interfaces/deep_backend_plugin.hpp>
+
+namespace deep_ort_backend
+{
+
+/**
+ * @brief ONNX Runtime backend plugin
+ *
+ * Combines ORT CPU memory allocator and inference executor into a single
+ * backend plugin for use with pluginlib.
+ */
+class OrtBackendPlugin : public deep_ros::DeepBackendPlugin
+{
+public:
+  /**
+   * @brief Constructor - initializes ORT allocator and executor
+   */
+  OrtBackendPlugin();
+
+  /**
+   * @brief Destructor
+   */
+  ~OrtBackendPlugin() override = default;
+
+  /**
+   * @brief Get backend name
+   * @return "onnxruntime"
+   */
+  std::string backend_name() const override;
+
+  /**
+   * @brief Get the ORT CPU memory allocator
+   * @return Shared pointer to ORT memory allocator
+   */
+  std::shared_ptr<deep_ros::BackendMemoryAllocator> get_allocator() const override;
+
+  /**
+   * @brief Get the ORT inference executor
+   * @return Shared pointer to ORT inference executor
+   */
+  std::shared_ptr<deep_ros::BackendInferenceExecutor> get_inference_executor() const override;
+
+private:
+  std::shared_ptr<deep_ros::BackendMemoryAllocator> allocator_;
+  std::shared_ptr<deep_ros::BackendInferenceExecutor> executor_;
+};
+
+}  // namespace deep_ort_backend