lucas/onnx_gpu_backend (#24)

* Initial deep_ort_gpu plugin with internal library issues

* Functioning onnxruntime with CUDA as execution provider

* Removed requirement for cuda toolkit

* cleaning up cmake

* updating tests

* parameterizing parameters that weren't actually parameters, updating tests in the process

* addressing issues with IO binding, thread-loca caching bug (idk this one), unused members and hardcoded float types

* launch tests for local use only

* trying to get the tensorrt EP working

* tensorrt tested to be working apparently, check deep_sample

* debugging ci

* testing

---------

Co-authored-by: Eddy Zhou <edsteredward@gmail.com>

Author: lucasreljic

.devcontainer/Dockerfile

@@ -1,7 +1,6 @@
 # Multi-stage ROS2 development container (CPU/GPU compatible)
 ARG ROS_DISTRO=humble
-ARG CUDA_VERSION=12.4.0
-ARG UBUNTU_VERSION=22.04
+ARG NVIDIA_CONTAINER_TAG=12.4.1-cudnn-runtime-ubuntu22.04
 
 # ===============================================
 # CPU Base - Standard ROS2 image
@@ -11,7 +10,7 @@ FROM ros:${ROS_DISTRO}-ros-base AS cpu-base
 # ===============================================
 # GPU Base - CUDA with manual ROS2 install
 # ===============================================
-FROM nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION} AS gpu-base
+FROM nvidia/cuda:${NVIDIA_CONTAINER_TAG} AS gpu-base
 ARG ROS_DISTRO
 ARG TENSORRT_RUNTIME_VERSION
 ARG TENSORRT_CUDA_VERSION
@@ -39,16 +38,19 @@ ENV DEBIAN_FRONTEND=interactive
 RUN curl -fsSL -o cuda-keyring_1.1-1_all.deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb \
     && dpkg -i cuda-keyring_1.1-1_all.deb \
     && apt-get update && apt-get install -y --no-install-recommends \
-    libnvinfer-lean10=${TENSORRT_RUNTIME_VERSION}-1+cuda${TENSORRT_CUDA_VERSION} \
+    libnvinfer10=${TENSORRT_RUNTIME_VERSION}-1+cuda${TENSORRT_CUDA_VERSION} \
+    libnvinfer-plugin10=${TENSORRT_RUNTIME_VERSION}-1+cuda${TENSORRT_CUDA_VERSION} \
+    libnvonnxparsers10=${TENSORRT_RUNTIME_VERSION}-1+cuda${TENSORRT_CUDA_VERSION} \
     && rm cuda-keyring_1.1-1_all.deb
 
 # ===============================================
-# Install Common Development Tools from Either Base
+# Install Common Development Tools in Either Base
 # ===============================================
 FROM ${TARGETARCH:-cpu}-base AS dev-tools
 
 # Install development tools not in base image
 RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
     python3-pip \
     python3-colcon-common-extensions \
     python3-rosdep \
@@ -70,7 +72,9 @@ FROM dev-tools AS source
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN --mount=type=bind,source=.,target=/tmp/src \
-    apt-get -qq update && rosdep update && \
+    apt-get -qq update && \
+    rosdep init || true && \
+    rosdep update && \
     rosdep install --from-paths /tmp/src --ignore-src -r -s \
         | (grep 'apt-get install' || true) \
         | awk '{print $3}' \

.devcontainer/generate_devcontainer.sh

@@ -15,14 +15,13 @@
 # limitations under the License.
 # generate_devcontainer.sh
 # Usage:
-#   ./generate_devcontainer.sh <ros_distro> <container_type> [cuda_version] [ubuntu_version]
+#   ./generate_devcontainer.sh <ros_distro> <container_type> [nvidia_container_tag]
 
 set -e
 
 ROS_DISTRO=${1:-humble}
 CONTAINER_TYPE=${2:-cpu}
-CUDA_VERSION=${3:-12.4.0}
-UBUNTU_VERSION=${4:-22.04}
+NVIDIA_CONTAINER_TAG=${3:-12.4.1-cudnn-runtime-ubuntu22.04}
 USERNAME=${USER:-vscode}
 
 # TensorRT configuration
@@ -35,8 +34,7 @@ echo "ROS Distribution: $ROS_DISTRO"
 echo "Container Type: $CONTAINER_TYPE"
 
 if [ "$CONTAINER_TYPE" = "gpu" ]; then
-    echo "CUDA Version: $CUDA_VERSION"
-    echo "Ubuntu Version: $UBUNTU_VERSION"
+    echo "NVIDIA Container Tag: $NVIDIA_CONTAINER_TAG"
     echo "TensorRT Runtime Version: $TENSORRT_RUNTIME_VERSION"
     echo "TensorRT CUDA Version: $TENSORRT_CUDA_VERSION"
 fi
@@ -47,8 +45,7 @@ if [ "$CONTAINER_TYPE" = "gpu" ]; then
     BUILD_ARGS='"ROS_DISTRO": "'$ROS_DISTRO'",
       "USERNAME": "'$USERNAME'",
       "TARGETARCH": "gpu",
-      "CUDA_VERSION": "'$CUDA_VERSION'",
-      "UBUNTU_VERSION": "'$UBUNTU_VERSION'",
+      "NVIDIA_CONTAINER_TAG": "'$NVIDIA_CONTAINER_TAG'",
       "TENSORRT_RUNTIME_VERSION": "'$TENSORRT_RUNTIME_VERSION'",
       "TENSORRT_CUDA_VERSION": "'$TENSORRT_CUDA_VERSION'",
       "USER_UID": "'$(id -u)'",

.github/actions/build-and-test/action.yml

@@ -29,13 +29,13 @@ runs:
       shell: bash
       run: |
         source /opt/ros/${{ inputs.ros-distro }}/setup.bash
-        colcon build --merge-install
+        IS_CI=1 colcon build --merge-install
 
     - name: ✅ Run tests
       shell: bash
       run: |
         source /opt/ros/${{ inputs.ros-distro }}/setup.bash
-        colcon test --merge-install --event-handlers console_cohesion+
+        IS_CI=1 colcon test --merge-install --event-handlers console_cohesion+
         colcon test-result --verbose
 
     - name: 🗃️ Upload test logs as artifacts

.gitignore

@@ -7,6 +7,9 @@ log/
 .devcontainer/devcontainer.json
 .devcontainer/.env
 
+# Launch test cache
+__pycache__/
+
 # Claude helpers
 .claude/
 CLAUDE.md

.vscode/tasks.json

@@ -27,8 +27,7 @@
       "args": [
         "${input:rosDistro}",
         "gpu",
-        "${input:cudaVersion}",
-        "${input:ubuntuVersion}"
+        "${input:nvidiaContainer}"
       ],
       "problemMatcher": [],
       "group": "build",
@@ -72,41 +71,32 @@
       "default": "cpu"
     },
     {
-      "id": "cudaVersion",
+      "id": "nvidiaContainer",
       "type": "pickString",
-      "description": "Select CUDA version",
+      "description": "Select NVIDIA CUDA container (runtime + cuDNN, Ubuntu 20/22/24)",
       "options": [
-        "12.8.0",
-        "12.6.2",
-        "12.5.1",
-        "12.4.1",
-        "12.4.0",
-        "12.3.2",
-        "12.2.2",
-        "12.1.1",
-        "12.0.1"
+        "12.8.0-cudnn-runtime-ubuntu24.04",
+        "12.8.0-cudnn-runtime-ubuntu22.04",
+        "12.8.0-cudnn-runtime-ubuntu20.04",
+        "12.6.2-cudnn-runtime-ubuntu24.04",
+        "12.6.2-cudnn-runtime-ubuntu22.04",
+        "12.6.2-cudnn-runtime-ubuntu20.04",
+        "12.5.1-cudnn-runtime-ubuntu24.04",
+        "12.5.1-cudnn-runtime-ubuntu22.04",
+        "12.5.1-cudnn-runtime-ubuntu20.04",
+        "12.4.1-cudnn-runtime-ubuntu24.04",
+        "12.4.1-cudnn-runtime-ubuntu22.04",
+        "12.4.1-cudnn-runtime-ubuntu20.04",
+        "12.3.2-cudnn-runtime-ubuntu22.04",
+        "12.3.2-cudnn-runtime-ubuntu20.04",
+        "12.2.2-cudnn8-runtime-ubuntu22.04",
+        "12.2.2-cudnn8-runtime-ubuntu20.04",
+        "12.1.1-cudnn8-runtime-ubuntu22.04",
+        "12.1.1-cudnn8-runtime-ubuntu20.04",
+        "12.0.1-cudnn8-runtime-ubuntu22.04",
+        "12.0.1-cudnn8-runtime-ubuntu20.04"
       ],
-      "default": "12.4.0"
-    },
-    {
-      "id": "ubuntuVersion",
-      "type": "pickString",
-      "description": "Select Ubuntu version",
-      "options": [
-        {
-          "label": "Ubuntu 24.04 (Noble)",
-          "value": "24.04"
-        },
-        {
-          "label": "Ubuntu 22.04 (Jammy)",
-          "value": "22.04"
-        },
-        {
-          "label": "Ubuntu 20.04 (Focal)",
-          "value": "20.04"
-        }
-      ],
-      "default": "22.04"
+      "default": "12.4.1-cudnn-runtime-ubuntu22.04"
     }
   ]
 }

DEVELOPING.md

@@ -29,3 +29,37 @@ This project includes VS Code dev container configurations for easy ROS2 develop
 ### Common Commands
 
 Inside the container, you can do ros2 commands, colcon commands, rosdep, etc.
+
+## Testing
+
+### CI Testing (CPU Only)
+
+```bash
+export IS_CI=1
+colcon build
+source install/setup.bash && colcon test
+colcon test-result --verbose
+```
+
+GPU backends are automatically skipped when `IS_CI=1`.
+
+### Local GPU Testing
+
+**Requires**: NVIDIA container with GPU access (Runtime Libraries and CuDNN)
+Testing with GPU can be done using one of the support GPU devcontainers.
+
+```bash
+export IS_CI=0  # or unset IS_CI
+colcon build
+source install/setup.bash && colcon test
+colcon test-result --verbose
+```
+
+### Testing Philosophy
+
+Given that access to GPU runners are limited, we settled on a testing procedure to test GPU-related software locally. The codebase is designed such that **if a backend works with `deep_sample`, it works everywhere.**
+
+- CPU backends: Tested in CI automatically
+- GPU backends: Must be tested locally with GPU before release
+- `deep_sample` validates all backend plugins end-to-end
+- Other packages tested with CPU only in CI

deep_core/CMakeLists.txt

@@ -47,9 +47,9 @@ target_include_directories(${DEEP_CORE_LIB} PUBLIC
 target_link_libraries(${DEEP_CORE_LIB}
   PUBLIC
     pluginlib::pluginlib
-  PRIVATE
     rclcpp::rclcpp
     rclcpp_lifecycle::rclcpp_lifecycle
+  PRIVATE
     bondcpp::bondcpp
 )
 

deep_core/include/deep_core/plugin_interfaces/backend_inference_executor.hpp

@@ -19,6 +19,8 @@
 #include <string>
 #include <vector>
 
+#include <rclcpp/rclcpp.hpp>
+
 #include "deep_core/types/tensor.hpp"
 
 namespace deep_ros

deep_core/include/deep_core/plugin_interfaces/backend_memory_allocator.hpp

@@ -18,6 +18,8 @@
 #include <string>
 #include <vector>
 
+#include <rclcpp/rclcpp.hpp>
+
 namespace deep_ros
 {
 

deep_core/include/deep_core/plugin_interfaces/deep_backend_plugin.hpp

@@ -17,6 +17,9 @@
 #include <memory>
 #include <string>
 
+#include <rclcpp/rclcpp.hpp>
+#include <rclcpp_lifecycle/lifecycle_node.hpp>
+
 #include "deep_core/plugin_interfaces/backend_inference_executor.hpp"
 #include "deep_core/plugin_interfaces/backend_memory_allocator.hpp"
 
@@ -34,6 +37,12 @@ class DeepBackendPlugin
 public:
   virtual ~DeepBackendPlugin() = default;
 
+  /**
+   * @brief Initialize plugin with node instance to enable parameter loading
+   * @param node Shared pointer to the lifecycle node instance
+   */
+  virtual void initialize(rclcpp_lifecycle::LifecycleNode::SharedPtr node) = 0;
+
   /**
    * @brief Get the backend name
    * @return Backend name (e.g., "onnxruntime", "tensorrt")