|
1 | 1 | # deep_conversions |
2 | 2 |
|
3 | | -Conversion utilities for seamlessly transforming ROS 2 sensor messages into tensors for machine learning inference and back to ROS messages for publishing results. |
| 3 | +ROS 2 sensor message to tensor conversion library. |
4 | 4 |
|
5 | 5 | ## Overview |
6 | 6 |
|
7 | | -`deep_conversions` provides high-performance, zero-copy conversions between ROS 2 sensor messages and the `deep_core` tensor system. This enables efficient integration of ML models with ROS 2 sensor data streams. |
8 | | - |
9 | | -## Features |
10 | | - |
11 | | -### 🔄 Bidirectional Conversions |
12 | | -- **ROS → Tensor**: Convert sensor messages to ML-ready tensors |
13 | | -- **Tensor → ROS**: Convert inference results back to ROS messages |
14 | | -- **Batch processing**: Handle multiple messages efficiently |
15 | | -- **Memory-efficient**: Minimal copying with custom allocators |
16 | | - |
17 | | -### 📷 Comprehensive Sensor Support |
18 | | -- **Images**: All standard ROS image encodings |
19 | | -- **Point Clouds**: PointCloud2 with arbitrary field configurations |
20 | | -- **Laser Scans**: Range and intensity data |
21 | | -- **IMU Data**: Orientation, acceleration, and angular velocity |
22 | | - |
23 | | -### ⚡ Performance Optimized |
24 | | -- **Custom allocators**: Use backend-specific memory (CPU, GPU) |
25 | | -- **Zero-copy operations**: Direct memory mapping where possible |
26 | | -- **Batch processing**: Efficient multi-message tensor creation |
27 | | -- **Type safety**: Compile-time type checking and validation |
| 7 | +Provides generic conversions between ROS 2 sensor messages and `deep_core` tensors. |
28 | 8 |
|
29 | 9 | ## Supported Conversions |
30 | 10 |
|
31 | | -### Image Messages |
32 | | - |
33 | | -Convert `sensor_msgs::msg::Image` to tensors with proper shape and data type handling: |
34 | | - |
35 | | -```cpp |
36 | | -#include <deep_conversions/ros_conversions.hpp> |
37 | | - |
38 | | -// Single image conversion |
39 | | -sensor_msgs::msg::Image ros_image; |
40 | | -auto allocator = get_gpu_allocator(); // Custom allocator |
41 | | -auto tensor = deep_ros::ros_conversions::from_image(ros_image, allocator); |
42 | | -// Shape: [1, height, width, channels] |
43 | | - |
44 | | -// Batch conversion |
45 | | -std::vector<sensor_msgs::msg::Image> images; |
46 | | -auto batch_tensor = deep_ros::ros_conversions::from_image(images, allocator); |
47 | | -// Shape: [batch_size, height, width, channels] |
48 | | - |
49 | | -// Convert back to ROS |
50 | | -sensor_msgs::msg::Image output_image; |
51 | | -deep_ros::ros_conversions::to_image(result_tensor, output_image, "bgr8"); |
52 | | -``` |
53 | | -
|
54 | | -### Point Cloud Messages |
55 | | -
|
56 | | -Convert `sensor_msgs::msg::PointCloud2` to structured tensors: |
57 | | -
|
58 | | -```cpp |
59 | | -// Point cloud conversion |
60 | | -sensor_msgs::msg::PointCloud2 cloud; |
61 | | -auto tensor = deep_ros::ros_conversions::from_pointcloud2(cloud, allocator); |
62 | | -// Shape: [num_points, num_fields] (e.g., [100000, 4] for XYZI) |
63 | | -
|
64 | | -// Access structured data |
65 | | -// Field order: [x, y, z, intensity, ...] as defined in cloud.fields |
66 | | -``` |
67 | | - |
68 | | -### Laser Scan Messages |
69 | | - |
70 | | -Convert `sensor_msgs::msg::LaserScan` to range tensors: |
71 | | - |
72 | | -```cpp |
73 | | -// Laser scan conversion |
74 | | -sensor_msgs::msg::LaserScan scan; |
75 | | -auto tensor = deep_ros::ros_conversions::from_laserscan(scan, allocator); |
76 | | -// Shape: [num_ranges] or [num_ranges, 2] if intensities present |
77 | | -``` |
78 | | - |
79 | | -### IMU Messages |
80 | | - |
81 | | -Convert `sensor_msgs::msg::Imu` to standardized tensors: |
82 | | - |
83 | | -```cpp |
84 | | -// IMU conversion |
85 | | -sensor_msgs::msg::Imu imu; |
86 | | -auto tensor = deep_ros::ros_conversions::from_imu(imu, allocator); |
87 | | -// Shape: [10] -> [qx, qy, qz, qw, ax, ay, az, gx, gy, gz] |
88 | | -``` |
89 | | - |
90 | | -## Usage Examples |
91 | | - |
92 | | -### Basic Image Processing Pipeline |
93 | | - |
94 | | -```cpp |
95 | | -#include <deep_conversions/ros_conversions.hpp> |
96 | | -#include <deep_core/deep_node_base.hpp> |
97 | | - |
98 | | -class ImageProcessorNode : public deep_ros::DeepNodeBase |
99 | | -{ |
100 | | -public: |
101 | | - ImageProcessorNode() : DeepNodeBase("image_processor") { |
102 | | - // Load model and setup subscriptions |
103 | | - load_plugin("onnxruntime_cpu"); |
104 | | - load_model("/models/object_detection.onnx"); |
105 | | - |
106 | | - image_sub_ = create_subscription<sensor_msgs::msg::Image>( |
107 | | - "/camera/image", 10, |
108 | | - std::bind(&ImageProcessorNode::image_callback, this, std::placeholders::_1) |
109 | | - ); |
110 | | - |
111 | | - result_pub_ = create_publisher<sensor_msgs::msg::Image>("/detection/result", 10); |
112 | | - } |
113 | | - |
114 | | -private: |
115 | | - void image_callback(const sensor_msgs::msg::Image::SharedPtr msg) { |
116 | | - try { |
117 | | - // Convert ROS image to tensor |
118 | | - auto allocator = get_current_allocator(); |
119 | | - auto input_tensor = deep_ros::ros_conversions::from_image(*msg, allocator); |
120 | | - |
121 | | - // Run inference |
122 | | - auto output_tensor = run_inference(input_tensor); |
123 | | - |
124 | | - // Convert result back to ROS image |
125 | | - sensor_msgs::msg::Image result_msg; |
126 | | - deep_ros::ros_conversions::to_image(output_tensor, result_msg, "bgr8", msg->header); |
127 | | - |
128 | | - // Publish result |
129 | | - result_pub_->publish(result_msg); |
130 | | - |
131 | | - } catch (const std::exception& e) { |
132 | | - RCLCPP_ERROR(get_logger(), "Processing failed: %s", e.what()); |
133 | | - } |
134 | | - } |
135 | | - |
136 | | - rclcpp::Subscription<sensor_msgs::msg::Image>::SharedPtr image_sub_; |
137 | | - rclcpp::Publisher<sensor_msgs::msg::Image>::SharedPtr result_pub_; |
138 | | -}; |
139 | | -``` |
140 | | -
|
141 | | -### Multi-Sensor Fusion |
142 | | -
|
143 | | -```cpp |
144 | | -class FusionNode : public rclcpp::Node |
145 | | -{ |
146 | | -public: |
147 | | - FusionNode() : Node("fusion_node") { |
148 | | - // Setup synchronized subscribers |
149 | | - image_sub_.subscribe(this, "/camera/image"); |
150 | | - lidar_sub_.subscribe(this, "/lidar/scan"); |
151 | | - imu_sub_.subscribe(this, "/imu/data"); |
152 | | -
|
153 | | - // Synchronize messages |
154 | | - sync_.reset(new Synchronizer(SyncPolicy(10), image_sub_, lidar_sub_, imu_sub_)); |
155 | | - sync_->registerCallback(std::bind(&FusionNode::fusion_callback, this, _1, _2, _3)); |
156 | | - } |
157 | | -
|
158 | | -private: |
159 | | - void fusion_callback( |
160 | | - const sensor_msgs::msg::Image::ConstSharedPtr& image, |
161 | | - const sensor_msgs::msg::LaserScan::ConstSharedPtr& scan, |
162 | | - const sensor_msgs::msg::Imu::ConstSharedPtr& imu) |
163 | | - { |
164 | | - // Convert all sensors to tensors |
165 | | - auto image_tensor = deep_ros::ros_conversions::from_image(*image, allocator_); |
166 | | - auto lidar_tensor = deep_ros::ros_conversions::from_laserscan(*scan, allocator_); |
167 | | - auto imu_tensor = deep_ros::ros_conversions::from_imu(*imu, allocator_); |
168 | | -
|
169 | | - // Concatenate or process as needed for fusion model |
170 | | - auto fused_input = create_fusion_tensor(image_tensor, lidar_tensor, imu_tensor); |
171 | | -
|
172 | | - // Run fusion inference |
173 | | - auto result = run_fusion_model(fused_input); |
| 11 | +### From ROS to Tensor |
| 12 | +- **Images** (`sensor_msgs/msg/Image`) → Tensor with shape `[batch, height, width, channels]` or `[batch, height, width]` for grayscale |
| 13 | +- **PointCloud2** (`sensor_msgs/msg/PointCloud2`) → Tensor with shape `[num_points, num_fields]` |
| 14 | +- **LaserScan** (`sensor_msgs/msg/LaserScan`) → Tensor with shape `[num_ranges]` or `[num_ranges, 2]` with intensities |
| 15 | +- **IMU** (`sensor_msgs/msg/Imu`) → Tensor with shape `[10]` containing quaternion, linear acceleration, and angular velocity |
174 | 16 |
|
175 | | - // Publish results... |
176 | | - } |
| 17 | +### From Tensor to ROS |
| 18 | +- Tensor → **Image** with specified encoding |
177 | 19 |
|
178 | | - // Subscriber and synchronizer setup... |
179 | | -}; |
180 | | -``` |
| 20 | +## Usage |
181 | 21 |
|
182 | | -## Image Encoding Support |
| 22 | +Include the appropriate header for your message type: |
| 23 | +- `deep_conversions/image_conversions.hpp` |
| 24 | +- `deep_conversions/pointcloud_conversions.hpp` |
| 25 | +- `deep_conversions/laserscan_conversions.hpp` |
| 26 | +- `deep_conversions/imu_conversions.hpp` |
183 | 27 |
|
184 | | -### Input Encodings (ROS → Tensor) |
185 | | -- **Grayscale**: `mono8`, `mono16` |
186 | | -- **Color**: `rgb8`, `bgr8`, `rgba8`, `bgra8` |
187 | | -- **High Dynamic Range**: `rgb16`, `bgr16`, `rgba16`, `bgra16` |
188 | | -- **YUV**: `yuv422`, `yuv444` |
189 | | -- **Bayer**: `bayer_rggb8`, `bayer_bggr8`, `bayer_gbrg8`, `bayer_grbg8` |
190 | | - |
191 | | -### Output Encodings (Tensor → ROS) |
192 | | -- **Standard formats**: Automatic data type conversion |
193 | | -- **Normalization**: [0,1] float tensors → 8-bit integers |
194 | | -- **Multi-channel**: Support for arbitrary channel counts |
195 | | - |
196 | | -### Automatic Type Conversion |
197 | | - |
198 | | -```cpp |
199 | | -// The conversion system automatically handles: |
200 | | -ImageEncoding encoding_info = get_image_encoding_info("bgr8"); |
201 | | -// encoding_info.dtype = DataType::UINT8 |
202 | | -// encoding_info.channels = 3 |
203 | | -// encoding_info.bytes_per_channel = 1 |
204 | | -``` |
205 | | - |
206 | | -## Memory Management |
207 | | - |
208 | | -### Custom Allocator Integration |
209 | | - |
210 | | -```cpp |
211 | | -// Use backend-specific allocators |
212 | | -auto gpu_allocator = get_cuda_allocator(); |
213 | | -auto cpu_allocator = get_cpu_allocator(); |
214 | | - |
215 | | -// GPU-based processing |
216 | | -auto gpu_tensor = deep_ros::ros_conversions::from_image(image, gpu_allocator); |
217 | | -auto gpu_result = run_gpu_inference(gpu_tensor); |
218 | | - |
219 | | -// CPU-based processing |
220 | | -auto cpu_tensor = deep_ros::ros_conversions::from_image(image, cpu_allocator); |
221 | | -auto cpu_result = run_cpu_inference(cpu_tensor); |
222 | | -``` |
223 | | - |
224 | | -### Zero-Copy Operations |
225 | | - |
226 | | -```cpp |
227 | | -// When possible, conversions avoid memory copying: |
228 | | -// 1. Direct memory mapping for compatible formats |
229 | | -// 2. In-place transformations for layout changes |
230 | | -// 3. Shared memory for batch operations |
231 | | -``` |
232 | | - |
233 | | -## Performance Considerations |
234 | | - |
235 | | -### Efficient Batch Processing |
236 | | - |
237 | | -```cpp |
238 | | -// Batch multiple images for better throughput |
239 | | -std::vector<sensor_msgs::msg::Image> image_batch; |
240 | | -// ... collect images ... |
241 | | - |
242 | | -// Single conversion call for entire batch |
243 | | -auto batch_tensor = deep_ros::ros_conversions::from_image(image_batch, allocator); |
244 | | -// Much faster than individual conversions |
245 | | -``` |
246 | | - |
247 | | -### Memory Layout Optimization |
248 | | - |
249 | | -```cpp |
250 | | -// Tensors use optimal memory layouts: |
251 | | -// - Contiguous memory for cache efficiency |
252 | | -// - Proper alignment for SIMD operations |
253 | | -// - Backend-specific memory locations (CPU/GPU) |
254 | | -``` |
255 | | - |
256 | | -## Error Handling |
257 | | - |
258 | | -### Validation and Safety |
259 | | - |
260 | | -```cpp |
261 | | -try { |
262 | | - auto tensor = deep_ros::ros_conversions::from_image(image, allocator); |
263 | | -} catch (const std::invalid_argument& e) { |
264 | | - // Unsupported encoding or invalid dimensions |
265 | | - RCLCPP_ERROR(logger, "Conversion failed: %s", e.what()); |
266 | | -} catch (const std::runtime_error& e) { |
267 | | - // Memory allocation or data conversion errors |
268 | | - RCLCPP_ERROR(logger, "Runtime error: %s", e.what()); |
269 | | -} |
270 | | -``` |
271 | | - |
272 | | -### Common Issues |
273 | | - |
274 | | -1. **Encoding mismatch**: Ensure output encoding matches tensor data type |
275 | | -2. **Memory alignment**: Some backends require specific memory alignment |
276 | | -3. **Batch size consistency**: All images in batch must have same dimensions |
277 | | - |
278 | | -## Package Structure |
279 | | - |
280 | | -``` |
281 | | -deep_conversions/ |
282 | | -├── include/ |
283 | | -│ └── ros_conversions.hpp # Main conversion interface |
284 | | -├── src/ |
285 | | -│ └── ros_conversions.cpp # Implementation |
286 | | -├── test/ |
287 | | -│ └── test_ros_conversions.cpp # Unit tests |
288 | | -└── CMakeLists.txt |
289 | | -``` |
290 | | - |
291 | | -## Dependencies |
292 | | - |
293 | | -- **deep_core**: Tensor system and memory allocators |
294 | | -- **sensor_msgs**: ROS 2 sensor message types |
295 | | -- **std_msgs**: Standard ROS 2 message headers |
296 | | -- **rclcpp**: ROS 2 C++ client library |
297 | | - |
298 | | -## Testing |
299 | | - |
300 | | -```bash |
301 | | -# Run conversion tests |
302 | | -colcon test --packages-select deep_conversions |
303 | | - |
304 | | -# Performance benchmarks |
305 | | -ros2 run deep_conversions benchmark_conversions |
306 | | -``` |
307 | | - |
308 | | -## Integration Examples |
309 | | - |
310 | | -### With Computer Vision Pipeline |
311 | | - |
312 | | -```cpp |
313 | | -// YOLOv8 object detection pipeline |
314 | | -auto image_tensor = from_image(ros_image, gpu_allocator); |
315 | | -auto detections = yolo_model->run_inference(image_tensor); |
316 | | -auto annotated_image = draw_detections(image_tensor, detections); |
317 | | -to_image(annotated_image, output_msg, "bgr8"); |
318 | | -``` |
319 | | -
|
320 | | -### With SLAM Systems |
321 | | -
|
322 | | -```cpp |
323 | | -// Visual-LiDAR SLAM |
324 | | -auto image_tensor = from_image(camera_msg, allocator); |
325 | | -auto lidar_tensor = from_pointcloud2(lidar_msg, allocator); |
326 | | -auto pose_estimate = slam_model->process(image_tensor, lidar_tensor); |
327 | | -``` |
328 | | - |
329 | | -## Future Enhancements |
330 | | - |
331 | | -- [ ] **Compressed image support**: JPEG, PNG encoding/decoding |
332 | | -- [ ] **Advanced color spaces**: HSV, LAB, YCbCr conversions |
333 | | -- [ ] **Video streams**: Temporal tensor sequences |
334 | | -- [ ] **Point cloud compression**: Octree and voxel grid representations |
335 | | -- [ ] **Custom message types**: Extension framework for user-defined conversions |
| 28 | +All conversion functions are in the `deep_ros::ros_conversions` namespace and require a `BackendMemoryAllocator` for tensor creation. |
336 | 29 |
|
337 | 30 | ## License |
338 | 31 |
|
| 32 | +Copyright (c) 2025-present WATonomous. All rights reserved. |
| 33 | + |
339 | 34 | Licensed under the Apache License, Version 2.0. |
0 commit comments