|
| 1 | +/* |
| 2 | + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. |
| 3 | + * SPDX-License-Identifier: Apache-2.0 |
| 4 | + */ |
| 5 | + |
| 6 | +#include "runtime/models/fast_foundation_stereo/native_plugins/plugins.h" |
| 7 | + |
| 8 | +#include <NvInferRuntime.h> |
| 9 | +#include <algorithm> |
| 10 | +#include <array> |
| 11 | +#include <cstddef> |
| 12 | +#include <cstdint> |
| 13 | +#include <cuda_runtime_api.h> |
| 14 | +#include <iostream> |
| 15 | +#include <stdexcept> |
| 16 | +#include <string> |
| 17 | +#include <vector> |
| 18 | + |
| 19 | +namespace { |
| 20 | + |
| 21 | +using FullVolumePlugin = trtmc::FastFoundationStereoFullVolumeLeakyPlugin; |
| 22 | +using Post8SumPlugin = trtmc::FastFoundationStereoPost8SumPlugin; |
| 23 | + |
| 24 | +constexpr std::size_t kPositions = static_cast<std::size_t>(FullVolumePlugin::kBatch) * |
| 25 | + FullVolumePlugin::kDisparities * FullVolumePlugin::kHeight * |
| 26 | + FullVolumePlugin::kWidth; |
| 27 | +constexpr std::size_t kPackedElements = kPositions * FullVolumePlugin::kChannelPitch; |
| 28 | +constexpr std::size_t kLinearElements = kPositions * FullVolumePlugin::kChannels; |
| 29 | +constexpr std::size_t kHalfBytes = sizeof(std::uint16_t); |
| 30 | +constexpr std::uint16_t kPositiveHalfBits = 0x3C3CU; |
| 31 | + |
| 32 | +static_assert(FullVolumePlugin::kBatch == Post8SumPlugin::kBatch); |
| 33 | +static_assert(FullVolumePlugin::kChannels == Post8SumPlugin::kChannels); |
| 34 | +static_assert(FullVolumePlugin::kDisparities == Post8SumPlugin::kDisparities); |
| 35 | +static_assert(FullVolumePlugin::kHeight == Post8SumPlugin::kHeight); |
| 36 | +static_assert(FullVolumePlugin::kWidth == Post8SumPlugin::kWidth); |
| 37 | +static_assert(FullVolumePlugin::kChannelPitch == Post8SumPlugin::kChannelPitch); |
| 38 | + |
| 39 | +void require(bool condition, const std::string& message) { |
| 40 | + if (!condition) |
| 41 | + throw std::runtime_error(message); |
| 42 | +} |
| 43 | + |
| 44 | +void require_cuda(cudaError_t result, const char* operation) { |
| 45 | + if (result != cudaSuccess) { |
| 46 | + throw std::runtime_error(std::string(operation) + ": " + cudaGetErrorString(result)); |
| 47 | + } |
| 48 | +} |
| 49 | + |
| 50 | +class DeviceBuffer { |
| 51 | + public: |
| 52 | + explicit DeviceBuffer(std::size_t bytes) { |
| 53 | + require_cuda(cudaMalloc(&pointer_, bytes), "cudaMalloc"); |
| 54 | + } |
| 55 | + |
| 56 | + ~DeviceBuffer() { |
| 57 | + if (pointer_ != nullptr) |
| 58 | + cudaFree(pointer_); |
| 59 | + } |
| 60 | + |
| 61 | + DeviceBuffer(DeviceBuffer const&) = delete; |
| 62 | + DeviceBuffer& operator=(DeviceBuffer const&) = delete; |
| 63 | + |
| 64 | + void* get() const noexcept { return pointer_; } |
| 65 | + |
| 66 | + private: |
| 67 | + void* pointer_{nullptr}; |
| 68 | +}; |
| 69 | + |
| 70 | +class Stream { |
| 71 | + public: |
| 72 | + Stream() { require_cuda(cudaStreamCreate(&stream_), "cudaStreamCreate"); } |
| 73 | + |
| 74 | + ~Stream() { |
| 75 | + if (stream_ != nullptr) |
| 76 | + cudaStreamDestroy(stream_); |
| 77 | + } |
| 78 | + |
| 79 | + Stream(Stream const&) = delete; |
| 80 | + Stream& operator=(Stream const&) = delete; |
| 81 | + |
| 82 | + cudaStream_t get() const noexcept { return stream_; } |
| 83 | + |
| 84 | + private: |
| 85 | + cudaStream_t stream_{nullptr}; |
| 86 | +}; |
| 87 | + |
| 88 | +nvinfer1::PluginFieldCollection empty_fields() { |
| 89 | + nvinfer1::PluginFieldCollection fields{}; |
| 90 | + fields.nbFields = 0; |
| 91 | + fields.fields = nullptr; |
| 92 | + return fields; |
| 93 | +} |
| 94 | + |
| 95 | +nvinfer1::PluginTensorDesc |
| 96 | +make_desc(int32_t channels, nvinfer1::DataType type = nvinfer1::DataType::kHALF, |
| 97 | + nvinfer1::TensorFormat format = nvinfer1::TensorFormat::kDHWC8) { |
| 98 | + nvinfer1::PluginTensorDesc desc{}; |
| 99 | + desc.dims.nbDims = 5; |
| 100 | + desc.dims.d[0] = FullVolumePlugin::kBatch; |
| 101 | + desc.dims.d[1] = channels; |
| 102 | + desc.dims.d[2] = FullVolumePlugin::kDisparities; |
| 103 | + desc.dims.d[3] = FullVolumePlugin::kHeight; |
| 104 | + desc.dims.d[4] = FullVolumePlugin::kWidth; |
| 105 | + desc.type = type; |
| 106 | + desc.format = format; |
| 107 | + return desc; |
| 108 | +} |
| 109 | + |
| 110 | +nvinfer1::DynamicPluginTensorDesc make_dynamic_desc(nvinfer1::PluginTensorDesc const& desc) { |
| 111 | + nvinfer1::DynamicPluginTensorDesc dynamic{}; |
| 112 | + dynamic.desc = desc; |
| 113 | + dynamic.min = desc.dims; |
| 114 | + dynamic.max = desc.dims; |
| 115 | + return dynamic; |
| 116 | +} |
| 117 | + |
| 118 | +void require_full_volume_rejects(FullVolumePlugin& plugin, |
| 119 | + nvinfer1::PluginTensorDesc const& invalid, |
| 120 | + nvinfer1::PluginTensorDesc const& valid, |
| 121 | + const std::string& case_name) { |
| 122 | + require(plugin.onShapeChange(&invalid, 1, &valid, 1) != 0, |
| 123 | + "full-volume plugin accepted invalid input " + case_name); |
| 124 | + require(plugin.onShapeChange(&valid, 1, &invalid, 1) != 0, |
| 125 | + "full-volume plugin accepted invalid output " + case_name); |
| 126 | +} |
| 127 | + |
| 128 | +void test_full_volume_runtime_descriptors() { |
| 129 | + auto fields = empty_fields(); |
| 130 | + FullVolumePlugin plugin(fields); |
| 131 | + auto logical = make_desc(FullVolumePlugin::kChannels); |
| 132 | + auto padded = make_desc(FullVolumePlugin::kChannelPitch); |
| 133 | + |
| 134 | + require(plugin.onShapeChange(&logical, 1, &logical, 1) == 0, |
| 135 | + "full-volume plugin rejected runtime C=28"); |
| 136 | + require(plugin.onShapeChange(&padded, 1, &padded, 1) == 0, |
| 137 | + "full-volume plugin rejected TensorRT runtime C=32 padding"); |
| 138 | + |
| 139 | + require_full_volume_rejects(plugin, make_desc(24), logical, "C=24"); |
| 140 | + require_full_volume_rejects(plugin, make_desc(40), logical, "C=40"); |
| 141 | + require_full_volume_rejects(plugin, |
| 142 | + make_desc(FullVolumePlugin::kChannels, nvinfer1::DataType::kFLOAT), |
| 143 | + logical, "dtype"); |
| 144 | + require_full_volume_rejects(plugin, |
| 145 | + make_desc(FullVolumePlugin::kChannels, nvinfer1::DataType::kHALF, |
| 146 | + nvinfer1::TensorFormat::kLINEAR), |
| 147 | + logical, "format"); |
| 148 | +} |
| 149 | + |
| 150 | +void test_full_volume_build_descriptors_remain_logical() { |
| 151 | + auto fields = empty_fields(); |
| 152 | + FullVolumePlugin plugin(fields); |
| 153 | + auto logical = make_dynamic_desc(make_desc(FullVolumePlugin::kChannels)); |
| 154 | + std::array<nvinfer1::DynamicPluginTensorDesc, 2> valid{logical, logical}; |
| 155 | + |
| 156 | + require(plugin.supportsFormatCombination(0, valid.data(), 1, 1), |
| 157 | + "full-volume plugin rejected build-time input C=28"); |
| 158 | + require(plugin.supportsFormatCombination(1, valid.data(), 1, 1), |
| 159 | + "full-volume plugin rejected build-time output C=28"); |
| 160 | + require(plugin.configurePlugin(valid.data(), 1, valid.data() + 1, 1) == 0, |
| 161 | + "full-volume plugin rejected build-time profile C=28"); |
| 162 | + |
| 163 | + auto padded = make_dynamic_desc(make_desc(FullVolumePlugin::kChannelPitch)); |
| 164 | + std::array<nvinfer1::DynamicPluginTensorDesc, 2> invalid_input{padded, logical}; |
| 165 | + std::array<nvinfer1::DynamicPluginTensorDesc, 2> invalid_output{logical, padded}; |
| 166 | + require(!plugin.supportsFormatCombination(0, invalid_input.data(), 1, 1), |
| 167 | + "full-volume plugin accepted build-time input C=32"); |
| 168 | + require(!plugin.supportsFormatCombination(1, invalid_output.data(), 1, 1), |
| 169 | + "full-volume plugin accepted build-time output C=32"); |
| 170 | + require(plugin.configurePlugin(invalid_input.data(), 1, invalid_input.data() + 1, 1) != 0, |
| 171 | + "full-volume plugin accepted build-time input profile C=32"); |
| 172 | + require(plugin.configurePlugin(invalid_output.data(), 1, invalid_output.data() + 1, 1) != 0, |
| 173 | + "full-volume plugin accepted build-time output profile C=32"); |
| 174 | +} |
| 175 | + |
| 176 | +int32_t post8_shape_status(Post8SumPlugin& plugin, nvinfer1::PluginTensorDesc const& linear, |
| 177 | + nvinfer1::PluginTensorDesc const& packed_input, |
| 178 | + nvinfer1::PluginTensorDesc const& output) { |
| 179 | + std::array<nvinfer1::PluginTensorDesc, 2> inputs{linear, packed_input}; |
| 180 | + return plugin.onShapeChange(inputs.data(), static_cast<int32_t>(inputs.size()), &output, 1); |
| 181 | +} |
| 182 | + |
| 183 | +void require_post8_rejects_packed(Post8SumPlugin& plugin, nvinfer1::PluginTensorDesc const& invalid, |
| 184 | + nvinfer1::PluginTensorDesc const& linear, |
| 185 | + nvinfer1::PluginTensorDesc const& valid, |
| 186 | + const std::string& case_name) { |
| 187 | + require(post8_shape_status(plugin, linear, invalid, valid) != 0, |
| 188 | + "post8-sum plugin accepted invalid packed input " + case_name); |
| 189 | + require(post8_shape_status(plugin, linear, valid, invalid) != 0, |
| 190 | + "post8-sum plugin accepted invalid output " + case_name); |
| 191 | +} |
| 192 | + |
| 193 | +void test_post8_runtime_descriptors() { |
| 194 | + auto fields = empty_fields(); |
| 195 | + Post8SumPlugin plugin(fields); |
| 196 | + auto linear = make_desc(Post8SumPlugin::kChannels, nvinfer1::DataType::kHALF, |
| 197 | + nvinfer1::TensorFormat::kLINEAR); |
| 198 | + auto logical = make_desc(Post8SumPlugin::kChannels); |
| 199 | + auto padded = make_desc(Post8SumPlugin::kChannelPitch); |
| 200 | + |
| 201 | + require(post8_shape_status(plugin, linear, logical, logical) == 0, |
| 202 | + "post8-sum plugin rejected runtime C=28"); |
| 203 | + require(post8_shape_status(plugin, linear, padded, padded) == 0, |
| 204 | + "post8-sum plugin rejected TensorRT runtime C=32 padding"); |
| 205 | + |
| 206 | + require_post8_rejects_packed(plugin, make_desc(24), linear, logical, "C=24"); |
| 207 | + require_post8_rejects_packed(plugin, make_desc(40), linear, logical, "C=40"); |
| 208 | + require_post8_rejects_packed(plugin, |
| 209 | + make_desc(Post8SumPlugin::kChannels, nvinfer1::DataType::kFLOAT), |
| 210 | + linear, logical, "dtype"); |
| 211 | + require_post8_rejects_packed(plugin, |
| 212 | + make_desc(Post8SumPlugin::kChannels, nvinfer1::DataType::kHALF, |
| 213 | + nvinfer1::TensorFormat::kLINEAR), |
| 214 | + linear, logical, "format"); |
| 215 | + |
| 216 | + auto wrong_linear_dtype = linear; |
| 217 | + wrong_linear_dtype.type = nvinfer1::DataType::kFLOAT; |
| 218 | + require(post8_shape_status(plugin, wrong_linear_dtype, logical, logical) != 0, |
| 219 | + "post8-sum plugin accepted invalid linear input dtype"); |
| 220 | + auto wrong_linear_format = linear; |
| 221 | + wrong_linear_format.format = nvinfer1::TensorFormat::kDHWC8; |
| 222 | + require(post8_shape_status(plugin, wrong_linear_format, logical, logical) != 0, |
| 223 | + "post8-sum plugin accepted invalid linear input format"); |
| 224 | +} |
| 225 | + |
| 226 | +void test_post8_build_descriptors_remain_logical() { |
| 227 | + auto fields = empty_fields(); |
| 228 | + Post8SumPlugin plugin(fields); |
| 229 | + auto linear = make_dynamic_desc(make_desc(Post8SumPlugin::kChannels, nvinfer1::DataType::kHALF, |
| 230 | + nvinfer1::TensorFormat::kLINEAR)); |
| 231 | + auto logical = make_dynamic_desc(make_desc(Post8SumPlugin::kChannels)); |
| 232 | + std::array<nvinfer1::DynamicPluginTensorDesc, 3> valid{linear, logical, logical}; |
| 233 | + |
| 234 | + for (int32_t position = 0; position < static_cast<int32_t>(valid.size()); ++position) { |
| 235 | + require(plugin.supportsFormatCombination(position, valid.data(), 2, 1), |
| 236 | + "post8-sum plugin rejected build-time C=28 descriptor"); |
| 237 | + } |
| 238 | + require(plugin.configurePlugin(valid.data(), 2, valid.data() + 2, 1) == 0, |
| 239 | + "post8-sum plugin rejected build-time profile C=28"); |
| 240 | + |
| 241 | + auto padded = make_dynamic_desc(make_desc(Post8SumPlugin::kChannelPitch)); |
| 242 | + std::array<nvinfer1::DynamicPluginTensorDesc, 3> invalid_input{linear, padded, logical}; |
| 243 | + std::array<nvinfer1::DynamicPluginTensorDesc, 3> invalid_output{linear, logical, padded}; |
| 244 | + require(!plugin.supportsFormatCombination(1, invalid_input.data(), 2, 1), |
| 245 | + "post8-sum plugin accepted build-time packed input C=32"); |
| 246 | + require(!plugin.supportsFormatCombination(2, invalid_output.data(), 2, 1), |
| 247 | + "post8-sum plugin accepted build-time output C=32"); |
| 248 | + require(plugin.configurePlugin(invalid_input.data(), 2, invalid_input.data() + 2, 1) != 0, |
| 249 | + "post8-sum plugin accepted build-time input profile C=32"); |
| 250 | + require(plugin.configurePlugin(invalid_output.data(), 2, invalid_output.data() + 2, 1) != 0, |
| 251 | + "post8-sum plugin accepted build-time output profile C=32"); |
| 252 | +} |
| 253 | + |
| 254 | +void require_padded_lanes_are_zero(void* device_output, const std::string& plugin_name) { |
| 255 | + constexpr std::size_t row_pitch = FullVolumePlugin::kChannelPitch * kHalfBytes; |
| 256 | + constexpr std::size_t padding_bytes = |
| 257 | + (FullVolumePlugin::kChannelPitch - FullVolumePlugin::kChannels) * kHalfBytes; |
| 258 | + constexpr std::size_t padding_lanes = |
| 259 | + FullVolumePlugin::kChannelPitch - FullVolumePlugin::kChannels; |
| 260 | + std::vector<std::uint16_t> padding(kPositions * padding_lanes); |
| 261 | + auto* first_padding_lane = |
| 262 | + static_cast<std::uint8_t*>(device_output) + FullVolumePlugin::kChannels * kHalfBytes; |
| 263 | + require_cuda(cudaMemcpy2D(padding.data(), padding_bytes, first_padding_lane, row_pitch, |
| 264 | + padding_bytes, kPositions, cudaMemcpyDeviceToHost), |
| 265 | + "cudaMemcpy2D padded lanes"); |
| 266 | + require( |
| 267 | + std::all_of(padding.begin(), padding.end(), [](std::uint16_t bits) { return bits == 0U; }), |
| 268 | + plugin_name + " left a non-zero padded lane"); |
| 269 | +} |
| 270 | + |
| 271 | +void test_full_volume_enqueue_zeros_padded_lanes() { |
| 272 | + auto fields = empty_fields(); |
| 273 | + FullVolumePlugin plugin(fields); |
| 274 | + auto logical = make_desc(FullVolumePlugin::kChannels); |
| 275 | + auto padded = make_desc(FullVolumePlugin::kChannelPitch); |
| 276 | + std::array<nvinfer1::PluginTensorDesc, 2> runtime_descs{logical, padded}; |
| 277 | + DeviceBuffer input(kPackedElements * kHalfBytes); |
| 278 | + DeviceBuffer output(kPackedElements * kHalfBytes); |
| 279 | + Stream stream; |
| 280 | + |
| 281 | + require_cuda(cudaMemset(input.get(), 0x3C, kPackedElements * kHalfBytes), |
| 282 | + "cudaMemset full-volume input"); |
| 283 | + void const* inputs[]{input.get()}; |
| 284 | + void* outputs[]{output.get()}; |
| 285 | + for (auto const& desc : runtime_descs) { |
| 286 | + std::string case_name = "runtime C=" + std::to_string(desc.dims.d[1]); |
| 287 | + require_cuda(cudaMemset(output.get(), 0x7F, kPackedElements * kHalfBytes), |
| 288 | + "cudaMemset full-volume output"); |
| 289 | + require(plugin.enqueue(&desc, &desc, inputs, outputs, nullptr, stream.get()) == 0, |
| 290 | + "full-volume plugin enqueue failed for " + case_name); |
| 291 | + require_cuda(cudaStreamSynchronize(stream.get()), "full-volume kernel synchronization"); |
| 292 | + |
| 293 | + std::uint16_t logical_output = 0U; |
| 294 | + require_cuda(cudaMemcpy(&logical_output, output.get(), sizeof(logical_output), |
| 295 | + cudaMemcpyDeviceToHost), |
| 296 | + "cudaMemcpy full-volume logical output"); |
| 297 | + require(logical_output == kPositiveHalfBits, |
| 298 | + "full-volume kernel did not preserve a positive logical lane for " + case_name); |
| 299 | + require_padded_lanes_are_zero(output.get(), "full-volume plugin " + case_name); |
| 300 | + } |
| 301 | +} |
| 302 | + |
| 303 | +void test_post8_enqueue_zeros_padded_lanes() { |
| 304 | + auto fields = empty_fields(); |
| 305 | + Post8SumPlugin plugin(fields); |
| 306 | + auto linear_desc = make_desc(Post8SumPlugin::kChannels, nvinfer1::DataType::kHALF, |
| 307 | + nvinfer1::TensorFormat::kLINEAR); |
| 308 | + auto logical_desc = make_desc(Post8SumPlugin::kChannels); |
| 309 | + auto padded_desc = make_desc(Post8SumPlugin::kChannelPitch); |
| 310 | + std::array<nvinfer1::PluginTensorDesc, 2> runtime_packed_descs{logical_desc, padded_desc}; |
| 311 | + DeviceBuffer linear(kLinearElements * kHalfBytes); |
| 312 | + DeviceBuffer skip(kPackedElements * kHalfBytes); |
| 313 | + DeviceBuffer output(kPackedElements * kHalfBytes); |
| 314 | + Stream stream; |
| 315 | + |
| 316 | + require_cuda(cudaMemset(linear.get(), 0x3C, kLinearElements * kHalfBytes), |
| 317 | + "cudaMemset post8 linear input"); |
| 318 | + require_cuda(cudaMemset(skip.get(), 0x3C, kPackedElements * kHalfBytes), |
| 319 | + "cudaMemset post8 packed input"); |
| 320 | + void const* inputs[]{linear.get(), skip.get()}; |
| 321 | + void* outputs[]{output.get()}; |
| 322 | + for (auto const& packed_desc : runtime_packed_descs) { |
| 323 | + std::array<nvinfer1::PluginTensorDesc, 2> input_descs{linear_desc, packed_desc}; |
| 324 | + std::string case_name = "runtime C=" + std::to_string(packed_desc.dims.d[1]); |
| 325 | + require_cuda(cudaMemset(output.get(), 0x7F, kPackedElements * kHalfBytes), |
| 326 | + "cudaMemset post8 output"); |
| 327 | + require(plugin.enqueue(input_descs.data(), &packed_desc, inputs, outputs, nullptr, |
| 328 | + stream.get()) == 0, |
| 329 | + "post8-sum plugin enqueue failed for " + case_name); |
| 330 | + require_cuda(cudaStreamSynchronize(stream.get()), "post8-sum kernel synchronization"); |
| 331 | + |
| 332 | + std::uint16_t logical_output = 0U; |
| 333 | + require_cuda(cudaMemcpy(&logical_output, output.get(), sizeof(logical_output), |
| 334 | + cudaMemcpyDeviceToHost), |
| 335 | + "cudaMemcpy post8 logical output"); |
| 336 | + require(logical_output != 0U, |
| 337 | + "post8-sum kernel did not write a logical lane for " + case_name); |
| 338 | + require_padded_lanes_are_zero(output.get(), "post8-sum plugin " + case_name); |
| 339 | + } |
| 340 | +} |
| 341 | + |
| 342 | +bool gpu_available() { |
| 343 | + int32_t device_count = 0; |
| 344 | + cudaError_t result = cudaGetDeviceCount(&device_count); |
| 345 | + if (result == cudaSuccess) { |
| 346 | + if (device_count > 0) |
| 347 | + return true; |
| 348 | + std::cerr << "SKIP: no CUDA device available\n"; |
| 349 | + return false; |
| 350 | + } |
| 351 | + if (result == cudaErrorNoDevice) { |
| 352 | + cudaGetLastError(); |
| 353 | + std::cerr << "SKIP: no CUDA device available\n"; |
| 354 | + return false; |
| 355 | + } |
| 356 | + require_cuda(result, "cudaGetDeviceCount"); |
| 357 | + return false; // Unreachable, keeps all compiler control-flow analyses satisfied. |
| 358 | +} |
| 359 | + |
| 360 | +} // namespace |
| 361 | + |
| 362 | +int main() { |
| 363 | + try { |
| 364 | + test_full_volume_runtime_descriptors(); |
| 365 | + test_full_volume_build_descriptors_remain_logical(); |
| 366 | + test_post8_runtime_descriptors(); |
| 367 | + test_post8_build_descriptors_remain_logical(); |
| 368 | + if (!gpu_available()) |
| 369 | + return 0; |
| 370 | + test_full_volume_enqueue_zeros_padded_lanes(); |
| 371 | + test_post8_enqueue_zeros_padded_lanes(); |
| 372 | + } catch (const std::exception& error) { |
| 373 | + std::cerr << "FAIL: " << error.what() << '\n'; |
| 374 | + return 1; |
| 375 | + } |
| 376 | + return 0; |
| 377 | +} |
0 commit comments