diff --git a/autoware_ml/configs/experiments/detection3d/bevfusion/base.yaml b/autoware_ml/configs/experiments/detection3d/bevfusion/base.yaml new file mode 100644 index 00000000..ae5d495c --- /dev/null +++ b/autoware_ml/configs/experiments/detection3d/bevfusion/base.yaml @@ -0,0 +1,205 @@ +# @package _global_ +defaults: + - /experiments/default_experiment + - /optimizers@optimizer: default_adamw + - /metrics/t4dataset@metrics: default_detection3d_metrics + - _self_ + +database: ??? +point_cloud_range: ??? +voxel_size: ??? +metric_ranges: ??? +out_size_factor: ??? +score_threshold: 0.0 +max_time_lag: ??? +max_num_points: 32 +max_voxels: 120000 +# Typical j6gen2 val frames occupy 120k-136k voxels; the reference evaluates with a +# 160k budget so evaluation must not silently truncate at the training budget. +eval_max_voxels: 160000 + +optimizer: + lr: 2.0e-4 + +data_preprocessor: + preprocessor_modules: + - _target_: autoware_ml.preprocessing.detection3d.point_pillar_preprocessor.PointPillarPreprocessor + voxel_size: ${voxel_size} + point_cloud_range: ${point_cloud_range} + max_num_points: ${max_num_points} + max_voxels: ${max_voxels} + eval_max_voxels: ${eval_max_voxels} + # BEVFusion's sparse middle encoder (and the deployment runtime's Point2Voxel) + # consume (z, y, x) voxel coordinates. + voxelization_z_order_first: true + +trainer: + max_epochs: ??? + check_val_every_n_epoch: ??? + gradient_clip_val: 35.0 + gradient_clip_algorithm: norm + +deploy: + onnx: + dynamo: false + opset_version: 17 + do_constant_folding: false + # Deployed BEVFusion runs FP16 outside the quantized (Q/DQ) regions; AutoCast + # bakes that into the exported dense graph (engines build strongly typed). + precision: fp16 + # Stage names come from BEVFusionLidarDetectionModel.build_stages() — the AWML + # split-deployment ABI: bevfusion_sparse (external libspconv ONNX; torch fallback + # on onnx/tensorrt backends until its exporter lands) and bevfusion_dense. + stages: + bevfusion_sparse: + onnx: + # The runtime voxelizes each frame independently, so the voxel count is a + # runtime property and must stay dynamic in the artifact. + dynamic_axes: + voxels: { 0: voxels_num } + coors: { 0: voxels_num } + num_points_per_voxel: { 0: voxels_num } + # The sparse stage's TensorRT profile also depends on the voxel budget and on + # the rulebook inputs the spconv plugin adds, so it is configured per variant. + bevfusion_dense: + onnx: + # Single-sample static graph: lidar_bev in, packed detections out. + dynamic_axes: {} + # lidar_bev's TensorRT profile depends on the middle encoder's + # dense_output_shapes and is therefore configured per experiment variant. + # TensorRT build, verification, and evaluation are heavyweight and stay opt-in here; + # the release/ab experiment variants enable them explicitly. + tensorrt: + enabled: false + # The sparse graph's autoware::ImplicitGemm / GetIndicePairsImplicitGemm nodes are + # plugin operators; TensorRT must load their library before parsing the ONNX. The + # image builds it (docker/tensorrt_plugins). + plugin_libraries: [/opt/plugins/libautoware_tensorrt_plugins.so] + verification: + # Raw positional comparison is meaningless for BEVFusion's packed outputs: the + # dense graph selects top-500 proposals, and the zero-padded heatmap borders + # produce mass ties, so backends legitimately pick different near-zero-score + # proposals (measured 2026-09-01: high-score proposals align to 0.038 while the + # positional bbox_pred diff is 159). Cross-backend correctness is gated by + # deploy.evaluation (mAP equality across backends) instead. + enabled: false + num_verify_batches: 1 + scenarios: [] + evaluation: + enabled: false + num_samples: -1 + num_warmup: 2 + backends: + pytorch: { enabled: true, device: cuda } + onnx: { enabled: true, device: cuda } + tensorrt: { enabled: true, device: cuda } + +model: + _target_: autoware_ml.models.detection3d.main_modules.bevfusion.BEVFusionLidarDetectionModel + metrics: ${metrics} + log_dict_configs: + _target_: autoware_ml.models.multi_task_base_model.LogDictConfigs + _convert_: all + on_step: true + on_epoch: true + prog_bar: true + sync_dist: true + + pts_voxel_encoder: + _target_: autoware_ml.models.detection3d.encoders.voxel.HardSimpleVoxelSinCosEncoder + in_channels: 5 + # Normalization ranges for (x, y, z, intensity, time_lag). + min_norm_values: + - ${point_cloud_range.0} + - ${point_cloud_range.1} + - ${point_cloud_range.2} + - 0.0 + - 0.0 + max_norm_values: + - ${point_cloud_range.3} + - ${point_cloud_range.4} + - ${point_cloud_range.5} + - 255.0 + - ${max_time_lag} + pts_middle_encoder: + _target_: autoware_ml.models.detection3d.encoders.sparse.SparseEncoder + in_channels: 50 + sparse_shape: ??? + output_channels: 128 + dense_output_shapes: ??? + # Pair-mask sorting in the deployed graph: locality only, identical detections, so + # this is a per-target latency choice. Measured here (100 frames, TensorRT): sorting + # 6.70 ms vs not sorting 7.13 ms per frame, so it stays on. Re-measure on the vehicle + # target before trusting either value. + export_do_sort: true + pts_backbone: + _target_: autoware_ml.models.detection3d.backbones.second.SECONDBackbone + in_channels: 256 + out_channels: [128, 256] + layer_nums: [5, 5] + layer_strides: [1, 2] + pts_neck: + _target_: autoware_ml.models.detection3d.necks.second_fpn.SECONDFPN + in_channels: [128, 256] + out_channels: [256, 256] + upsample_strides: [1, 2] + bbox_head: + _target_: autoware_ml.models.detection3d.heads.transfusion.TransFusionHead + num_proposals: ??? + auxiliary: true + in_channels: 512 + hidden_channel: 128 + num_classes: ??? + class_names: ${database.database_task_configs.detection3d.label_names} + num_decoder_layers: 1 + num_heads: 8 + feedforward_channels: 256 + common_heads: + center: [2, 2] + height: [1, 2] + dim: [3, 2] + rot: [2, 2] + vel: [2, 2] + bbox_coder: + _target_: autoware_ml.models.detection3d.task_modules.bbox_coders.TransFusionBBoxCoder + pc_range: + - ${point_cloud_range.0} + - ${point_cloud_range.1} + voxel_size: + - ${voxel_size.0} + - ${voxel_size.1} + out_size_factor: ${out_size_factor} + post_center_range: ??? + score_threshold: ${score_threshold} + code_size: 10 + assigner: + _target_: autoware_ml.models.detection3d.task_modules.assigners.HungarianAssigner3D + cls_cost: + _target_: autoware_ml.models.detection3d.task_modules.match_costs.ClassificationCost + weight: 0.15 + reg_cost: + _target_: autoware_ml.models.detection3d.task_modules.match_costs.BBoxBEVL1Cost + weight: 0.25 + iou_cost: + _target_: autoware_ml.models.detection3d.task_modules.match_costs.IoU3DCost + weight: 0.25 + point_cloud_range: ${point_cloud_range} + voxel_size: + - ${voxel_size.0} + - ${voxel_size.1} + out_size_factor: ${out_size_factor} + code_weights: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.2, 0.2] + min_radius: 2 + gaussian_overlap: 0.1 + score_threshold: ${score_threshold} + post_max_size: 300 + nms_min_radius: ??? + heatmap_target: oriented + optimizer: ${optimizer} + scheduler: + _target_: autoware_ml.utils.schedulers.cyclic_cosine_annealing.CyclicCosineAnnealingLR + _partial_: true + warmup_epochs: 3 + decay_epochs: 27 + max_lr_factor: 10.0 + min_lr_factor: 0.0001 diff --git a/autoware_ml/configs/experiments/detection3d/bevfusion/lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_j6gen2_base.yaml b/autoware_ml/configs/experiments/detection3d/bevfusion/lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_j6gen2_base.yaml new file mode 100644 index 00000000..3db35038 --- /dev/null +++ b/autoware_ml/configs/experiments/detection3d/bevfusion/lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_j6gen2_base.yaml @@ -0,0 +1,213 @@ +# @package _global_ +defaults: + - base + - /database@database: t4dataset/t4dataset_j6gen2_base + - /datamodule@datamodule: t4dataset/default_detection3d_datamodule + - /metrics/t4dataset@metric_ranges: default_120m_metric_ranges + - _self_ + +batch_size: 16 +num_workers: 16 +max_num_3d_gt_bboxes: 500 + +experiment_group_name: bevfusion/${database.version}/j6gen2_base +experiment_name: lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m +point_cloud_range: [-122.4, -122.4, -3.0, 122.4, 122.4, 5.0] +voxel_size: [0.17, 0.17, 0.2] +out_size_factor: 8 +max_time_lag: 0.2 + +eval_class_range: + car: 121.0 + truck: 121.0 + bus: 121.0 + bicycle: 121.0 + pedestrian: 121.0 + traffic_cone: 121.0 + barrier: 121.0 + +trainer: + max_epochs: 30 + check_val_every_n_epoch: 5 + +deploy: + stages: + bevfusion_sparse: + tensorrt: + input_shapes: + # Voxel count varies per frame; the max matches the evaluation pillar budget + # (eval_max_voxels), which is the largest the runtime can hand the engine. + voxels: + min_shape: [1000, 32, 5] + opt_shape: [80000, 32, 5] + max_shape: [160000, 32, 5] + coors: + min_shape: [1000, 3] + opt_shape: [80000, 3] + max_shape: [160000, 3] + num_points_per_voxel: + min_shape: [1000] + opt_shape: [80000] + max_shape: [160000] + bevfusion_dense: + tensorrt: + input_shapes: + # Static BEV canvas: (batch, output_channels * dense_z, *dense_output_shapes[:2]). + lidar_bev: + min_shape: [1, 256, 180, 180] + opt_shape: [1, 256, 180, 180] + max_shape: [1, 256, 180, 180] + +model: + pts_middle_encoder: + sparse_shape: [1440, 1440, 41] + dense_output_shapes: [180, 180, 2] + bbox_head: + num_proposals: 500 + num_classes: 7 + # Export emits fusion-ready attention (no max-subtraction before softmax) so + # TensorRT's Myelin fuses the decoder attention into one MHA kernel — the max-sub + # pattern blocks the fusion (measured 0.68 vs 0.25 ms). Same graph shape as the AWML + # 2.8 deployment (plain matmul-softmax-matmul); training is untouched (export copy). + fuse_export_attention: true + dense_heatmap_pooling_classes: [car, truck, bus, barrier] + # Reference-trained head geometry (mmdet3d SeparateHead head_conv=64, PyTorch + # default BN eps/momentum) — required to load the parity-verified checkpoint. + head_hidden_channels: 64 + norm_eps: 1.0e-5 + norm_momentum: 0.1 + # Reference form: single biased Conv2d, signed output (no BN+ReLU clipping). + # TODO(vividf): remove once BEVFusion is retrained natively (AWML reference + # checkpoint no longer needed). + shared_conv_norm_act: false + nms_type: circle + # mmdet3d's circle_nms 0.25 is a SQUARED distance -> 0.5 m Euclidean radius here. + nms_min_radius: 0.5 + nms_groups: + - class_names: [car, truck, bus] + nms_radius: 0.5 + post_max_size: 300 + - class_names: [bicycle] + nms_radius: 0.0 + post_max_size: 50 + - class_names: [pedestrian] + nms_radius: 0.0 + post_max_size: 100 + - class_names: [traffic_cone] + nms_radius: 0.0 + post_max_size: 100 + - class_names: [barrier] + nms_radius: 0.0 + post_max_size: 50 + bbox_coder: + post_center_range: [-200.0, -200.0, -10.0, 200.0, 200.0, 10.0] + score_threshold: [0.015, 0.010, 0.010, 0.020, 0.030, 0.040, 0.020] + +datamodule: + # Overwrite dataloader + train_dataloader: + shuffle: true + predict_dataloader: + shuffle: false + batch_size: 1 + + # Overwrite transforms + train_dataset: + max_num_3d_gt_bboxes: ${max_num_3d_gt_bboxes} + + transforms: + _target_: autoware_ml.transforms.multi_task.base.MultiTaskTransformsCompose + _convert_: all + + pipeline: + - _target_: autoware_ml.transforms.multi_task.point_cloud.loading.LoadPointsFromFile + _convert_: all + + load_dim: ${database.lidar_pointcloud_num_features} + use_dim: [0, 1, 2, 3] # Refer PointFieldIndex for the name of each dimensionality, [x, y, z, intensity] + bev_remove_radius: 0.0 + + - _target_: autoware_ml.transforms.multi_task.point_cloud.loading.LoadMultiSweepPointsFromFile + _convert_: all + + sweeps_num: 2 + bev_remove_radius: 1.0 + test_mode: false + use_timestamp_difference: true + load_dim: ${database.lidar_pointcloud_num_features} + use_dim: [0, 1, 2, 3] # Refer PointFieldIndex for the name of each dimensionality, [x, y, z, intensity] + + - _target_: autoware_ml.transforms.multi_task.point_cloud.geometry.GlobalRotScaleTrans + _convert_: all + + # In radians, the range of rotation angles for random rotation around the Z-axis. + # The range is specified as [min_angle, max_angle]. + yaw_rot_range: [-0.78539816, 0.78539816] + # The range of scale factors for random scaling. + scale_ratio_range: [0.95, 1.05] + # The standard deviation for Gaussian translation noise. + translation_std: [0.5, 0.5, 0.2] + + - _target_: autoware_ml.transforms.multi_task.point_cloud.geometry.GlobalBEVRandomFlip + _convert_: all + + horizontal_flip_ratio: 0.5 + vertical_flip_ratio: 0.5 + + - _target_: autoware_ml.transforms.multi_task.point_cloud.geometry.PointsRangeFilter + _convert_: all + + points_range: ${point_cloud_range} + + - _target_: autoware_ml.transforms.multi_task.point_cloud.geometry.PointsRandomShuffle + _convert_: all + + - _target_: autoware_ml.transforms.multi_task.bboxes_3d.loading.BBoxesLabelNameFilter + _convert_: all + + label_names_to_keep: ${database.database_task_configs.detection3d.label_names} + + - _target_: autoware_ml.transforms.multi_task.bboxes_3d.geometry.BBoxesBEVDistanceFilter + _convert_: all + + bev_range: + - ${point_cloud_range.0} + - ${point_cloud_range.1} + - ${point_cloud_range.3} + - ${point_cloud_range.4} + + # Order matters, it first filters bboxes within the default meters with less than 2 points. + # Then it filters bboxes within 60 meters with less than 3 points + - _target_: autoware_ml.transforms.multi_task.bboxes_3d.geometry.BBoxesMinPointsFilter + _convert_: all + + min_points: 2 + bev_range: + - ${point_cloud_range.0} + - ${point_cloud_range.1} + - ${point_cloud_range.3} + - ${point_cloud_range.4} + + - _target_: autoware_ml.transforms.multi_task.bboxes_3d.geometry.BBoxesMinPointsFilter + _convert_: all + + min_points: 3 + bev_range: [-60.0, -60.0, 60.0, 60.0] + + - _target_: autoware_ml.transforms.multi_task.bboxes_3d.loading.BBoxesAttributeFilter + _convert_: all + + attributes_to_filter: + bicycle: + - vehicle_state.parked + - cycle_state.without_rider + - motorcycle_state.without_rider + + test_dataset: + max_num_3d_gt_bboxes: ${max_num_3d_gt_bboxes} + + validation_dataset: + max_num_3d_gt_bboxes: ${max_num_3d_gt_bboxes} + + predict_dataset: + max_num_3d_gt_bboxes: ${max_num_3d_gt_bboxes} diff --git a/autoware_ml/configs/experiments/detection3d/bevfusion/lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_j6gen2_base_fp8.yaml b/autoware_ml/configs/experiments/detection3d/bevfusion/lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_j6gen2_base_fp8.yaml new file mode 100644 index 00000000..cc0a94e9 --- /dev/null +++ b/autoware_ml/configs/experiments/detection3d/bevfusion/lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_j6gen2_base_fp8.yaml @@ -0,0 +1,51 @@ +# @package _global_ +# FP8 (E4M3 Q/DQ) variant: decoder FFN Linear layers only, everything else FP16. +# +# Positioning is accuracy, not latency (see the PTv3 FP8 experiment: INT8 PTQ cost +# 6.4 mIoU on linears, FP8 PTQ held within 0.4). This isolates the FP8-linear effect on +# a detection head: every conv subtree is skipped, so the only quantized modules are +# the decoder FFN's two Linears — the head's attention projections are structurally +# unreachable at calibration time (packed nn.MultiheadAttention parameters; the +# export-form q/k/v/out_proj Linears exist only after prepare_for_export). +# +# autoware-ml quantize \ +# --config-name experiments/detection3d/bevfusion/lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_j6gen2_base_fp8 \ +# --weights +# autoware-ml deploy \ +# --config-name experiments/detection3d/bevfusion/lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_j6gen2_base_fp8 \ +# --weights +defaults: + - lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_j6gen2_base + - _self_ + +experiment_name: lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_fp8 + +deploy: + tensorrt: + enabled: true + evaluation: + enabled: true + backends: + # ONNX Runtime has no implementation of modelopt's FP8 spelling (trt-domain + # TRT_FP8QuantizeLinear custom ops) and fails to even load the graph. + onnx: { enabled: false, device: cuda } + +quantization: + enabled: true + mode: ptq + fuse_bn: true + # The FFN linears are pinned fp8 by the model rules regardless; the default only + # governs the conv kinds, all of which this experiment skips. + default_precision: fp8 + skip_quantize: + - pts_backbone + - pts_neck + - bbox_head.shared_conv + - bbox_head.heatmap_head + # Same calibration set as the INT8 reference recipe; FP8 amax comes from the max + # calibrator (E4M3 convention), not histogram+MSE. + ptq: + calibrate_samples: 400 + batch_size: 1 + calib_seed: 0 + calib_shuffle: false diff --git a/autoware_ml/configs/experiments/detection3d/bevfusion/lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_j6gen2_base_int8.yaml b/autoware_ml/configs/experiments/detection3d/bevfusion/lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_j6gen2_base_int8.yaml new file mode 100644 index 00000000..490f42cc --- /dev/null +++ b/autoware_ml/configs/experiments/detection3d/bevfusion/lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_j6gen2_base_int8.yaml @@ -0,0 +1,56 @@ +# @package _global_ +# INT8 (Q/DQ explicit quantization) variant of the j6gen2_base BEVFusion-lidar experiment. +# +# Produce the quantized checkpoint, then deploy it (export + evaluate). The checkpoint +# describes its own quantization, so deploy/test read no quantization config: +# +# autoware-ml quantize \ +# --config-name experiments/detection3d/bevfusion/lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_j6gen2_base_int8 \ +# --weights +# autoware-ml deploy \ +# --config-name experiments/detection3d/bevfusion/lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_j6gen2_base_int8 \ +# --weights +# +# Only the dense graph quantizes (backbone / neck / head Conv2d); the sparse stage's +# INT8 form is the libspconv engine from the dedicated exporter (module TODO in +# main_modules/bevfusion/stages.py). Engines build strongly typed; the non-quantized +# graph regions inherit deploy.onnx.precision: fp16 from base.yaml (AutoCast). +defaults: + - lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_j6gen2_base + - _self_ + +experiment_name: lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_int8 + +# The INT8 variants exist to produce deployment artifacts and numbers: build the +# engine and evaluate by default. Verification stays disabled — BEVFusion's packed +# outputs are proposal-selection variant (see base.yaml); mAP equality is the gate. +deploy: + tensorrt: + enabled: true + evaluation: + enabled: true + +quantization: + enabled: true + mode: ptq + fuse_bn: true + default_precision: int8 + # Keep the validated conv-only INT8 tree: the model rules also pin the decoder FFN + # linears to FP8 (see the *_fp8 experiment); this recipe measures INT8 convolutions. + # The two input-side blocks stay FP16: the per-block sensitivity sweep (2026-09-04, + # work_dirs/reviews/quant-sensitivity-bevfusion.md) charges them -2.2 / -1.8 mAP alone + # while every other block is within +-0.1; with both skipped the 100-frame TRT mAP is + # 0.4513 (FP16 0.4512, all-conv INT8 0.4217) and the dense stage runs 1.60 ms + # (FP16 1.77, all-conv INT8 1.44). + skip_quantize: + - bbox_head.decoder + - pts_neck.blocks.0 + - pts_backbone.blocks.0 + # Reference recipe: 400 samples @ batch_size=1, seed 0, histogram + MSE amax. + # `calibration: {method: mse|entropy|percentile|max|smoothquant}` picks the amax + # algorithm (default mse); see CalibrationConfig. + ptq: + calibrate_samples: 400 + batch_size: 1 + calib_seed: 0 + calib_shuffle: false diff --git a/autoware_ml/configs/experiments/detection3d/bevfusion/lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_j6gen2_base_int8_qat.yaml b/autoware_ml/configs/experiments/detection3d/bevfusion/lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_j6gen2_base_int8_qat.yaml new file mode 100644 index 00000000..0d060ab1 --- /dev/null +++ b/autoware_ml/configs/experiments/detection3d/bevfusion/lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_j6gen2_base_int8_qat.yaml @@ -0,0 +1,37 @@ +# @package _global_ +# QAT variant: frozen-amax STE fine-tuning on top of the INT8 BEVFusion-lidar experiment. +# +# autoware-ml quantize \ +# --config-name experiments/detection3d/bevfusion/lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_j6gen2_base_int8_qat \ +# --weights +# autoware-ml deploy \ +# --config-name experiments/detection3d/bevfusion/lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_j6gen2_base_int8_qat \ +# --weights +# +# A QAT checkpoint deploys exactly like a PTQ one — the loader never branches on mode. +# Recipe (Wu et al. 2020 / modelopt): epochs ≈ 10% of original training, cosine schedule +# with peak lr = 1% of the original peak (2e-4 -> 2e-6), frozen skip_quantize layers, +# validation 4x per epoch, 400 calibration samples (calibrated at epoch 0 on the val split). +defaults: + - lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_j6gen2_base_int8 + - _self_ + +experiment_name: lidar_voxel0170_second_secfpn_b16_30e_t4dataset_120m_int8_qat + +quantization: + mode: qat + # Drop the inherited producer block for the other mode (a ptq block under + # mode=qat is a config lie and is rejected). + ptq: null + qat: + epochs: 3 + # Schedule PEAK ≈ 1% of the original training peak lr (2e-4 -> 2e-6, Wu et al. 2020); + # higher peaks degrade a converged model within the epoch. + lr: 2.0e-6 + # cosine from the peak down to 1% (no warmup). See QATScheduleConfig for one_cycle. + schedule: cosine + # skip_quantize layers take no STE gradient masking and would drift past the frozen + # downstream amax; QAT quality also peaks mid-epoch, so validate often for best.ckpt. + freeze_unquantized: true + val_check_interval: 0.25 + calibrate_samples: 400 diff --git a/autoware_ml/models/detection3d/main_modules/bevfusion/__init__.py b/autoware_ml/models/detection3d/main_modules/bevfusion/__init__.py new file mode 100644 index 00000000..0f8ffef4 --- /dev/null +++ b/autoware_ml/models/detection3d/main_modules/bevfusion/__init__.py @@ -0,0 +1,24 @@ +# Copyright 2026 TIER IV, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""BEVFusion (lidar-only): model (:mod:`.model`) and deployment stages (:mod:`.stages`). +Everything BEVFusion-specific lives in this directory; quantization rules land with +the PTQ milestone as :mod:`.quantization`. +""" + +from autoware_ml.models.detection3d.main_modules.bevfusion.model import ( + BEVFusionLidarDetectionModel, +) + +__all__ = ["BEVFusionLidarDetectionModel"] diff --git a/autoware_ml/models/detection3d/main_modules/bevfusion/model.py b/autoware_ml/models/detection3d/main_modules/bevfusion/model.py new file mode 100644 index 00000000..8ebc8bbb --- /dev/null +++ b/autoware_ml/models/detection3d/main_modules/bevfusion/model.py @@ -0,0 +1,253 @@ +# Copyright 2026 TIER IV, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native BEVFusion lidar detector (multi-task interface). + +The multi-task wrapper around the AWML-parity-verified BEVFusion components +(``models/detection3d/bevfusion.py`` holds the legacy single-task wrapper until +Q5). The numerics live in the reused submodules and ``TransFusionHead``; this +module only adapts between the framework's typed containers and the head's dict +API — the head itself is untouched to preserve the verified parity. + +Coordinate contract: ``voxels_data.coords`` arrives in ``(z, y, x)`` order (set +``voxelization_z_order_first: true`` on the PointPillarPreprocessor) — the same +layout the deployment runtime uses — and the batch column is prepended here. + +Deployment (stage graph) lives in :mod:`.stages`; the checkpoint is the converted +native ``best_epoch_25_autoware_ml_native.pth`` (see the BEVFusion parity notes). +""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from types import MappingProxyType +from typing import Any, Mapping + +from jaxtyping import Float32 +import torch +from torch.optim import Optimizer +from torch.optim.lr_scheduler import LRScheduler + +from autoware_ml.dataclasses.detection3d.head_outputs import ( + Detection3DHeadOutputs, + TransFusionHeadOutputs, +) +from autoware_ml.dataclasses.detection3d.predictions import Detection3DSamplePredictions +from autoware_ml.dataclasses.multi_task_batch_inputs import MultiTaskBatchInputs +from autoware_ml.dataclasses.multi_task_outputs import MultiTaskOutputs +from autoware_ml.dataclasses.multi_task_predictions import MultiTaskPredictions +from autoware_ml.deployment.stages import Stage +from autoware_ml.metrics.base import MetricSuite +from autoware_ml.metrics.detection3d.eval_output import multi_task_eval_output +from autoware_ml.models.detection3d.feature_extractors import LidarBEVFeatureExtractor +from autoware_ml.models.detection3d.main_modules.bevfusion.quantization import ( + build_bevfusion_quantization_plan, +) +from autoware_ml.models.detection3d.main_modules.bevfusion.stages import ( + build_bevfusion_lidar_stages, + decode_packed_detections, +) +from autoware_ml.quantization.config import QuantizationConfig +from autoware_ml.quantization.plan import QuantizationPlan +from autoware_ml.models.multi_task_base_model import LogDictConfigs, MultiTaskBaseModel +from autoware_ml.preprocessing.data_preprocessor import DataPreprocessor + +_HEAD_DICT_KEYS = ( + "center", + "height", + "dim", + "rot", + "vel", + "heatmap", + "dense_heatmap", + "query_heatmap_score", + "query_labels", +) + + +def head_dict_to_outputs(outputs: Mapping[str, torch.Tensor]) -> TransFusionHeadOutputs: + """Wrap the TransFusion head's output dict into the typed container.""" + return TransFusionHeadOutputs(**{key: outputs.get(key) for key in _HEAD_DICT_KEYS}) + + +def _as_predictions(samples: Sequence[Mapping[str, torch.Tensor]]) -> MultiTaskPredictions: + """Wrap the head's per-sample detection dicts into the typed predictions container.""" + return MultiTaskPredictions( + detection3d_predictions=[ + Detection3DSamplePredictions( + bboxes_3d=sample["bboxes_3d"], + scores_3d=sample["scores_3d"], + labels_3d=sample["labels_3d"], + ) + for sample in samples + ] + ) + + +def outputs_to_head_dict(outputs: TransFusionHeadOutputs) -> dict[str, torch.Tensor]: + """Unwrap the typed container back into the dict the head's loss/predict expect.""" + head_dict = {key: getattr(outputs, key) for key in _HEAD_DICT_KEYS} + if head_dict["vel"] is None: + del head_dict["vel"] + return head_dict + + +class BEVFusionLidarDetectionModel(MultiTaskBaseModel): + """Compose a lidar-only BEVFusion detector from the parity-verified modules.""" + + verification_caveat = ( + "the dense graph packs its top-500 proposals, and the zero-padded heatmap " + "borders produce mass score ties, so backends legitimately select different " + "near-zero-score proposals — positional raw-output comparison is meaningless " + "(measured 2026-09-01: high-score proposals align to 0.038 while the " + "positional bbox_pred diff is 159)." + ) + + def __init__( + self, + data_preprocessor: DataPreprocessor, + pts_voxel_encoder: torch.nn.Module, + pts_middle_encoder: torch.nn.Module, + pts_backbone: torch.nn.Module, + pts_neck: torch.nn.Module, + bbox_head: torch.nn.Module, + log_dict_configs: LogDictConfigs, + optimizer: Callable[..., Optimizer] | None = None, + scheduler: Callable[[Optimizer], LRScheduler] | None = None, + scheduler_config: Mapping[str, Any] | None = None, + metrics: Sequence[MetricSuite] | None = None, + ) -> None: + """Initialize the lidar-only BEVFusion detector. + + Args: + data_preprocessor: Multi-task data preprocessor (hard voxelization with + ``voxelization_z_order_first: true``). + pts_voxel_encoder: Lidar voxel feature encoder (HardSimpleVFE). + pts_middle_encoder: Sparse (spconv) middle encoder producing the BEV map. + pts_backbone: BEV backbone. + pts_neck: BEV neck. + bbox_head: TransFusion detection head (dict API, kept untouched). + log_dict_configs: Logging configuration for training and validation. + optimizer: Optimizer factory. + scheduler: Scheduler factory. + scheduler_config: Lightning scheduler metadata. + metrics: Detection metrics accumulated during validation and test. + """ + super().__init__( + data_preprocessor=data_preprocessor, + optimizer=optimizer, + scheduler=scheduler, + scheduler_config=scheduler_config, + metrics=metrics, + log_dict_configs=log_dict_configs, + ) + self.pts_voxel_encoder = pts_voxel_encoder + self.pts_middle_encoder = pts_middle_encoder + self.pts_backbone = pts_backbone + self.pts_neck = pts_neck + self.bbox_head = bbox_head + # The exact VFE+spconv composition the legacy wrapper (and the runtime) uses. + self.lidar_feature_extractor = LidarBEVFeatureExtractor( + pts_voxel_encoder=pts_voxel_encoder, + pts_middle_encoder=pts_middle_encoder, + pts_backbone=None, + pts_neck=None, + ) + + def forward(self, multi_task_batch_inputs: MultiTaskBatchInputs) -> MultiTaskOutputs: + """Run the detector on voxelized lidar inputs.""" + voxels_data = multi_task_batch_inputs.voxels_data + if voxels_data is None: + raise ValueError( + "MultiTaskBatchInputs must contain voxels_data for BEVFusion forward pass." + ) + batch_size = multi_task_batch_inputs.multi_task_gt_batch.infer_batch_size() + # coords arrive as (z, y, x); the sparse encoder consumes (batch, z, y, x). + voxel_coords = torch.cat((voxels_data.batch_indices.view(-1, 1), voxels_data.coords), dim=1) + bev_features = self.lidar_feature_extractor( + voxels_data.voxels, voxels_data.num_points, voxel_coords, batch_size=batch_size + ) + bev_features = self.pts_neck(self.pts_backbone(bev_features)) + head_dict = self.bbox_head(bev_features) + return MultiTaskOutputs( + detection3d_head_outputs=Detection3DHeadOutputs( + center_head_outputs=None, + transfusion_head_outputs=head_dict_to_outputs(head_dict), + ) + ) + + def compute_metrics( + self, multi_task_batch_inputs: MultiTaskBatchInputs, multi_task_outputs: MultiTaskOutputs + ) -> MappingProxyType[str, Float32[torch.Tensor, " 1"]]: + """Compute BEVFusion training losses through the head's per-sample-list API.""" + gt_batch = multi_task_batch_inputs.multi_task_gt_batch.detection3d_gt_batch + if gt_batch is None: + raise ValueError( + "MultiTaskBatchInputs must contain detection3d_gt_batch for BEVFusion." + ) + head_outputs = self._transfusion_outputs(multi_task_outputs) + # The head's loss takes per-sample lists; slice the padded batch by valid counts. + gt_boxes = [ + boxes[:count] for boxes, count in zip(gt_batch.gt_bboxes_3d, gt_batch.gt_valid_bboxes) + ] + gt_labels = [ + labels[:count].long() + for labels, count in zip(gt_batch.gt_labels_3d, gt_batch.gt_valid_bboxes) + ] + return MappingProxyType( + dict(self.bbox_head.loss(outputs_to_head_dict(head_outputs), gt_boxes, gt_labels)) + ) + + def decode_outputs(self, outputs: MultiTaskOutputs) -> MultiTaskPredictions: + """Decode predictions through the head's dict API into the typed container.""" + head_outputs = self._transfusion_outputs(outputs) + return _as_predictions(self.bbox_head.predict(outputs_to_head_dict(head_outputs))) + + def build_eval_output_from_predictions( + self, batch: MultiTaskBatchInputs, predictions: MultiTaskPredictions + ) -> dict[str, Any]: + """Pair decoded detections with ground truth for the metric suites.""" + return multi_task_eval_output( + multi_task_predictions=predictions, multi_task_batch_inputs=batch + ) + + @staticmethod + def _transfusion_outputs(outputs: MultiTaskOutputs) -> TransFusionHeadOutputs: + head_outputs = outputs.detection3d_head_outputs + if head_outputs is None or head_outputs.transfusion_head_outputs is None: + raise ValueError( + "MultiTaskOutputs must contain transfusion_head_outputs for BEVFusion." + ) + return head_outputs.transfusion_head_outputs + + # ------------------------------------------------------------------ deployment hooks + + def build_quantization_plan(self, quantization_config: QuantizationConfig) -> QuantizationPlan: + """Bind BEVFusion's quantization rules to a parsed config (see :mod:`.quantization`).""" + return build_bevfusion_quantization_plan(quantization_config) + + def build_stages(self) -> tuple[Stage, ...]: + """Declare the BEVFusion lidar split stage graph (see :mod:`.stages`).""" + return build_bevfusion_lidar_stages(self) + + def assemble_predictions(self, outputs: Mapping[str, torch.Tensor]) -> MultiTaskPredictions: + """Decode the deployed graph's packed tensors the way the runtime does. + + The dense graph performs the proposal selection itself, so a backend returns + detections rather than head outputs — there is no :meth:`assemble_outputs` step + for this model. The unpacking is BEVFusion's runtime ABI; the post-processing + that follows is the head's own :meth:`decode_detections`, so the deployed + behaviour cannot drift from the model's. + """ + return _as_predictions(decode_packed_detections(self.bbox_head, outputs)) diff --git a/autoware_ml/models/detection3d/main_modules/bevfusion/quantization.py b/autoware_ml/models/detection3d/main_modules/bevfusion/quantization.py new file mode 100644 index 00000000..5ec30558 --- /dev/null +++ b/autoware_ml/models/detection3d/main_modules/bevfusion/quantization.py @@ -0,0 +1,66 @@ +# Copyright 2026 TIER IV, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""BEVFusion (lidar-only) quantization declaration. + +Quantization covers the dense deployment graph only: ``pts_backbone`` / +``pts_neck`` (Conv2d) and the head's Conv2d layers (shared conv, heatmap head), +plus the decoder FFN's Linear layers — pinned FP8, never INT8 (INT8 linears cost +PTv3 6 mIoU for nothing; E4M3 held accuracy). The sparse side +(``pts_voxel_encoder`` / ``pts_middle_encoder``) is deliberately absent — its +INT8 form is the libspconv engine produced by the dedicated sparse exporter, +not Q/DQ replacement. + +The attention projections are *structurally* out of reach at calibration time: +the trained head holds ``nn.MultiheadAttention`` (packed ``in_proj_weight`` +Parameter — not a module — and an ``out_proj`` whose forward the fast path +bypasses; the walker refuses it), and the export-form ``q/k/v/out_proj`` +Linears only come into existence in ``prepare_for_export``, after calibration. +Quantizing them means swapping to the export-form attention *before* +calibration — the deferred attention-recipe infrastructure. + +Every stage — quantize (PTQ / QAT) and deploy-load — reaches this declaration +through ``BEVFusionLidarDetectionModel.build_quantization_plan``, so the same +plan builds the same tree everywhere. NOTE: adding the ``linear`` kind changed +the placement record; quantized checkpoints produced before 2026-09-03 need a +re-run of ``quantize`` (experimental ckpts carry no format versioning by +design). +""" + +from __future__ import annotations + +from autoware_ml.quantization.config import QuantizationConfig +from autoware_ml.quantization.plan import QuantizationPlan, QuantRules + +#: BEVFusion lidar quantization declaration (dense graph only; see module docstring). +BEVFUSION_LIDAR_QUANT_RULES = QuantRules( + quantize_submodules={ + "pts_backbone": ("conv",), + "pts_neck": ("conv",), + "bbox_head": {"conv": None, "linear": "fp8"}, + }, +) + + +def build_bevfusion_quantization_plan(config: QuantizationConfig) -> QuantizationPlan: + """Bind BEVFusion's quantization rules to a parsed config. + + Args: + config: Parsed ``quantization`` config block. + + Returns: + The :class:`~autoware_ml.quantization.plan.QuantizationPlan` shared by + the quantize stage (PTQ / QAT) and the deploy loader. + """ + return QuantizationPlan(rules=BEVFUSION_LIDAR_QUANT_RULES, config=config) diff --git a/autoware_ml/models/detection3d/main_modules/bevfusion/stages.py b/autoware_ml/models/detection3d/main_modules/bevfusion/stages.py new file mode 100644 index 00000000..ec45f8c1 --- /dev/null +++ b/autoware_ml/models/detection3d/main_modules/bevfusion/stages.py @@ -0,0 +1,248 @@ +# Copyright 2026 TIER IV, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""BEVFusion (lidar-only) deployment stage graph. + +Split form (the INT8 deployment line, mirroring AWML's +``bevfusion_split_int8_deployment`` artifacts): + + fetch_voxels (torch) -> bevfusion_sparse (graph) -> bevfusion_dense (graph) + +Runtime ABI carried over from the AWML split artifacts: + +- ``bevfusion_sparse``: ``voxels`` / ``coors`` / ``num_points_per_voxel`` in, + ``lidar_bev`` out. Exported through :meth:`SparseEncoder.prepare_for_export`, + which swaps the native spconv layers for the wrappers in + :mod:`autoware_ml.ops.spconv`; their symbolics emit the runtime's libspconv + ABI (``autoware::GetIndicePairsImplicitGemm`` / ``autoware::ImplicitGemm`` with + the rulebook tensors as graph inputs). TensorRT executes those nodes through + ``libautoware_tensorrt_plugins.so``, which the image builds and every deploy + config lists in ``deploy.tensorrt.plugin_libraries``; ONNX Runtime has no + implementation for them, so only that backend falls back to PyTorch here. +- ``bevfusion_dense``: ``lidar_bev`` in; ``bbox_pred`` / ``score`` / ``label_pred`` + out — the AWML dense graph DECODES in-graph (unlike CenterPoint's raw-map ABI), + so the wrapper ends at the head's export decode. + +Contract with the interface migration: + +- **Submodules**: ``pts_voxel_encoder``, ``pts_middle_encoder`` (spconv), + ``pts_backbone``, ``pts_neck``, ``bbox_head``. +- **Batch inputs**: ``MultiTaskBatchInputs.voxels_data`` provides voxel features, + coordinates and per-voxel point counts (the ``_first_sample_voxel_inputs`` + tensors of the legacy export). +- **Backend evaluation decode**: because the graph decodes in-graph, a backend returns + detections rather than head outputs, so the model implements ``assemble_predictions`` + (not ``assemble_outputs``) and reaches it through :func:`decode_packed_detections`. + +.. todo:: TODO(vividf): INT8 for the sparse stage needs the quantized libspconv ABI + (``ImplicitGemmInt8`` with per-layer ``*_channel_scale`` / ``*_bias_scaled`` + inputs) plus its own plugin; the current quantization declaration deliberately + covers the dense graph only. +""" + +from __future__ import annotations + +from typing import Any, Mapping + +import torch +import torch.nn.functional as F +from torch import nn + +from autoware_ml.deployment.stages import GraphStage, Stage, StageContext, TorchStage +from autoware_ml.models.detection3d.feature_extractors import LidarBEVFeatureExtractor +from autoware_ml.deployment.onnx.autocast import keep_topk_in_fp16 +from autoware_ml.ops.spconv.onnx_fusion import fuse_sparse_graph +from autoware_ml.types.backend import Backend + +# Stage / artifact names (AWML split-deployment ABI: .onnx / .engine). +FETCH_VOXELS_STAGE = "fetch_voxels" +SPARSE_STAGE = "bevfusion_sparse" +DENSE_STAGE = "bevfusion_dense" + +# Context tensor names — the ONNX input/output names (AWML runtime ABI). +VOXELS = "voxels" +COORS = "coors" +NUM_POINTS_PER_VOXEL = "num_points_per_voxel" +LIDAR_BEV = "lidar_bev" +BBOX_PRED = "bbox_pred" +SCORE = "score" +LABEL_PRED = "label_pred" + +# ONNX output name -> the key ``decode_packed_detections`` reads it under. This graph +# emits detections, not head outputs, so the names simply carry through. +OUTPUT_FIELDS: tuple[tuple[str, str], ...] = ( + (BBOX_PRED, "bbox_pred"), + (SCORE, "score"), + (LABEL_PRED, "label_pred"), +) + + +class BEVFusionSparseExportWrapper(nn.Module): + """Voxel inputs -> dense lidar BEV features (VFE + spconv middle encoder). + + Single-sample graph in the runtime layout: ``coors`` is ``(z, y, x)`` without a + batch column (the runtime voxelizes with spconv's Point2Voxel), so a zero batch + column is prepended — the same adaptation the legacy ``_forward_export`` does. + """ + + def __init__(self, voxel_encoder: nn.Module, middle_encoder: nn.Module) -> None: + super().__init__() + # Export-ready deep copy: native spconv layers swapped for the wrappers in + # autoware_ml.ops.spconv, whose symbolics emit the runtime's + # autoware::GetIndicePairsImplicitGemm / autoware::ImplicitGemm nodes. + self.extractor = LidarBEVFeatureExtractor( + pts_voxel_encoder=voxel_encoder, + pts_middle_encoder=( + middle_encoder.prepare_for_export() + if hasattr(middle_encoder, "prepare_for_export") + else middle_encoder + ), + pts_backbone=None, + pts_neck=None, + ) + + def forward( + self, + voxels: torch.Tensor, + coors: torch.Tensor, + num_points_per_voxel: torch.Tensor, + ) -> torch.Tensor: + batch_column = torch.zeros((coors.shape[0], 1), dtype=coors.dtype, device=coors.device) + voxel_coords = torch.cat((batch_column, coors), dim=1) + return self.extractor(voxels, num_points_per_voxel, voxel_coords, batch_size=1) + + +class BEVFusionDenseExportWrapper(nn.Module): + """Lidar BEV features -> packed runtime detections (backbone + neck + head). + + The output packing mirrors the legacy ``_export_detection_outputs`` (the + runtime ABI): raw regression channels concatenated into ``bbox_pred`` plus the + fused ``score`` and ``label_pred`` — the runtime decodes and NMS-filters + itself, so no metric-space decoding happens in the graph. + """ + + def __init__(self, backbone: nn.Module, neck: nn.Module, bbox_head: nn.Module) -> None: + super().__init__() + self.backbone = backbone + self.neck = neck + # Export-ready deep copy: decoder attention swapped for the exportable + # equivalent (torch.onnx cannot trace nn.MultiheadAttention faithfully). + self.bbox_head = ( + bbox_head.prepare_for_export() + if hasattr(bbox_head, "prepare_for_export") + else bbox_head + ) + + def forward(self, lidar_bev: torch.Tensor) -> tuple[torch.Tensor, ...]: + outputs = self.bbox_head(self.neck(self.backbone(lidar_bev))) + num_proposals = self.bbox_head.num_proposals + query_labels = outputs["query_labels"] + heatmap = outputs["heatmap"][..., -num_proposals:].sigmoid() + one_hot = ( + F.one_hot(query_labels, num_classes=self.bbox_head.num_classes) + .permute(0, 2, 1) + .to(heatmap.dtype) + ) + score = (heatmap * outputs["query_heatmap_score"] * one_hot)[0].max(dim=0).values + if outputs.get("vel") is None: + raise ValueError("BEVFusion export requires a velocity branch in the detection head.") + bbox_pred = torch.cat( + [ + outputs[key][0, :, -num_proposals:] + for key in ("center", "height", "dim", "rot", "vel") + ], + dim=0, + ) + return bbox_pred, score, query_labels[0] + + +def build_bevfusion_lidar_stages(model: Any) -> tuple[Stage, ...]: + """Declare the lidar-only BEVFusion split stage graph over ``model``'s submodules.""" + + def fetch_voxels(context: StageContext) -> Mapping[str, torch.Tensor]: + voxels_data = context.batch_inputs.voxels_data + if voxels_data is None: + raise ValueError("MultiTaskBatchInputs must contain voxels_data for BEVFusion.") + # Single-sample export graph: keep the first sample's voxels only. + first_sample = voxels_data.batch_indices == 0 + return { + VOXELS: voxels_data.voxels[first_sample], + COORS: voxels_data.coords[first_sample].int().contiguous(), + NUM_POINTS_PER_VOXEL: voxels_data.num_points[first_sample].int(), + } + + return ( + TorchStage(FETCH_VOXELS_STAGE, run=fetch_voxels), + GraphStage( + SPARSE_STAGE, + module=BEVFusionSparseExportWrapper(model.pts_voxel_encoder, model.pts_middle_encoder), + inputs=(VOXELS, COORS, NUM_POINTS_PER_VOXEL), + outputs=(LIDAR_BEV,), + # The exported graph carries autoware::GetIndicePairsImplicitGemm / + # autoware::ImplicitGemm custom ops. TensorRT executes them through + # libautoware_tensorrt_plugins.so (deploy.tensorrt.plugin_libraries); ONNX + # Runtime has no implementation at all, so only that backend falls back. + torch_fallback_backends=(Backend.ONNX,), + # TensorRT cannot fuse a standard operator into a plugin node, so the traced + # bias adds and block ReLUs are folded into the plugin's own bias input and + # act_type instead. + onnx_transforms=(fuse_sparse_graph,), + ), + GraphStage( + DENSE_STAGE, + module=BEVFusionDenseExportWrapper(model.pts_backbone, model.pts_neck, model.bbox_head), + inputs=(LIDAR_BEV,), + outputs=(BBOX_PRED, SCORE, LABEL_PRED), + output_fields=OUTPUT_FIELDS, + # The proposal TopK ranks FP16 scores directly instead of an FP32 copy of the + # whole flattened heatmap (measured 0.81 -> 0.45 ms). Near-ties may reorder, + # which this model already declares (verification_caveat); the gate is mAP. + onnx_transforms=(keep_topk_in_fp16,), + ), + ) + + +def decode_packed_detections( + bbox_head: Any, outputs: Mapping[str, torch.Tensor] +) -> list[dict[str, torch.Tensor]]: + """Turn the deployed graph's packed tensors into the head's detection dicts. + + Only the unpacking is deployment-specific. The graph already fused the per-proposal + score and picked the winning label, so the class scores are re-scattered into the + per-class layout the head's post-processing expects, and that post-processing — + metric-space decoding, score and range filtering, NMS — is the head's own + :meth:`TransFusionHead.decode_detections`, not a copy of it. + + Args: + bbox_head: The model's detection head, providing the post-processing. + outputs: Field name -> tensor for the final stage (``bbox_pred`` / ``score`` / + ``label_pred``), single sample. + + Returns: + One ``{bboxes_3d, scores_3d, labels_3d}`` dict, matching the head's own return. + """ + bbox_pred = outputs[BBOX_PRED] + scores = outputs[SCORE] + labels = outputs[LABEL_PRED].long() + num_proposals = scores.shape[0] + score_matrix = bbox_pred.new_zeros((1, bbox_head.num_classes, num_proposals)) + score_matrix[0, labels, torch.arange(num_proposals, device=bbox_pred.device)] = scores + return bbox_head.decode_detections( + score_matrix, + bbox_pred[6:8].unsqueeze(0), + bbox_pred[3:6].unsqueeze(0), + bbox_pred[0:2].unsqueeze(0), + bbox_pred[2:3].unsqueeze(0), + bbox_pred[8:10].unsqueeze(0), + ) diff --git a/autoware_ml/tests/deployment/test_bevfusion_stages.py b/autoware_ml/tests/deployment/test_bevfusion_stages.py new file mode 100644 index 00000000..3b9d002e --- /dev/null +++ b/autoware_ml/tests/deployment/test_bevfusion_stages.py @@ -0,0 +1,276 @@ +# Copyright 2026 TIER IV, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""BEVFusion split stage graph: declaration validity, ABI names, the per-stage +fallback semantics, and the packed-output decode's agreement with the head.""" + +from __future__ import annotations + +from pathlib import Path +import types +from types import SimpleNamespace + +from omegaconf import OmegaConf +import torch +from torch import nn + +from autoware_ml.deployment.config import DeployConfig +from autoware_ml.deployment.export import available_backends +from autoware_ml.deployment.pipeline import StagedPipeline, _ModuleRunner +from autoware_ml.deployment.stages import GraphStage, TorchStage, validate_stages +from autoware_ml.models.detection3d.main_modules.bevfusion.stages import ( + DENSE_STAGE, + LIDAR_BEV, + SPARSE_STAGE, + build_bevfusion_lidar_stages, + decode_packed_detections, +) +from autoware_ml.models.detection3d.heads.transfusion import TransFusionHead +from autoware_ml.models.detection3d.task_modules.bbox_coders import TransFusionBBoxCoder +from autoware_ml.types.backend import Backend + + +def _stub_model() -> SimpleNamespace: + return SimpleNamespace( + pts_voxel_encoder=nn.Identity(), + pts_middle_encoder=nn.Identity(), + pts_backbone=nn.Identity(), + pts_neck=nn.Identity(), + bbox_head=nn.Identity(), + ) + + +def test_declaration_is_valid_with_the_awml_split_abi() -> None: + stages = validate_stages(build_bevfusion_lidar_stages(_stub_model())) + sparse, dense = stages[1], stages[2] + assert sparse.name == SPARSE_STAGE and dense.name == DENSE_STAGE + assert sparse.inputs == ("voxels", "coors", "num_points_per_voxel") + assert sparse.outputs == (LIDAR_BEV,) and dense.inputs == (LIDAR_BEV,) + assert dense.outputs == ("bbox_pred", "score", "label_pred") + # TensorRT executes the sparse graph's plugin ops (deploy.tensorrt.plugin_libraries); + # ONNX Runtime has no implementation for them, so only that backend falls back. + assert sparse.torch_fallback_backends == (Backend.ONNX,) + + +def _fallback_test_stages() -> tuple: + """A minimal two-graph declaration exercising the fallback fields.""" + + def seed(context): + return {"x": torch.ones(1, 2)} + + sparse_like = GraphStage( + "sparse_like", + module=nn.Identity(), + inputs=("x",), + outputs=("mid",), + torch_fallback_backends=(Backend.ONNX,), + ) + dense_like = GraphStage( + "dense_like", + module=nn.Identity(), + inputs=("mid",), + outputs=("y",), + output_fields=(("y", "y"),), + ) + return (TorchStage("seed", run=seed), sparse_like, dense_like) + + +def test_fallback_stage_uses_the_torch_module_on_its_fallback_backend(tmp_path) -> None: + import onnx + from onnx import TensorProto, helper + + # Only the dense-like stage needs an ONNX artifact on the onnx backend. + x = helper.make_tensor_value_info("mid", TensorProto.FLOAT, [1, 2]) + y = helper.make_tensor_value_info("y", TensorProto.FLOAT, [1, 2]) + graph = helper.make_graph( + [helper.make_node("Identity", ["mid"], ["y"])], "dense_like", [x], [y] + ) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)]) + onnx.save(model, str(tmp_path / "dense_like.onnx")) + + pipeline = StagedPipeline( + _fallback_test_stages(), + backend=Backend.ONNX, + device=torch.device("cpu"), + artifacts_dir=tmp_path, + ) + assert isinstance(pipeline._runners["sparse_like"], _ModuleRunner) + assert not isinstance(pipeline._runners["dense_like"], _ModuleRunner) + + +def test_available_backends_exempts_fallback_stages_from_artifacts(tmp_path) -> None: + import onnx + from onnx import TensorProto, helper + + x = helper.make_tensor_value_info("mid", TensorProto.FLOAT, [1, 2]) + y = helper.make_tensor_value_info("y", TensorProto.FLOAT, [1, 2]) + graph = helper.make_graph( + [helper.make_node("Identity", ["mid"], ["y"])], "dense_like", [x], [y] + ) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)]) + onnx.save(model, str(tmp_path / "dense_like.onnx")) + + available = available_backends(_fallback_test_stages(), tmp_path) + # onnx is available without sparse_like.onnx (fallback); tensorrt is not + # (no engines, and sparse_like has no tensorrt fallback). + assert Backend.ONNX in available and Backend.TENSORRT not in available + + +def test_export_skips_engines_for_tensorrt_fallback_stages(tmp_path, monkeypatch) -> None: + """A stage TensorRT cannot execute must not have an engine built for it. + + The sparse stage exports an ONNX full of runtime plugin ops; building an engine + from it fails (the plugin is not registered) and the pipeline would never use it, + because the stage runs in PyTorch on the tensorrt backend. + """ + from autoware_ml.deployment import export as export_module + + built: list[str] = [] + monkeypatch.setattr( + export_module, + "build_engine", + lambda onnx_path, engine_path, **kwargs: built.append(Path(onnx_path).stem), + ) + + plugin_stage = GraphStage( + "plugin_like", + module=nn.Identity(), + inputs=("x",), + outputs=("y",), + torch_fallback_backends=(Backend.ONNX, Backend.TENSORRT), + ) + plain_stage = GraphStage( + "plain_like", + module=nn.Identity(), + inputs=("y",), + outputs=("z",), + output_fields=(("z", "z"),), + ) + + def seed(context): + return {"x": torch.ones(1, 2)} + + deploy_cfg = DeployConfig.from_dict( + OmegaConf.create( + { + "onnx": {"enabled": True, "dynamo": False, "opset_version": 17}, + "tensorrt": {"enabled": True}, + "stages": {}, + } + ) + ) + export_module.export_stages( + (TorchStage("seed", run=seed), plugin_stage, plain_stage), + batch_inputs=None, + deploy_cfg=deploy_cfg, + output_dir=tmp_path, + device=torch.device("cpu"), + ) + + assert built == ["plain_like"] + # The ONNX is still written for the fallback stage: it is the deployed artifact. + assert (tmp_path / "plugin_like.onnx").exists() + + +def _coder() -> TransFusionBBoxCoder: + return TransFusionBBoxCoder( + pc_range=[-10.0, -10.0], + out_size_factor=2, + voxel_size=[0.5, 0.5], + post_center_range=[-100.0, -100.0, -100.0, 100.0, 100.0, 100.0], + score_threshold=0.1, + code_size=10, + ) + + +def _packed_channels() -> torch.Tensor: + """Two proposals in the runtime's packed channel layout; the second scores low.""" + return torch.tensor( + [ + [4.0, 8.0], # center x (grid) + [6.0, 2.0], # center y (grid) + [1.0, 0.0], # height + [0.0, 0.0], # dim log l + [0.0, 0.0], # dim log w + [0.0, 0.0], # dim log h + [1.0, 0.0], # rot sin + [0.0, 1.0], # rot cos + [0.5, 0.0], # vel x + [0.25, 0.0], # vel y + ] + ) + + +def test_packed_decode_applies_coder_math_and_score_filter() -> None: + head = SimpleNamespace( + num_classes=3, + bbox_coder=_coder(), + nms_type=None, + ) + # Borrow the head's real post-processing rather than restating it here. + head.decode_detections = types.MethodType(TransFusionHead.decode_detections, head) + outputs = { + "bbox_pred": _packed_channels(), + "score": torch.tensor([0.9, 0.05]), + "label_pred": torch.tensor([1.0, 2.0]), + } + + detections = decode_packed_detections(head, outputs) + + assert len(detections) == 1 + sample = detections[0] + assert sample["scores_3d"].tolist() == [torch.tensor(0.9).item()] + assert sample["labels_3d"].tolist() == [1] + box = sample["bboxes_3d"][0] + assert box[0].item() == 4.0 * 2 * 0.5 - 10.0 # metric x + assert box[1].item() == 6.0 * 2 * 0.5 - 10.0 # metric y + assert abs(box[2].item() - 0.5) < 1e-6 # height - h/2 (dim exp(0)=1) + assert abs(box[6].item() - torch.atan2(torch.tensor(1.0), torch.tensor(0.0)).item()) < 1e-6 + assert abs(box[7].item() - 0.5) < 1e-6 and abs(box[8].item() - 0.25) < 1e-6 + + +def test_packed_decode_matches_the_head_on_the_same_proposals() -> None: + """The deployed path and the PyTorch path must produce the same detections. + + The graph fuses the per-proposal score and picks the winning label before the + framework sees it, so the two paths start from different tensors and can only be + compared by construction: feed the head raw maps whose fusion yields exactly the + packed score/label the graph would have emitted. + """ + head = SimpleNamespace(num_classes=3, bbox_coder=_coder(), nms_type=None) + head.decode_detections = types.MethodType(TransFusionHead.decode_detections, head) + channels = _packed_channels() + scores = torch.tensor([0.9, 0.4]) + labels = torch.tensor([1, 2]) + + # PyTorch path: a per-class score matrix carrying the same winning scores. + score_matrix = torch.zeros((1, 3, 2)) + score_matrix[0, labels, torch.arange(2)] = scores + from_head = head.decode_detections( + score_matrix, + channels[6:8].unsqueeze(0), + channels[3:6].unsqueeze(0), + channels[0:2].unsqueeze(0), + channels[2:3].unsqueeze(0), + channels[8:10].unsqueeze(0), + )[0] + + # Deployed path: the packed tensors the graph emits. + from_packed = decode_packed_detections( + head, + {"bbox_pred": channels, "score": scores, "label_pred": labels.float()}, + )[0] + + for key in ("bboxes_3d", "scores_3d", "labels_3d"): + assert torch.equal(from_head[key], from_packed[key]), key diff --git a/autoware_ml/tests/deployment/test_sparse_graph_contract.py b/autoware_ml/tests/deployment/test_sparse_graph_contract.py new file mode 100644 index 00000000..d1222c93 --- /dev/null +++ b/autoware_ml/tests/deployment/test_sparse_graph_contract.py @@ -0,0 +1,103 @@ +# Copyright 2026 TIER IV, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The exported sparse graph's contract with the runtime plugin. + +Every attribute here is read by ``libautoware_tensorrt_plugins.so`` at engine-build +time, and the plugin's tolerance for a missing one is temporary: it logs a "legacy +ONNX ... will be removed later" warning and assumes a default. A silent regression on +either side therefore surfaces as a vehicle-side failure, so the emitted attribute +names are pinned here rather than left to be noticed in a TensorRT log. + +Tracks spconv 2.3.6 (``spconv-cu120`` in pyproject) and the plugin sources in +autoware_universe ``perception/autoware_tensorrt_plugins``. +""" + +from __future__ import annotations + +import inspect + +from autoware_ml.ops.spconv import sparse_functional + + +def _emitted_attributes(symbolic) -> set[str]: + """Attribute names a symbolic passes to ``g.op`` (``name_i=`` / ``_f=`` / ``_s=``).""" + source = inspect.getsource(symbolic) + body = source[source.index("g.op(") :] + return { + line.split("=")[0].strip().rsplit("_", 1)[0] + for line in body.splitlines() + if "=" in line and line.strip().split("=")[0].strip().endswith(("_i", "_f", "_s")) + } + + +def test_get_indice_pairs_implicit_gemm_emits_the_plugin_field_set() -> None: + """The 12-field form; without ``do_sort`` the plugin takes its deprecated path.""" + expected = { + "batch_size", + "spatial_shape", + "algo", + "ksize", + "stride", + "padding", + "dilation", + "out_padding", + "subm", + "transpose", + "is_train", + "do_sort", + } + assert _emitted_attributes(sparse_functional.GetIndicePairsImplicitGemm.symbolic) == expected + + +def test_implicit_gemm_emits_the_plugin_field_set() -> None: + """The 7-field form; ``act_type`` is what the post-export ReLU fusion sets.""" + expected = { + "is_train", + "is_subm", + "fp32_accum", + "act_alpha", + "act_beta", + "output_scale", + "output_add_scale", + "act_type", + } + assert _emitted_attributes(sparse_functional.ImplicitGemm.symbolic) == expected + + +def test_the_sparse_stage_wires_the_bias_activation_fusion() -> None: + """The fusion is only useful if the stage declaration actually runs it.""" + from torch import nn + + from autoware_ml.models.detection3d.main_modules.bevfusion.stages import ( + SPARSE_STAGE, + build_bevfusion_lidar_stages, + ) + from autoware_ml.ops.spconv.onnx_fusion import fuse_sparse_graph + + model = type( + "Stub", + (), + { + "pts_voxel_encoder": nn.Identity(), + "pts_middle_encoder": nn.Identity(), + "pts_backbone": nn.Identity(), + "pts_neck": nn.Identity(), + "bbox_head": nn.Identity(), + }, + )() + sparse = next( + stage for stage in build_bevfusion_lidar_stages(model) if stage.name == SPARSE_STAGE + ) + assert fuse_sparse_graph in sparse.onnx_transforms