models: BEVFusion onboarding (stage graph, sparse plugin export, INT8/FP8)

autoware_ml/configs/experiments/detection3d/bevfusion/base.yaml

@@ -0,0 +1,205 @@
+# @package _global_
+defaults:
+  - /experiments/default_experiment
+  - /optimizers@optimizer: default_adamw
+  - /metrics/t4dataset@metrics: default_detection3d_metrics
+  - _self_
+
+database: ???
+point_cloud_range: ???
+voxel_size: ???
+metric_ranges: ???
+out_size_factor: ???
+score_threshold: 0.0
+max_time_lag: ???
+max_num_points: 32
+max_voxels: 120000
+# Typical j6gen2 val frames occupy 120k-136k voxels; the reference evaluates with a
+# 160k budget so evaluation must not silently truncate at the training budget.
+eval_max_voxels: 160000
+
+optimizer:
+  lr: 2.0e-4
+
+data_preprocessor:
+  preprocessor_modules:
+    - _target_: autoware_ml.preprocessing.detection3d.point_pillar_preprocessor.PointPillarPreprocessor
+      voxel_size: ${voxel_size}
+      point_cloud_range: ${point_cloud_range}
+      max_num_points: ${max_num_points}
+      max_voxels: ${max_voxels}
+      eval_max_voxels: ${eval_max_voxels}
+      # BEVFusion's sparse middle encoder (and the deployment runtime's Point2Voxel)
+      # consume (z, y, x) voxel coordinates.
+      voxelization_z_order_first: true
+
+trainer:
+  max_epochs: ???
+  check_val_every_n_epoch: ???
+  gradient_clip_val: 35.0
+  gradient_clip_algorithm: norm
+
+deploy:
+  onnx:
+    dynamo: false
+    opset_version: 17
+    do_constant_folding: false
+    # Deployed BEVFusion runs FP16 outside the quantized (Q/DQ) regions; AutoCast
+    # bakes that into the exported dense graph (engines build strongly typed).
+    precision: fp16
+  # Stage names come from BEVFusionLidarDetectionModel.build_stages() — the AWML
+  # split-deployment ABI: bevfusion_sparse (external libspconv ONNX; torch fallback
+  # on onnx/tensorrt backends until its exporter lands) and bevfusion_dense.
+  stages:
+    bevfusion_sparse:
+      onnx:
+        # The runtime voxelizes each frame independently, so the voxel count is a
+        # runtime property and must stay dynamic in the artifact.
+        dynamic_axes:
+          voxels: { 0: voxels_num }
+          coors: { 0: voxels_num }
+          num_points_per_voxel: { 0: voxels_num }
+      # The sparse stage's TensorRT profile also depends on the voxel budget and on
+      # the rulebook inputs the spconv plugin adds, so it is configured per variant.
+    bevfusion_dense:
+      onnx:
+        # Single-sample static graph: lidar_bev in, packed detections out.
+        dynamic_axes: {}
+      # lidar_bev's TensorRT profile depends on the middle encoder's
+      # dense_output_shapes and is therefore configured per experiment variant.
+  # TensorRT build, verification, and evaluation are heavyweight and stay opt-in here;
+  # the release/ab experiment variants enable them explicitly.
+  tensorrt:
+    enabled: false
+    # The sparse graph's autoware::ImplicitGemm / GetIndicePairsImplicitGemm nodes are
+    # plugin operators; TensorRT must load their library before parsing the ONNX. The
+    # image builds it (docker/tensorrt_plugins).
+    plugin_libraries: [/opt/plugins/libautoware_tensorrt_plugins.so]
+  verification:
+    # Raw positional comparison is meaningless for BEVFusion's packed outputs: the
+    # dense graph selects top-500 proposals, and the zero-padded heatmap borders
+    # produce mass ties, so backends legitimately pick different near-zero-score
+    # proposals (measured 2026-09-01: high-score proposals align to 0.038 while the
+    # positional bbox_pred diff is 159). Cross-backend correctness is gated by
+    # deploy.evaluation (mAP equality across backends) instead.
+    enabled: false
+    num_verify_batches: 1
+    scenarios: []
+  evaluation:
+    enabled: false
+    num_samples: -1
+    num_warmup: 2
+    backends:
+      pytorch: { enabled: true, device: cuda }
+      onnx: { enabled: true, device: cuda }
+      tensorrt: { enabled: true, device: cuda }
+
+model:
+  _target_: autoware_ml.models.detection3d.main_modules.bevfusion.BEVFusionLidarDetectionModel
+  metrics: ${metrics}
+  log_dict_configs:
+    _target_: autoware_ml.models.multi_task_base_model.LogDictConfigs
+    _convert_: all
+    on_step: true
+    on_epoch: true
+    prog_bar: true
+    sync_dist: true
+
+  pts_voxel_encoder:
+    _target_: autoware_ml.models.detection3d.encoders.voxel.HardSimpleVoxelSinCosEncoder
+    in_channels: 5
+    # Normalization ranges for (x, y, z, intensity, time_lag).
+    min_norm_values:
+      - ${point_cloud_range.0}
+      - ${point_cloud_range.1}
+      - ${point_cloud_range.2}
+      - 0.0
+      - 0.0
+    max_norm_values:
+      - ${point_cloud_range.3}
+      - ${point_cloud_range.4}
+      - ${point_cloud_range.5}
+      - 255.0
+      - ${max_time_lag}
+  pts_middle_encoder:
+    _target_: autoware_ml.models.detection3d.encoders.sparse.SparseEncoder
+    in_channels: 50
+    sparse_shape: ???
+    output_channels: 128
+    dense_output_shapes: ???
+    # Pair-mask sorting in the deployed graph: locality only, identical detections, so
+    # this is a per-target latency choice. Measured here (100 frames, TensorRT): sorting
+    # 6.70 ms vs not sorting 7.13 ms per frame, so it stays on. Re-measure on the vehicle
+    # target before trusting either value.
+    export_do_sort: true
+  pts_backbone:
+    _target_: autoware_ml.models.detection3d.backbones.second.SECONDBackbone
+    in_channels: 256
+    out_channels: [128, 256]
+    layer_nums: [5, 5]
+    layer_strides: [1, 2]
+  pts_neck:
+    _target_: autoware_ml.models.detection3d.necks.second_fpn.SECONDFPN
+    in_channels: [128, 256]
+    out_channels: [256, 256]
+    upsample_strides: [1, 2]
+  bbox_head:
+    _target_: autoware_ml.models.detection3d.heads.transfusion.TransFusionHead
+    num_proposals: ???
+    auxiliary: true
+    in_channels: 512
+    hidden_channel: 128
+    num_classes: ???
+    class_names: ${database.database_task_configs.detection3d.label_names}
+    num_decoder_layers: 1
+    num_heads: 8
+    feedforward_channels: 256
+    common_heads:
+      center: [2, 2]
+      height: [1, 2]
+      dim: [3, 2]
+      rot: [2, 2]
+      vel: [2, 2]
+    bbox_coder:
+      _target_: autoware_ml.models.detection3d.task_modules.bbox_coders.TransFusionBBoxCoder
+      pc_range:
+        - ${point_cloud_range.0}
+        - ${point_cloud_range.1}
+      voxel_size:
+        - ${voxel_size.0}
+        - ${voxel_size.1}
+      out_size_factor: ${out_size_factor}
+      post_center_range: ???
+      score_threshold: ${score_threshold}
+      code_size: 10
+    assigner:
+      _target_: autoware_ml.models.detection3d.task_modules.assigners.HungarianAssigner3D
+      cls_cost:
+        _target_: autoware_ml.models.detection3d.task_modules.match_costs.ClassificationCost
+        weight: 0.15
+      reg_cost:
+        _target_: autoware_ml.models.detection3d.task_modules.match_costs.BBoxBEVL1Cost
+        weight: 0.25
+      iou_cost:
+        _target_: autoware_ml.models.detection3d.task_modules.match_costs.IoU3DCost
+        weight: 0.25
+    point_cloud_range: ${point_cloud_range}
+    voxel_size:
+      - ${voxel_size.0}
+      - ${voxel_size.1}
+    out_size_factor: ${out_size_factor}
+    code_weights: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.2, 0.2]
+    min_radius: 2
+    gaussian_overlap: 0.1
+    score_threshold: ${score_threshold}
+    post_max_size: 300
+    nms_min_radius: ???
+    heatmap_target: oriented
+  optimizer: ${optimizer}
+  scheduler:
+    _target_: autoware_ml.utils.schedulers.cyclic_cosine_annealing.CyclicCosineAnnealingLR
+    _partial_: true
+    warmup_epochs: 3
+    decay_epochs: 27
+    max_lr_factor: 10.0
+    min_lr_factor: 0.0001