feat(models): add StreamPETR with production three-stage training

.cspell.json

@@ -65,6 +65,7 @@
     "arrowstyle",
     "ascontiguousarray",
     "atleast",
+    "autocast",
     "autohide",
     "avgpool",
     "backprop",
@@ -123,11 +124,13 @@
     "getitem",
     "gloo",
     "graphsurgeon",
+    "Hardsigmoid",
     "Hardswish",
     "hilberts",
     "hparams",
     "hsigmoid",
     "hyperparams",
+    "iccv",
     "iinfo",
     "indice",
     "indptr",
@@ -151,8 +154,10 @@
     "lovasz",
     "LSSFPN",
     "lucide",
+    "matmuls",
     "maxpool",
     "memlock",
+    "metas",
     "mgrid",
     "miou",
     "miscalibrated",

.gitignore

@@ -2,13 +2,16 @@
 data/
 mlruns/
 outputs/
+work_dirs/
 
 # Model files and artifacts
 *.bin
 *.db
 *.onnx
 *.engine
 *.pth
+*.pt
+*.npz
 *.ckpt
 *.log
 *.out

autoware_ml/configs/tasks/detection3d/streampetr/base.yaml

@@ -0,0 +1,158 @@
+# @package _global_
+defaults:
+  - /defaults/default_runtime
+  - _self_
+
+seed: 0
+
+dataset: ???
+
+point_cloud_range: ???
+post_center_range: ???
+
+# RGB channel order, matching the RGB output of the image loading transform.
+img_norm_cfg:
+  mean: [123.675, 116.280, 103.530]
+  std: [58.395, 57.120, 57.375]
+  to_rgb: false
+
+datamodule:
+  collation_map:
+    img: list
+    camera_intrinsics: list
+    lidar2cam: list
+    lidar2img: list
+    timestamp: list
+    prev_exists: list
+    ego_pose: list
+    ego_pose_inv: list
+    gt_boxes: list
+    gt_labels: list
+    gt_num_points: list
+model:
+  _target_: autoware_ml.models.detection3d.streampetr.StreamPETRDetectionModel
+  metrics: ${dataset.detection3d.metrics}
+  img_backbone:
+    _target_: autoware_ml.models.common.backbones.resnet.ResNet50MultiScale
+  img_neck:
+    _target_: autoware_ml.models.common.necks.lss_fpn.GeneralizedLSSFPN
+    in_channels: [512, 1024, 2048]
+    out_channels: 256
+  use_grid_mask: true
+  bbox_head:
+    _target_: autoware_ml.models.detection3d.heads.streampetr.StreamPETRHead
+    num_classes: ${dataset.detection3d.num_classes}
+    in_channels: 256
+    hidden_dim: 256
+    num_queries: ???
+    num_decoder_layers: 6
+    num_heads: 8
+    feedforward_channels: 2048
+    memory_len: 1024
+    topk_proposals: 256
+    num_propagated: 256
+    with_ego_pos: true
+    with_dn: true
+    loss_cls_weight: 2.0
+    loss_bbox_weight: 0.25
+    depth_num: 64
+    LID: true
+    position_range: ${post_center_range}
+    scalar: 10
+    noise_scale: 1.0
+    dn_weight: 1.0
+    split: 0.75
+    use_bottom_center: true
+    bbox_coder:
+      _target_: autoware_ml.models.detection3d.task_modules.bbox_coders.NMSFreeBBoxCoder3D
+      pc_range: ${point_cloud_range}
+      post_center_range: ${post_center_range}
+      score_threshold: 0.0
+      max_num: 300
+    assigner:
+      _target_: autoware_ml.models.detection3d.task_modules.assigners.HungarianAssigner3D
+      cls_cost:
+        _target_: autoware_ml.models.detection3d.task_modules.match_costs.ClassificationCost
+        weight: 2.0
+      reg_cost:
+        _target_: autoware_ml.models.detection3d.task_modules.match_costs.BBox3DL1Cost
+        weight: 0.25
+        # Velocity channels are excluded from matching (match_with_velo=False
+        # in the reference recipe).
+        code_weights: [2.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0]
+      iou_cost:
+        _target_: autoware_ml.models.detection3d.task_modules.match_costs.IoU3DCost
+        weight: 0.0
+    point_cloud_range: ${point_cloud_range}
+    code_weights: [2.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
+  optimizer:
+    _target_: torch.optim.AdamW
+    _partial_: true
+    lr: 5.0e-5
+    weight_decay: 0.01
+  optimizer_group_overrides:
+    img_backbone:
+      lr: 5.0e-6
+  # StreamPETR LR schedule: 500-iteration linear warmup (start
+  # factor 1/3) into a per-epoch cosine decay, stepped every iteration.
+  scheduler:
+    _target_: autoware_ml.utils.schedulers.iter_warmup_epoch_cosine.IterWarmupEpochCosineLR
+    _partial_: true
+    max_epochs: ${trainer.max_epochs}
+    warmup_iters: 500
+    warmup_start_factor: 0.3333333333333333
+    eta_min_factor: 1.0e-4
+  scheduler_config:
+    interval: step
+
+callbacks:
+  # Selects the best checkpoint by detection mAP, not
+  # validation loss, and always trains the configured number of epochs.
+  model_checkpoint:
+    _target_: lightning.pytorch.callbacks.ModelCheckpoint
+    monitor: val/det3d/mAP
+    dirpath: ${hydra:run.dir}/checkpoints
+    filename: best
+    save_top_k: 1
+    mode: max
+    verbose: true
+  early_stopping: null
+
+trainer:
+  max_epochs: 35
+  precision: bf16-mixed
+  # GroupStreamingSampler shards scenes by rank itself.
+  use_distributed_sampler: false
+  gradient_clip_val: 1.0
+  gradient_clip_algorithm: norm
+
+deploy:
+  onnx:
+    dynamo: false
+    opset_version: 17
+    do_constant_folding: true
+    modules:
+      extract_img_feat:
+        output_names: [img_feats]
+      position_embedding:
+        output_names: [pos_embed, cone]
+      pts_head_memory:
+        output_names:
+          [
+            all_cls_scores,
+            all_bbox_preds,
+            post_memory_embedding,
+            post_memory_reference_point,
+            post_memory_timestamp,
+            post_memory_egopose,
+            post_memory_velo,
+            reference_points,
+            tgt,
+            temp_memory,
+            temp_pos,
+            query_pos,
+            query_pos_in,
+            outs_dec,
+          ]
+  tensorrt:
+    enabled: false

autoware_ml/configs/tasks/detection3d/streampetr/focal2d_head.yaml

@@ -0,0 +1,15 @@
+# Auxiliary 2D FocalHead2D shared by the StreamPETR pretrain and T4 configs.
+# Loss weights follow the reference Focal-PETR recipe (2 / 5 / 2 / 10 + 1
+# centerness); the default HungarianAssigner2D mirrors the same cost weights,
+# so no assigner block is needed here.
+_target_: autoware_ml.models.detection3d.heads.focal2d.FocalHead2D
+num_classes: ${dataset.detection3d.num_classes}
+in_channels: 256
+embed_dims: 256
+stride: 16
+loss_cls_weight: 2.0
+loss_bbox_weight: 5.0
+loss_iou_weight: 2.0
+loss_centers2d_weight: 10.0
+loss_centerness_weight: 1.0
+class_names: ${dataset.detection3d.class_names}

autoware_ml/configs/tasks/detection3d/streampetr/vov_320x800_nuscenes.yaml

@@ -0,0 +1,122 @@
+# @package _global_
+defaults:
+  - /tasks/detection3d/streampetr/base
+  - /datasets/nuscenes/detection3d
+  - /datasets/nuscenes/camera
+  - _self_
+
+batch_size: 8
+num_workers: 16
+
+dataset: ${nuscenes}
+
+point_cloud_range: [-54.0, -54.0, -5.0, 54.0, 54.0, 3.0]
+post_center_range: [-61.2, -61.2, -10.0, 61.2, 61.2, 10.0]
+
+datamodule:
+  _target_: autoware_ml.datamodule.nuscenes.multiview_detection3d.NuscenesMultiviewDetection3DDataModule
+  data_root: ${dataset.data_root}
+  train_ann_file: nuscenes_infos_train.pkl
+  val_ann_file: nuscenes_infos_val.pkl
+  test_ann_file: nuscenes_infos_test.pkl
+  class_names: ${dataset.detection3d.class_names}
+  camera_order: ${dataset.camera.camera_order}
+  streaming: true
+  train_dataloader_cfg:
+    batch_size: ${batch_size}
+    num_workers: ${num_workers}
+    shuffle: false
+  # Streaming evaluation runs one lane per rank so every frame is scored;
+  # wider eval batches would trim lanes to the shortest one.
+  val_dataloader_cfg:
+    batch_size: 1
+    num_workers: ${num_workers}
+  test_dataloader_cfg:
+    batch_size: 1
+    num_workers: ${num_workers}
+  train_transforms:
+    _target_: autoware_ml.transforms.base.TransformsCompose
+    pipeline:
+      - _target_: autoware_ml.transforms.boxes3d.loading.LoadAnnotations3D
+        name_mapping: ${dataset.detection3d.name_mapping}
+      - _target_: autoware_ml.transforms.camera.loading.LoadMultiViewImagesFromFiles
+        # Keep pixels in [0,255]: img_norm_cfg mean/std are 0-255-scale ImageNet stats.
+        normalize_to_unit: false
+      - _target_: autoware_ml.transforms.boxes3d.filters.ObjectRangeFilter
+        point_cloud_range: ${point_cloud_range}
+      - _target_: autoware_ml.transforms.camera.resize.ResizeCropFlipRotImage
+        data_aug_conf:
+          resize_lim: [0.47, 0.625]
+          final_dim: [320, 800]
+          bot_pct_lim: [0.0, 0.0]
+          rot_lim: [0.0, 0.0]
+          rand_flip: false
+        training: true
+      - _target_: autoware_ml.transforms.camera.geometry.GlobalRotScaleTrans
+        rot_range: [0.0, 0.0]
+        scale_ratio_range: [1.0, 1.0]
+      - _target_: autoware_ml.transforms.camera.resize.PadMultiViewImage
+        size_divisor: 32
+      - _target_: autoware_ml.transforms.camera.normalize.NormalizeMultiviewImage
+        mean: ${img_norm_cfg.mean}
+        std: ${img_norm_cfg.std}
+        to_rgb: ${img_norm_cfg.to_rgb}
+  val_transforms:
+    _target_: autoware_ml.transforms.base.TransformsCompose
+    pipeline:
+      - _target_: autoware_ml.transforms.boxes3d.loading.LoadAnnotations3D
+        name_mapping: ${dataset.detection3d.name_mapping}
+      - _target_: autoware_ml.transforms.camera.loading.LoadMultiViewImagesFromFiles
+        # Keep pixels in [0,255]: img_norm_cfg mean/std are 0-255-scale ImageNet stats.
+        normalize_to_unit: false
+      - _target_: autoware_ml.transforms.camera.resize.ResizeCropFlipRotImage
+        data_aug_conf:
+          resize_lim: [0.47, 0.625]
+          final_dim: [320, 800]
+          bot_pct_lim: [0.0, 0.0]
+          rot_lim: [0.0, 0.0]
+          rand_flip: false
+        training: false
+      - _target_: autoware_ml.transforms.camera.resize.PadMultiViewImage
+        size_divisor: 32
+      - _target_: autoware_ml.transforms.camera.normalize.NormalizeMultiviewImage
+        mean: ${img_norm_cfg.mean}
+        std: ${img_norm_cfg.std}
+        to_rgb: ${img_norm_cfg.to_rgb}
+      - _target_: autoware_ml.transforms.boxes3d.filters.ObjectRangeFilter
+        point_cloud_range: ${point_cloud_range}
+  test_transforms: ${datamodule.val_transforms}
+  predict_transforms:
+    _target_: autoware_ml.transforms.base.TransformsCompose
+    pipeline:
+      - _target_: autoware_ml.transforms.camera.loading.LoadMultiViewImagesFromFiles
+        # Keep pixels in [0,255]: img_norm_cfg mean/std are 0-255-scale ImageNet stats.
+        normalize_to_unit: false
+      - _target_: autoware_ml.transforms.camera.resize.ResizeCropFlipRotImage
+        data_aug_conf:
+          resize_lim: [0.47, 0.625]
+          final_dim: [320, 800]
+          bot_pct_lim: [0.0, 0.0]
+          rot_lim: [0.0, 0.0]
+          rand_flip: false
+        training: false
+      - _target_: autoware_ml.transforms.camera.resize.PadMultiViewImage
+        size_divisor: 32
+      - _target_: autoware_ml.transforms.camera.normalize.NormalizeMultiviewImage
+        mean: ${img_norm_cfg.mean}
+        std: ${img_norm_cfg.std}
+        to_rgb: ${img_norm_cfg.to_rgb}
+
+model:
+  img_backbone:
+    _target_: autoware_ml.models.common.backbones.vovnet.VoVNet99MultiScale
+    input_ch: 3
+    out_features: [stage4, stage5]
+    frozen_stages: -1
+    norm_eval: true
+  img_neck:
+    _target_: autoware_ml.models.common.necks.lss_fpn.GeneralizedLSSFPN
+    in_channels: [768, 1024]
+    out_channels: 256
+  bbox_head:
+    num_queries: 644