feat(gdino): add multi-batch inference support; integrate mm-gdino & llmdet models; (#162)

And extend post-processing to token/phrase class-name modes

Author: jamjamjon

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "usls"
 edition = "2021"
-version = "0.1.9"
+version = "0.1.10"
 rust-version = "1.85"
 description = "A Rust library integrated with ONNXRuntime, providing a collection of ML models."
 repository = "https://github.com/jamjamjon/usls"

README.md

@@ -142,6 +142,8 @@ usls = { version = "latest-version", features = [ "cuda" ] }
 | [YOLO-World](https://github.com/AILab-CVC/YOLO-World) | Open-Set Detection With Language | `yolo` | [demo](examples/yolo) |
 | [YOLOE](https://github.com/THU-MIG/yoloe) | Open-Set Detection And Segmentation | `yoloe` | [demo-prompt-free](examples/yoloe-prompt-free)<br />[demo-prompt(visual & textual)](examples/yoloe-prompt) |
 | [GroundingDINO](https://github.com/IDEA-Research/GroundingDINO) | Open-Set Detection With Language | `grounding-dino` | [demo](examples/grounding-dino) |
+| [MM-GDINO](https://github.com/open-mmlab/mmdetection/blob/main/configs/mm_grounding_dino/README.md) | Open-Set Detection With Language | `grounding-dino` | [demo](examples/grounding-dino) |
+| [LLMDet](https://github.com/iSEE-Laboratory/LLMDet) | Open-Set Detection With Language | `grounding-dino` | [demo](examples/grounding-dino) |
 | [CLIP](https://github.com/openai/CLIP) | Vision-Language Embedding | `clip` | [demo](examples/clip) |
 | [jina-clip-v1](https://huggingface.co/jinaai/jina-clip-v1) | Vision-Language Embedding | `clip` | [demo](examples/clip) |
 | [jina-clip-v2](https://huggingface.co/jinaai/jina-clip-v2) | Vision-Language Embedding | `clip` | [demo](examples/clip) |

examples/grounding-dino/README.md

@@ -1,10 +1,6 @@
 ## Quick Start
 
 ```shell
-cargo run -r -F grounding-dino -F cuda --example grounding-dino -- --device cuda --dtype fp16
+cargo run -r -F grounding-dino -F cuda --example grounding-dino -- --device cuda --dtype q8
 ```
 
-
-## Results
-
-![](https://github.com/jamjamjon/assets/releases/download/grounding-dino/demo.png)

examples/grounding-dino/main.rs

@@ -4,6 +4,10 @@ use usls::{models::GroundingDINO, Annotator, Config, DataLoader};
 #[derive(argh::FromArgs)]
 /// Example
 struct Args {
+    /// source image
+    #[argh(option, default = "vec![String::from(\"./assets/bus.jpg\")]")]
+    source: Vec<String>,
+
     /// dtype
     #[argh(option, default = "String::from(\"q8\")")]
     dtype: String,
@@ -12,47 +16,50 @@ struct Args {
     #[argh(option, default = "String::from(\"cpu:0\")")]
     device: String,
 
-    /// source image
-    #[argh(option, default = "vec![String::from(\"./assets/bus.jpg\")]")]
-    source: Vec<String>,
-
-    /// open class names
+    /// captions
     #[argh(
         option,
         default = "vec![
             String::from(\"person\"),
-            String::from(\"a hand\"),
-            String::from(\"a shoe\"),
             String::from(\"bus\"),
-            String::from(\"dog\"),
+            String::from(\"a dog\"),
             String::from(\"cat\"),
-            String::from(\"sign\"),
+            String::from(\"stop sign\"),
             String::from(\"tie\"),
-            String::from(\"monitor\"),
-            String::from(\"glasses\"),
+            String::from(\"eye glasses\"),
             String::from(\"tree\"),
-            String::from(\"head\"),
+            String::from(\"camera\"),
+            String::from(\"hand\"),
+            String::from(\"a shoe\"),
+            String::from(\"balcony\"),
+            String::from(\"window\"),
         ]"
     )]
     labels: Vec<String>,
+
+    /// token level class
+    #[argh(option, default = "false")]
+    token_level_class: bool,
 }
 
 fn main() -> Result<()> {
     tracing_subscriber::fmt()
         .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
         .with_timer(tracing_subscriber::fmt::time::ChronoLocal::rfc_3339())
         .init();
-
     let args: Args = argh::from_env();
 
-    let config = Config::grounding_dino_tiny()
+    let config = Config::llmdet_tiny()
+        // mm_gdino_base_all()
+        // grounding_dino_tiny()
         .with_model_dtype(args.dtype.parse()?)
         .with_model_device(args.device.parse()?)
         .with_text_names_owned(&args.labels)
-        .with_class_confs(&[0.25])
+        .with_class_confs(&[0.35])
         .with_text_confs(&[0.25])
+        .with_model_num_dry_run(0)
+        .with_token_level_class(args.token_level_class)
         .commit()?;
-
     let mut model = GroundingDINO::new(config)?;
 
     // load images

src/core/config.rs

@@ -4,7 +4,6 @@ use aksr::Builder;
 use crate::models::SamKind;
 #[cfg(feature = "yolo")]
 use crate::models::YOLOPredsFormat;
-
 use crate::{ORTConfig, ProcessorConfig, Scale, Task, Version};
 
 /// Configuration for model inference including engines, processors, and task settings.
@@ -69,6 +68,7 @@ pub struct Config {
     #[cfg(feature = "sam")]
     pub sam_low_res_mask: Option<bool>,
     pub max_tokens: Option<usize>,
+    pub token_level_class: bool,
 }
 
 impl Default for Config {
@@ -102,6 +102,7 @@ impl Default for Config {
             #[cfg(feature = "sam")]
             sam_low_res_mask: None,
             topk: None,
+            token_level_class: false,
             model: Default::default(),
             encoder: Default::default(),
             decoder: Default::default(),

src/models/grounding_dino/README.md

@@ -1,8 +1,17 @@
-# Grounding DINO: Marrying DINO with Grounded Pre-Training for Open-Set Object Detection
+# Grounding DINO Series
 
-## Official Repository
+## Repository & Papers
+
+- **GroundingDINO**
+  - Paper: https://arxiv.org/abs/2303.05499
+  - Repo: https://github.com/IDEA-Research/GroundingDINO
+- **mm-GroundingDINO (mm-gdino)**
+  - Paper: https://arxiv.org/abs/2401.02361
+  - Repo: https://github.com/open-mmlab/mmdetection/blob/main/configs/mm_grounding_dino/README.md
+- **LLMDet**
+  - Paper: https://arxiv.org/abs/2501.18954
+  - Repo: https://github.com/iSEE-Laboratory/LLMDet
 
-The official repository can be found on: [GitHub](https://github.com/IDEA-Research/GroundingDINO)
 
 ## Example
 

src/models/grounding_dino/config.rs

@@ -1,11 +1,12 @@
-/// Model configuration for `GroundingDino`
+/// Model configuration for `GroundingDino` Series
 impl crate::Config {
     pub fn grounding_dino() -> Self {
         Self::default()
             .with_name("grounding-dino")
-            .with_model_ixx(0, 0, 1.into()) // TODO: current onnx model does not support bs > 1
-            .with_model_ixx(0, 2, 800.into()) // TODO: matters
-            .with_model_ixx(0, 3, 1200.into()) // TODO: matters
+            .with_model_ixx(0, 0, 1.into())
+            .with_model_ixx(0, 1, 3.into())
+            .with_model_ixx(0, 2, 800.into())
+            .with_model_ixx(0, 3, 1066.into())
             .with_resize_mode(crate::ResizeMode::FitAdaptive)
             .with_resize_filter("CatmullRom")
             .with_image_mean(&[0.485, 0.456, 0.406])
@@ -17,10 +18,54 @@ impl crate::Config {
     }
 
     pub fn grounding_dino_tiny() -> Self {
-        Self::grounding_dino().with_model_file("swint-ogc.onnx")
+        Self::grounding_dino().with_model_file("gdino-tiny.onnx")
     }
 
     pub fn grounding_dino_base() -> Self {
-        Self::grounding_dino().with_model_file("swinb-cogcoor.onnx")
+        Self::grounding_dino().with_model_file("gdino-base.onnx")
+    }
+
+    pub fn llmdet_tiny() -> Self {
+        Self::grounding_dino().with_model_file("llmdet-tiny.onnx")
+    }
+
+    pub fn llmdet_base() -> Self {
+        Self::grounding_dino().with_model_file("llmdet-base.onnx")
+    }
+
+    pub fn llmdet_large() -> Self {
+        Self::grounding_dino().with_model_file("llmdet-large.onnx")
+    }
+
+    pub fn mm_gdino_tiny_o365v1_goldg_grit_v3det() -> Self {
+        Self::grounding_dino().with_model_file("mm-gdino-tiny-o365v1-goldg-grit-v3det.onnx")
+    }
+
+    pub fn mm_gdino_tiny_o365v1_goldg_grit() -> Self {
+        Self::grounding_dino().with_model_file("mm-gdino-tiny-o365v1-goldg-grit.onnx")
+    }
+
+    pub fn mm_gdino_tiny_o365v1_goldg_v3det() -> Self {
+        Self::grounding_dino().with_model_file("mm-gdino-tiny-o365v1-goldg-v3det.onnx")
+    }
+
+    pub fn mm_gdino_tiny_o365v1_goldg() -> Self {
+        Self::grounding_dino().with_model_file("mm-gdino-tiny-o365v1-goldg.onnx")
+    }
+
+    pub fn mm_gdino_base_o365v1_goldg_v3det() -> Self {
+        Self::grounding_dino().with_model_file("mm-gdino-base-o365v1-goldg-v3det.onnx")
+    }
+
+    pub fn mm_gdino_base_all() -> Self {
+        Self::grounding_dino().with_model_file("mm-gdino-base-all.onnx")
+    }
+
+    pub fn mm_gdino_large_o365v2_oiv6_goldg() -> Self {
+        Self::grounding_dino().with_model_file("mm-gdino-large-o365v2-oiv6-goldg.onnx")
+    }
+
+    pub fn mm_gdino_large_all() -> Self {
+        Self::grounding_dino().with_model_file("mm-gdino-large-all.onnx")
     }
 }