Add DINOv3 models (#131)

- facebook/dinov3-vits16-pretrain-lvd1689m
- facebook/dinov3-vits16plus-pretrain-lvd1689m
- facebook/dinov3-vitb16-pretrain-lvd1689m
- facebook/dinov3-vitl16-pretrain-lvd1689m
- facebook/dinov3-vith16plus-pretrain-lvd1689m
- facebook/dinov3-vitl16-pretrain-sat493m

Author: jamjamjon

README.md

@@ -66,6 +66,7 @@ usls = "latest-version"
 | [MobileOne](https://github.com/apple/ml-mobileone) | Image Classification | [demo](examples/mobileone) |
 | [DeiT](https://github.com/facebookresearch/deit) | Image Classification | [demo](examples/deit) |
 | [DINOv2](https://github.com/facebookresearch/dinov2) | Vision Embedding | [demo](examples/dinov2) |
+| [DINOv3](https://github.com/facebookresearch/dinov3) | Vision Embedding | [demo](examples/dinov3) |
 | [YOLOv5](https://github.com/ultralytics/yolov5) | Image Classification<br />Object Detection<br />Instance Segmentation | [demo](examples/yolo) |
 | [YOLOv6](https://github.com/meituan/YOLOv6) | Object Detection | [demo](examples/yolo) |
 | [YOLOv7](https://github.com/WongKinYiu/yolov7) | Object Detection | [demo](examples/yolo) |

examples/dinov2/main.rs

@@ -17,8 +17,8 @@ fn main() -> Result<()> {
     let mut model = DINOv2::new(config)?;
 
     // encode images
-    let y = model.encode_images(&xs)?;
-    println!("Feat shape: {:?}", y.shape());
+    let feats = model.encode_images(&xs)?;
+    println!("Feat shape: {:?}", feats.shape());
 
     usls::perf(false);
 

examples/dinov3/README.md

@@ -0,0 +1,7 @@
+This demo showcases how to use `DINOv2` to compute image similarity, applicable for image-to-image retrieval tasks.
+
+## Quick Start
+
+```shell
+cargo run -r --example dinov2
+```

examples/dinov3/main.rs

@@ -0,0 +1,41 @@
+use anyhow::Result;
+use usls::{models::DINOv3, Config, DataLoader};
+
+/// DINOv3 Example
+#[derive(argh::FromArgs)]
+struct Args {
+    /// cpu:0, cuda:0
+    #[argh(option, default = "String::from(\"cpu:0\")")]
+    device: String,
+
+    /// fp32, fp16, q8, q4, q4f16, bnb4
+    #[argh(option, default = "String::from(\"fp16\")")]
+    dtype: String,
+}
+
+fn main() -> Result<()> {
+    tracing_subscriber::fmt()
+        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
+        .with_timer(tracing_subscriber::fmt::time::ChronoLocal::rfc_3339())
+        .init();
+    let args: Args = argh::from_env();
+
+    // images
+    let xs = DataLoader::try_read_n(&["./assets/bus.jpg", "./assets/bus.jpg"])?;
+
+    // model
+    let config = Config::dinov3_vits16_lvd1689m()
+        .with_batch_size_all(xs.len())
+        .with_dtype_all(args.dtype.parse()?)
+        .with_device_all(args.device.parse()?)
+        .commit()?;
+    let mut model = DINOv3::new(config)?;
+
+    // encode images
+    let feats = model.encode_images(&xs)?;
+    println!("Feat shape: {:?}", feats.shape());
+
+    usls::perf(false);
+
+    Ok(())
+}

src/models/dinov2/impl.rs

@@ -1,15 +1,14 @@
 use aksr::Builder;
 use anyhow::Result;
 
-use crate::{elapsed_module, Config, Engine, Image, Processor, Scale, Xs, X};
+use crate::{elapsed_module, Config, Engine, Image, Processor, Xs, X};
 
 #[derive(Builder, Debug)]
 pub struct DINOv2 {
     engine: Engine,
     height: usize,
     width: usize,
     batch: usize,
-    dim: usize,
     processor: Processor,
 }
 
@@ -21,14 +20,6 @@ impl DINOv2 {
             engine.try_height().unwrap_or(&384.into()).opt(),
             engine.try_width().unwrap_or(&384.into()).opt(),
         );
-        let dim = match &config.scale {
-            Some(Scale::S) => 384,
-            Some(Scale::B) => 768,
-            Some(Scale::L) => 1024,
-            Some(Scale::G) => 1536,
-            Some(x) => anyhow::bail!("Unsupported scale: {:?}", x),
-            None => anyhow::bail!("No model scale specified"),
-        };
         let processor = Processor::try_from_config(&config.processor)?
             .with_image_width(width as _)
             .with_image_height(height as _);
@@ -38,14 +29,12 @@ impl DINOv2 {
             height,
             width,
             batch,
-            dim,
             processor,
         })
     }
 
     fn preprocess(&mut self, xs: &[Image]) -> Result<Xs> {
         let x = self.processor.process_images(xs)?;
-
         Ok(x.into())
     }
 
@@ -57,7 +46,6 @@ impl DINOv2 {
         let xs = elapsed_module!("DINOv2", "visual-preprocess", self.preprocess(xs)?);
         let xs = elapsed_module!("DINOv2", "visual-inference", self.inference(xs)?);
         let x = elapsed_module!("DINOv2", "visual-postprocess", xs[0].to_owned());
-
         Ok(x)
     }
 }

src/models/dinov3/README.md

@@ -0,0 +1,9 @@
+# DINOv3
+
+## Official Repository
+
+The official repository can be found on: [GitHub](https://github.com/facebookresearch/dinov3)
+
+## Example
+
+Refer to the [example](../../../examples/dinov3)

src/models/dinov3/config.rs

@@ -0,0 +1,30 @@
+/// Model configuration for `DINOv3`
+impl crate::Config {
+    pub fn dinov3() -> Self {
+        Self::dinov2().with_name("dinov3")
+    }
+
+    pub fn dinov3_vits16_lvd1689m() -> Self {
+        Self::dinov3().with_model_file("vits16-pretrain-lvd1689m.onnx")
+    }
+
+    pub fn dinov3_vits16plus_lvd1689m() -> Self {
+        Self::dinov3().with_model_file("vits16plus-pretrain-lvd1689m.onnx")
+    }
+
+    pub fn dinov3_vitb16_lvd1689m() -> Self {
+        Self::dinov3().with_model_file("vitb16-pretrain-lvd1689m.onnx")
+    }
+
+    pub fn dinov3_vitl16_lvd1689m() -> Self {
+        Self::dinov3().with_model_file("vitl16-pretrain-lvd1689m.onnx")
+    }
+
+    pub fn dinov3_vith16plus_lvd1689m() -> Self {
+        Self::dinov3().with_model_file("vith16plus-pretrain-lvd1689m.onnx")
+    }
+
+    pub fn dinov3_vitl16_sat493m() -> Self {
+        Self::dinov3().with_model_file("vitl16-pretrain-sat493m.onnx")
+    }
+}

src/models/dinov3/mod.rs

@@ -0,0 +1,3 @@
+mod config;
+
+pub type DINOv3 = crate::DINOv2;

src/models/mod.rs

@@ -11,6 +11,7 @@ mod deit;
 mod depth_anything;
 mod depth_pro;
 mod dinov2;
+mod dinov3;
 mod dwpose;
 mod fast;
 mod fastvit;
@@ -47,6 +48,7 @@ pub use db::*;
 pub use depth_anything::*;
 pub use depth_pro::*;
 pub use dinov2::*;
+pub use dinov3::*;
 pub use florence2::*;
 pub use grounding_dino::*;
 pub use mediapipe_segmenter::*;