add Nanonets-OCR2

Author: foldl

README.md

@@ -31,6 +31,7 @@ LittleAcademia[<a href="https://github.com/foldl/little-academia"   style="text-
 
 **What's New:**
 
+* 2025-10-13: Nanonets-OCR2
 * 2025-10-13: dots.ocr
 * 2025-10-10: [I can draw](./docs/multimodal.md): Janus-Pro
 * 2025-09-23: Qwen2.5-VL

convert.py

@@ -241,6 +241,7 @@ class ModelType(Enum):
     Qwen2Audio              = ModelTypeTagChatAudioIn + 0x0000001
 
     Qwen2_5VL               = ModelTypeTagChatImageVideoIn + 0x0000001
+    Qwen2_VL                = ModelTypeTagChatImageVideoIn + 0x0000002
     KimiVL                  = ModelTypeTagChatImageVideoIn + 0x0000100
     SmolVLM                 = ModelTypeTagChatImageVideoIn + 0x0000200
 
@@ -4479,6 +4480,58 @@ def get_weight_names(config):
 
         return weight_names
 
+class QWen2_VLConverter(BaseConverter):
+    MODEL_TYPE = ModelType.Qwen2_VL
+
+    @classmethod
+    def state_dict_pp(cls, config, state_dict):
+        r = QWen2_5VLConverter.state_dict_pp(config, state_dict)
+        return r
+
+    @staticmethod
+    def dump_config(f, config, ggml_type):
+        assert config.vision_config['hidden_act'] == 'quick_gelu'
+        config.vision_config['hidden_act'] = 'silu'
+        config.vision_config['hidden_size'] = config.vision_config['embed_dim']
+        QWen2_5VLConverter.dump_config(f, config, ggml_type)
+
+    @staticmethod
+    def get_weight_names(config):
+        weight_names = QWen2Converter.get_weight_names(config if config.text_config is None else AttributeDict(config.text_config))
+
+        for i in range(config.vision_config['depth']):
+            weight_names += [
+                f"visual.blocks.{i}.attn.proj.bias",
+                f"visual.blocks.{i}.attn.proj.weight",
+                f"visual.blocks.{i}.attn.q_proj.bias",
+                f"visual.blocks.{i}.attn.q_proj.weight",
+                f"visual.blocks.{i}.attn.k_proj.bias",
+                f"visual.blocks.{i}.attn.k_proj.weight",
+                f"visual.blocks.{i}.attn.v_proj.bias",
+                f"visual.blocks.{i}.attn.v_proj.weight",
+                f"visual.blocks.{i}.mlp.fc1.bias",
+                f"visual.blocks.{i}.mlp.fc1.weight",
+                f"visual.blocks.{i}.mlp.fc2.bias",
+                f"visual.blocks.{i}.mlp.fc2.weight",
+                f"visual.blocks.{i}.norm1.bias",
+                f"visual.blocks.{i}.norm1.weight",
+                f"visual.blocks.{i}.norm2.bias",
+                f"visual.blocks.{i}.norm2.weight",
+            ]
+
+        weight_names += [
+            "visual.merger.ln_q.bias",
+            "visual.merger.ln_q.weight",
+            "visual.merger.mlp.0.bias",
+            "visual.merger.mlp.0.weight",
+            "visual.merger.mlp.2.bias",
+            "visual.merger.mlp.2.weight",
+            "visual.patch_embed.proj.0.weight",
+            "visual.patch_embed.proj.1.weight",
+        ]
+
+        return weight_names
+
 class QWen2_5VLConverter(BaseConverter):
     MODEL_TYPE = ModelType.Qwen2_5VL
 
@@ -4507,21 +4560,23 @@ def state_dict_pp(cls, config, state_dict):
 
     @staticmethod
     def dump_config(f, config, ggml_type):
-        assert config.rope_scaling['type'] == 'mrope', 'rope_scaling must be mrope'
+        #assert config.rope_scaling['type'] == 'mrope', 'rope_scaling must be mrope'
         assert config.vision_config['hidden_act'] == 'silu'
 
         QWen2Converter.dump_config(f, config, ggml_type)
 
         MROPE_SECTION_MAX = 4
 
+        text_config = config if config.text_config is None else AttributeDict(config.text_config)
+
         config_values = [
-            config.tie_word_embeddings if config.tie_word_embeddings is not None else 0
+            text_config.tie_word_embeddings if text_config.tie_word_embeddings is not None else 0
         ] + pad_to_len(config.rope_scaling['mrope_section'], MROPE_SECTION_MAX)
         f.write(struct.pack("<" + "i" * len(config_values), *config_values))
 
     @staticmethod
     def get_weight_names(config):
-        weight_names = QWen2Converter.get_weight_names(config)
+        weight_names = QWen2Converter.get_weight_names(config if config.text_config is None else AttributeDict(config.text_config))
 
         for i in range(config.vision_config['depth']):
             weight_names += [
@@ -8501,6 +8556,8 @@ def main():
         QWen2Converter.convert(config, model_files, vocab, ggml_type, args.save_path)
     elif arch == 'Qwen2AudioForConditionalGeneration':
         QWen2AudioConverter.convert(config, model_files, vocab, ggml_type, args.save_path)
+    elif arch == 'Qwen2VLForConditionalGeneration':
+        QWen2_VLConverter.convert(config, model_files, vocab, ggml_type, args.save_path)
     elif arch == 'Qwen2_5_VLForConditionalGeneration':
         QWen2_5VLConverter.convert(config, model_files, vocab, ggml_type, args.save_path)
     elif arch == 'KimiVLForConditionalGeneration':

docs/models.md

@@ -384,6 +384,9 @@ Please use `--format completion` for these models.
     Note: Prompt for OCR: _{{image:...}}Extract the text content from this image_. [Here](https://github.com/rednote-hilab/dots.ocr/blob/master/dots_ocr/utils/prompts.py)
     are other prompts for OCR. Use `+single-turn` to discard history automatically.
 
+* Nanonets-OCR2 (`Qwen2VLForConditionalGeneration`, `Qwen2_5_VLForConditionalGeneration`)
+    * [x] OCR2: [3B](https://huggingface.co/nanonets/Nanonets-OCR2-3B/tree/d0368059ad151ce9e38f526890cfd4f27b28be65), [1.5B](https://huggingface.co/nanonets/Nanonets-OCR2-1.5B-exp/tree/306a9b2a65672a3dbebd9bce9a9373a9a18674a2)
+
 ## RAG Models
 
 * Text Embedding (`XLMRobertaModel`)

models/dots.cpp

@@ -29,7 +29,7 @@ namespace chatllm::dots::vit
               norm(ctx, config.hidden_size)
         {}
 
-        ggml::tensor *forward(ComputeContext *ctx, ggml::tensor *input, int grid_h, int grid_w)
+        ggml::tensor *forward(ComputeContext *ctx, ggml::tensor *input, int grid_h, int grid_w) override
         {
             ggml::tensor *x = nullptr;
             x = proj.forward(ctx, input);
@@ -234,7 +234,7 @@ namespace chatllm::dots::vit
             loaded = true;
         }
 
-        ggml::tensor *forward(ComputeContext *ctx, ggml::tensor *input, int grid_h, int grid_w)
+        ggml::tensor *forward(ComputeContext *ctx, ggml::tensor *input, int grid_h, int grid_w) override
         {
             pos_helper->prepare(grid_h, grid_w);
 

models/moonshot.cpp

@@ -162,7 +162,7 @@ namespace chatllm::kimi::vit
             return ggml::nelements(pos_emb);
         }
 
-        ggml::tensor *forward(ComputeContext *ctx, ggml::tensor *input, int grid_h, int grid_w)
+        ggml::tensor *forward(ComputeContext *ctx, ggml::tensor *input, int grid_h, int grid_w) override
         {
             CHATLLM_CHECK(ggml::get_dim(input, 3) == 1);
 
@@ -204,7 +204,7 @@ namespace chatllm::kimi::vit
         {
         }
 
-        ggml::tensor *forward(ComputeContext *ctx, ggml::tensor *input, int grid_h, int grid_w)
+        ggml::tensor *forward(ComputeContext *ctx, ggml::tensor *input, int grid_h, int grid_w) override
         {
             ggml::tensor *x = proj.forward(ctx, input);
             x = ggml::reshape_3d(ctx, x, ggml::get_dim(x, 2), grid_h, grid_w);
@@ -422,7 +422,7 @@ namespace chatllm::kimi::vit
             loaded = true;
         }
 
-        ggml::tensor *forward(ComputeContext *ctx, ggml::tensor *input, int grid_h, int grid_w)
+        ggml::tensor *forward(ComputeContext *ctx, ggml::tensor *input, int grid_h, int grid_w) override
         {
             multi_modal_projector.merge_param.grid_h = grid_h;
             multi_modal_projector.merge_param.grid_w = grid_w;