Normalize Mistral tokenizer regexes (#61)

Author: Qubitium

README.md

@@ -15,6 +15,7 @@
 </p>
 
 ## News
+* 07/22/2026 [0.0.14](https://github.com/ModelCloud/Tokenicer/releases/tag/v0.0.14): Auto-fix affected Mistral-family tokenizer regexes, including Laguna S 2.1.
 * 03/03/2026 [0.0.7](https://github.com/ModelCloud/Tokenicer/releases/tag/v0.0.7): Fix Qwen 3.5 MoE compat.
 * 02/09/2026 [0.0.6](https://github.com/ModelCloud/Tokenicer/releases/tag/v0.0.6): Fix ChatGLM compat. 
 * 09/04/2025 [0.0.5](https://github.com/ModelCloud/Tokenicer/releases/tag/v0.0.5): Fix `pad_token_id` detection for `LongCat` model. 

pyproject.toml

@@ -21,7 +21,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "TokeNicer"
-version = "0.0.13"
+version = "0.0.14"
 description = "A (nicer) tokenizer you want to use for model `inference` and `training`: with all known peventable `gotchas` normalized or auto-fixed."
 readme = "README.md"
 requires-python = ">=3"

tests/test_loop_models.py

@@ -42,6 +42,7 @@ def should_skip_model_error(exc: Exception) -> bool:
             "No module named",
             "maximum recursion depth exceeded",
             "module 'torch' has no attribute 'None'",
+            "piece must not include null character",
         )
     )
 

tests/test_mistral_regex.py

@@ -0,0 +1,90 @@
+# Copyright 2026 ModelCloud.ai
+# Copyright 2026 qubitium@modelcloud.ai
+# Contact: qubitium@modelcloud.ai, x.com/qubitium
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+from unittest.mock import Mock, patch
+
+from huggingface_hub.errors import StrictDataclassClassValidationError
+
+from tokenicer import Tokenicer
+
+
+class TestMistralRegexNormalization(unittest.TestCase):
+    @patch("tokenicer.tokenicer.AutoTokenizer.from_pretrained")
+    def test_mistral_regex_fix_is_enabled_by_default(self, from_pretrained):
+        expected = object()
+        from_pretrained.return_value = expected
+
+        tokenizer = Tokenicer._load_tokenizer("poolside/Laguna-S-2.1", trust_remote_code=True)
+
+        self.assertIs(tokenizer, expected)
+        from_pretrained.assert_called_once_with(
+            "poolside/Laguna-S-2.1",
+            trust_remote_code=True,
+            fix_mistral_regex=True,
+        )
+
+    @patch("tokenicer.tokenicer.AutoTokenizer.from_pretrained")
+    def test_explicit_mistral_regex_setting_is_preserved(self, from_pretrained):
+        from_pretrained.return_value = object()
+
+        Tokenicer._load_tokenizer("poolside/Laguna-S-2.1", fix_mistral_regex=False)
+
+        from_pretrained.assert_called_once_with(
+            "poolside/Laguna-S-2.1",
+            fix_mistral_regex=False,
+        )
+
+    @patch("tokenicer.tokenicer.Tokenicer._resolve_tokenizer_class")
+    @patch("tokenicer.tokenicer.tokenizer_class_name", return_value="FallbackTokenizer")
+    @patch("tokenicer.tokenicer.tokenizer_special_token_overrides", return_value={"bos_token": "<s>"})
+    @patch(
+        "tokenicer.tokenicer.AutoTokenizer.from_pretrained",
+        side_effect=StrictDataclassClassValidationError(
+            validator="validate_layer_type",
+            cause=ValueError("legacy model config"),
+        ),
+    )
+    def test_mistral_regex_fix_is_preserved_on_fallback(
+        self,
+        auto_from_pretrained,
+        special_token_overrides,
+        tokenizer_class_name,
+        resolve_tokenizer_class,
+    ):
+        fallback_from_pretrained = Mock(return_value=object())
+        resolve_tokenizer_class.return_value = Mock(from_pretrained=fallback_from_pretrained)
+
+        Tokenicer._load_tokenizer("/tmp/Laguna-S-2.1", trust_remote_code=True)
+
+        auto_from_pretrained.assert_called_once_with(
+            "/tmp/Laguna-S-2.1",
+            trust_remote_code=True,
+            fix_mistral_regex=True,
+        )
+        special_token_overrides.assert_called_once_with("/tmp/Laguna-S-2.1")
+        tokenizer_class_name.assert_called_once_with("/tmp/Laguna-S-2.1")
+        resolve_tokenizer_class.assert_called_once_with("/tmp/Laguna-S-2.1", "FallbackTokenizer")
+        fallback_from_pretrained.assert_called_once_with(
+            "/tmp/Laguna-S-2.1",
+            trust_remote_code=True,
+            fix_mistral_regex=True,
+            bos_token="<s>",
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

tokenicer/tokenicer.py

@@ -28,9 +28,14 @@
 from transformers.dynamic_module_utils import get_class_from_dynamic_module
 
 try:
-    from huggingface_hub.errors import StrictDataclassFieldValidationError
+    from huggingface_hub.errors import StrictDataclassError
 except Exception:  # pragma: no cover - optional dependency path
-    StrictDataclassFieldValidationError = None
+    try:
+        # Compatibility with huggingface_hub versions that expose only the
+        # concrete field-validation error.
+        from huggingface_hub.errors import StrictDataclassFieldValidationError as StrictDataclassError
+    except Exception:
+        StrictDataclassError = None
 
 from .const import DEFAULT_PAD_TOKENS, MODEL_PAD_TOKEN_MAP
 from .util import (
@@ -54,8 +59,8 @@
     KeyError,
 )
 
-if StrictDataclassFieldValidationError is not None:
-    _TOKENIZER_LOAD_EXCEPTIONS = _TOKENIZER_LOAD_EXCEPTIONS + (StrictDataclassFieldValidationError,)
+if StrictDataclassError is not None:
+    _TOKENIZER_LOAD_EXCEPTIONS = _TOKENIZER_LOAD_EXCEPTIONS + (StrictDataclassError,)
 
 _KNOWN_LOAD_WARNING_SUPPRESSIONS = [
     (
@@ -143,12 +148,18 @@ def load(
     @staticmethod
     def _load_tokenizer(pretrained_model_name_or_path: str, **kwargs):
         Tokenicer._install_tokenizer_compatibility_shims()
+        load_kwargs = dict(kwargs)
+        # Let Transformers repair affected Mistral-family pre-tokenizer regexes by
+        # default. Transformers applies this only when its compatibility detector
+        # matches, and callers can retain the serialized regex with an explicit
+        # ``fix_mistral_regex=False``.
+        load_kwargs.setdefault("fix_mistral_regex", True)
         try:
             # Keep the normal Transformers path first so standard checkpoints behave unchanged.
-            return AutoTokenizer.from_pretrained(pretrained_model_name_or_path, **kwargs)
+            return AutoTokenizer.from_pretrained(pretrained_model_name_or_path, **load_kwargs)
         except _TOKENIZER_LOAD_EXCEPTIONS:
             overrides = tokenizer_special_token_overrides(pretrained_model_name_or_path)
-            retry_kwargs = dict(kwargs)
+            retry_kwargs = dict(load_kwargs)
             retry_kwargs.update(overrides)
 
             tokenizer_cls_name = tokenizer_class_name(pretrained_model_name_or_path)
@@ -185,10 +196,10 @@ def _load_tokenizer(pretrained_model_name_or_path: str, **kwargs):
                         pretrained_model_name_or_path,
                     )
 
-            if kwargs.get("trust_remote_code", False) or not has_custom_tokenizer_code(pretrained_model_name_or_path):
+            if load_kwargs.get("trust_remote_code", False) or not has_custom_tokenizer_code(pretrained_model_name_or_path):
                 raise
 
-            retry_kwargs = dict(kwargs)
+            retry_kwargs = dict(load_kwargs)
             retry_kwargs["trust_remote_code"] = True
             # Local checkpoints with custom tokenizer code can still succeed once remote code is explicitly allowed.
             logger.warning(