fix(tokenizer): treat special-token strings as text to prevent batch crashes (#3110)

Fixes #3110.

Author: 14790897

funasr/models/sense_voice/whisper_lib/tokenizer.py

@@ -194,11 +194,24 @@ def __post_init__(self):
 
     def encode(self, text, **kwargs):
         """Encode.
-        
+
             Args:
                 text: Text tensor or string input.
                 **kwargs: Additional keyword arguments.
-            """
+
+        ``tiktoken`` rejects any text containing a special-token string
+        (e.g. ``<|no|>``, ``<|zh|>``, ``<|nospeech|>``) by default
+        (``disallowed_special="all"``). ASR models such as Fun-ASR-Nano
+        occasionally emit such special-token strings as part of their
+        transcription text; re-encoding that text for downstream tasks
+        (forced alignment, loss computation, ...) then crashes the whole
+        batch with a single bad sample (see issue #3110).
+
+        Treat special-token strings as ordinary text unless the caller
+        explicitly opts into a stricter policy. Callers that already pass
+        ``allowed_special`` / ``disallowed_special`` are left untouched.
+        """
+        kwargs.setdefault("disallowed_special", ())
         return self.encoding.encode(text, **kwargs)
 
     def decode(self, token_ids: List[int], **kwargs) -> str:

tests/test_sensevoice_tokenizer_special_tokens.py

@@ -0,0 +1,110 @@
+"""Unit tests for SenseVoice Tokenizer handling of special-token strings.
+
+Regression guard for issue #3110: ASR models such as Fun-ASR-Nano occasionally
+emit special-token strings (e.g. ``<|no|>``, a language tag) as part of the
+transcription text. Re-encoding that text via ``tiktoken.Encoding.encode``
+crashes by default (``disallowed_special="all"``), which takes down the whole
+batch on a single bad sample during forced alignment / loss computation.
+``Tokenizer.encode`` now treats special-token strings as ordinary text unless
+the caller explicitly opts into a stricter policy.
+
+These tests build a minimal in-memory tiktoken encoding, so they run without
+the Fun-ASR-Nano ``multilingual.tiktoken`` vocab (which ships with the model,
+not this repo) and without a GPU or model download.
+"""
+
+import unittest
+
+import tiktoken
+
+from funasr.models.sense_voice.whisper_lib.tokenizer import Tokenizer
+
+
+def _build_test_encoding() -> tiktoken.Encoding:
+    """A minimal byte-level BPE encoding with the specials Tokenizer needs.
+
+    Every single byte is its own token, so arbitrary text can be encoded
+    without the model's multilingual.tiktoken vocab file.
+    """
+    mergeable_ranks = {bytes([b]): b for b in range(256)}
+    n_vocab = len(mergeable_ranks)
+
+    # Specials required by Tokenizer.__post_init__ (startoftranscript/translate/
+    # transcribe) plus a few language tags. "<|no|>" is the Norwegian tag that
+    # triggered #3110.
+    required_specials = [
+        "<|startoftranscript|>",
+        "<|no|>",
+        "<|zh|>",
+        "<|en|>",
+        "<|translate|>",
+        "<|transcribe|>",
+        "<|startoflm|>",
+        "<|startofprev|>",
+        "<|nospeech|>",
+        "<|notimestamps|>",
+        "<|0.00|>",
+    ]
+    special_tokens = {}
+    for tok in required_specials:
+        special_tokens[tok] = n_vocab
+        n_vocab += 1
+
+    return tiktoken.Encoding(
+        name="test-encoding",
+        pat_str=(
+            r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| """
+            r"""?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
+        ),
+        mergeable_ranks=mergeable_ranks,
+        special_tokens=special_tokens,
+    )
+
+
+def _build_test_tokenizer() -> Tokenizer:
+    return Tokenizer(encoding=_build_test_encoding(), num_languages=1)
+
+
+class TestTokenizerEncodeSpecialTokens(unittest.TestCase):
+    def setUp(self):
+        self.tokenizer = _build_test_tokenizer()
+        self.no_id = self.tokenizer.special_tokens["<|no|>"]
+
+    def test_plain_text_encodes(self):
+        ids = self.tokenizer.encode("hello world")
+        self.assertIsInstance(ids, list)
+        self.assertTrue(ids)
+
+    def test_language_tag_in_text_does_not_raise(self):
+        # The exact failure from issue #3110: ASR output containing "<|no|>"
+        # used to crash with "disallowed special token '<|no|>'".
+        ids = self.tokenizer.encode("hello <|no|> world")
+        self.assertIsInstance(ids, list)
+        self.assertTrue(ids)
+
+    def test_nospeech_tag_in_text_does_not_raise(self):
+        ids = self.tokenizer.encode("<|nospeech|> something")
+        self.assertIsInstance(ids, list)
+        self.assertTrue(ids)
+
+    def test_language_tag_alone_does_not_raise(self):
+        ids = self.tokenizer.encode("<|no|>")
+        self.assertIsInstance(ids, list)
+        self.assertTrue(ids)
+
+    def test_caller_disallowed_special_is_respected(self):
+        # A caller that explicitly wants the strict behaviour must still get
+        # it (setdefault must not override an explicit kwargs).
+        with self.assertRaises(ValueError):
+            self.tokenizer.encode("<|no|>", disallowed_special="all")
+
+    def test_allowed_special_all_keeps_special_token_ids(self):
+        # funasr/models/sense_voice/whisper_lib/decoding.py calls
+        # tokenizer.encode(prompt, allowed_special="all"); the special token
+        # must still be encoded as its token id, not as raw bytes.
+        ids = self.tokenizer.encode("<|no|>", allowed_special="all")
+        self.assertEqual(ids, [self.no_id])
+
+
+if __name__ == "__main__":
+    unittest.main()