fix: repair KWS symbol cleanup regex

Author: LauraGPT

funasr/utils/kws_utils.py

@@ -7,7 +7,7 @@
 from typing import List, Optional, Tuple
 
 
-symbol_str = '[’!"#$%&\'()*+,-./:;<>=?@，。?★、…【】《》？“”‘’！[\\]^_`{|}~\s]+'
+symbol_str = r'''[’!"#$%&'()*+,\-./:;<>=?@，。?★、…【】《》？“”‘’！\[\]\\^_`{|}~\s]+'''
 
 
 def split_mixed_label(input_str):

tests/run_test.py

@@ -7,6 +7,11 @@
 from fnmatch import fnmatch
 
 
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if PROJECT_ROOT not in sys.path:
+    sys.path.insert(0, PROJECT_ROOT)
+
+
 def gather_test_cases(test_dir, pattern, list_tests):
     case_list = []
     for dirpath, dirnames, filenames in os.walk(test_dir):

tests/test_kws_utils.py

@@ -0,0 +1,23 @@
+import re
+import unittest
+
+from funasr.utils.kws_utils import query_token_set, split_mixed_label, symbol_str
+
+
+class KwsUtilsTest(unittest.TestCase):
+    def test_symbol_regex_strips_punctuation_and_whitespace(self):
+        self.assertEqual(re.sub(symbol_str, "", "abc!@# 中文\t★"), "abc中文")
+
+    def test_split_mixed_label_lowercases_ascii_words(self):
+        self.assertEqual(split_mixed_label("Hello中文"), ["hello", "中", "文"])
+
+    def test_query_token_set_strips_symbols_before_character_lookup(self):
+        symbol_table = {"a": 1, "中": 2, "<unk>": 0}
+        tokens, token_ids = query_token_set("A!中", symbol_table, {})
+
+        self.assertEqual(tokens, ("a", "中"))
+        self.assertEqual(token_ids, (1, 2))
+
+
+if __name__ == "__main__":
+    unittest.main()