fix: handle Unicode punctuation post-processing (#3238)

Author: LauraGPT

fun_text_processing/text_normalization/data_loader_utils.py

@@ -284,7 +284,7 @@ def post_process_punct(input: str, normalized_text: str, add_unicode_punct: bool
     Args:
         input: input text (original input to the NN, before normalization or tokenization)
         normalized_text: output text (output of the TN NN model)
-        add_unicode_punct: set to True to handle unicode punctuation marks as well as default string.punctuation (increases post processing time)
+        add_unicode_punct: set to True to handle unicode punctuation marks as well as default string.punctuation
     """
     # in the post-processing WFST graph "``" are repalced with '"" quotes (otherwise single quotes "`" won't be handled correctly)
     # this function fixes spaces around them based on input sequence, so here we're making the same double quote replacement
@@ -296,12 +296,11 @@ def post_process_punct(input: str, normalized_text: str, add_unicode_punct: bool
     punct_marks = [x for x in string.punctuation if x in input]
 
     if add_unicode_punct:
-        punct_unicode = [
-            chr(i)
-            for i in range(sys.maxunicode)
-            if category(chr(i)).startswith("P") and chr(i) not in punct_default and chr(i) in input
-        ]
-        punct_marks = punct_marks.extend(punct_unicode)
+        punct_marks.extend(
+            char
+            for char in dict.fromkeys(input)
+            if char not in string.punctuation and category(char).startswith("P")
+        )
 
     for punct in punct_marks:
         try:

tests/test_text_normalization_punctuation.py

@@ -0,0 +1,16 @@
+from fun_text_processing.text_normalization.data_loader_utils import post_process_punct
+
+
+def test_post_process_punct_preserves_ascii_punctuation_spacing():
+    assert post_process_punct("test' example", "test 'example") == "test' example"
+
+
+def test_post_process_punct_handles_unicode_punctuation():
+    assert (
+        post_process_punct(
+            "\u4f60\u597d\uff0c\u4e16\u754c\uff01",
+            "\u4f60\u597d \uff0c \u4e16\u754c \uff01",
+            add_unicode_punct=True,
+        )
+        == "\u4f60\u597d\uff0c\u4e16\u754c\uff01"
+    )