fix: Japanese punctuation problem

Author: bdlukaa

example/bin/romanize_example.dart

@@ -89,70 +89,38 @@ void _hebrew() {
 }
 
 void _multiLanguages() {
-  // from home nct u
-  final multiLanguagesText = '''
-Ooh, yeah-yeah-yeah
-I remember like it's yesterday, oh-no
-외로움에 힘들던 그때 oh-ooh
-낯설기만 하던 이 공기도
-두렵기만 하던 이 떨림도
-Now I know
-그 어렸던 마음까지 모두
-추억이 되게 해준 너
-Cause of us I'm feeling strong again
-서로 믿어줄 때면
-이곳에 날 당연하게 해
-When we shine bright
-I'm alive in the CT 날 노래해
-이 조명 아래 서로를 바라보면
-나도 몰래 웃게 돼 다 잊게 돼 yeah
-Cause I'm not alone
-내게 따듯한 집이 돼준 너
-어제와 지금의 나 또 다가올 내일 우리
-It all starts from home
-彩虹是和你再见的誓言
-飘过初雪到仲夏的夜
-直到和你眼神交会
-拥抱让我遗落寂寞 ooh
-随时能和你连结 为你回应
-見つけたよ ココロが安らぐ
-My home, my own (My own)
-映照着自己 感受到笑意都一往如初
-Once again (Ooh)
-僕ら強くなれる
-活成彼此的阳光
-我的存在自然而耀眼
-When we shine bright
-I'm alive in the CT 날 노래해
-이 조명 아래 서로를 바라보면
-나도 몰래 웃게 돼 다 잊게 돼 yeah
-Cause I'm not alone
-내게 따듯한 집이 돼준 너
-어제와 지금의 나 또 다가올 내일 우리
-It all starts from home
-이젠 길을 잃을 두려움도
-겁내기 바빴던 날들도
-Now It's all gone and I
-Found a reason to be myself
-Know that you are not alone anymore
-When we shine bright
-I'm alive in the CT 날 노래해
-이 조명 아래 서로를 바라보면
-나도 몰래 웃게 돼 다 잊게 돼 yeah
-Cause I'm not alone
-내게 따듯한 집이 돼준 너 (어제와 지금의 나)
-어제와 지금의 나 또 다가올 내일 우리
-It all starts from home
-Na-na-na, na-na-na, na-na-na
-From home (From home)
-Na-na-na, na-na-na, from home
-Yeah-yeah-yeah
-Na-na-na (Na-na-na)
-Na-na-na (Na-na-na)
-Na-na-na
-From home
-And we start from here our home
-''';
+  final multiLanguagesText = '''Mixed Script Stress Test:
+-------------------------
+1. CJK Ambiguity (Should detect Chinese vs Japanese context):
+   中文 (Chinese) vs 日本語 (Japanese)
+   你好世界 (Hello World - CN) mixed with こんにちは (Hello - JP)
+   東京 (Tokyo - JP/CN chars) vs 北京 (Beijing - CN)
+   
+2. RTL/LTR Alternation (Arabic/Hebrew/English):
+   English -> العربية -> English -> עִבְרִית -> English
+   Start: مرحبا (Marhaban) -> Middle: שָׁלוֹם (Shalom) -> End.
+   Complex: "The letter 'ا' (Alif) and 'א' (Alef) start alphabets."
+
+3. Diacritic Heavy (Vowelization Stress):
+   Arabic: كَتَبَ الْوَلَدُ الرِّسَالَةَ (Kataba al-waladu ar-risalata)
+   Hebrew: בְּרֵאשִׁית בָּרָא אֱלֹהִים אֵת הַשָּׁמַיִם וְאֵת הָאָרֶץ (Genesis 1:1)
+   
+4. Cyrillic & Extended Latin:
+   Russian: Съешь же ещё этих мягких французских булок, да выпей чаю.
+   Mixed: "Privet (Привет) means Hello."
+
+5. Rapid Switching (Tokenization Stress):
+   KR:안녕하세요_JP:こんにちは_CN:你好_RU:Привет_AR:مرحبا_HE:שָׁלוֹם
+   123٤٥٦(Numbers)abc가나다(Hangul)カキク(Katakana)
+
+6. Long Paragraph (Performance):
+   Lorem ipsum dolor sit amet. 但是，如果我们切换到中文。
+   Then back to English. そして日本語に切り替えます。
+   Suddenly, Cyrillic appears: Внезапно появляется кириллица.
+   Followed by Arabic: ويتبع ذلك العربية.
+   And finally Hebrew: ולבסוף עברית.
+   
+   End of Stress Test.''';
   final multiLanguagesOutput = TextRomanizer.romanize(multiLanguagesText);
   print('Multi Languages Romanization: \n$multiLanguagesOutput');
 }

lib/romanize.dart

@@ -48,14 +48,15 @@ class TextRomanizer {
   static final Set<Romanizer> romanizers = <Romanizer>{
     HangulRomanizer(),
 
+    CyrillicRomanizer(),
+    ArabicRomanizer(),
+    HebrewRomanizer(),
+
     // Note: Chinese is placed before Japanese. Pure Kanji (e.g., "東京") will
     // default to Chinese. Mixed Japanese (Kanji + Kana) will be rejected by
     // ChineseRomanizer.isValid and fall through to JapaneseRomanizer.
     ChineseRomanizer(),
     JapaneseRomanizer(),
-    CyrillicRomanizer(),
-    ArabicRomanizer(),
-    HebrewRomanizer(),
   };
 
   /// Automatically detects the language of the input text.
@@ -75,7 +76,7 @@ class TextRomanizer {
     final String input, [
     Set<Romanizer>? romanizers,
   ]) {
-    if (input.isEmpty || !RegExp(r'\S').hasMatch(input)) {
+    if (input.isEmpty) {
       return const EmptyRomanizer();
     }
 
@@ -108,7 +109,7 @@ class TextRomanizer {
     return romanizers.where((romanizer) => romanizer.isValid(input)).toSet();
   }
 
-  static final _separatorPattern = RegExp(r'[\s\p{P}_]+');
+  static final _separatorPattern = RegExp(r'[\s\p{P}_()]+');
 
   /// Romanizes the input text by processing each word separately.
   ///
@@ -135,7 +136,7 @@ class TextRomanizer {
     if (languages.length == 1) {
       return languages.first.romanize(input);
     }
-    final wordCache = <String, Romanizer>{};
+    final wordCache = <String, String>{};
 
     return input.splitMapJoin(
       _separatorPattern,
@@ -144,13 +145,15 @@ class TextRomanizer {
       // Handle the content (words):
       onNonMatch: (String word) {
         if (word.isEmpty) return '';
-
-        final romanizer = wordCache.putIfAbsent(
+        return wordCache.putIfAbsent(
           word,
-          () => detectLanguage(word, languages),
+          () => detectLanguage(
+            word,
+            // [detectLanguages] may not return all languages in the set, so
+            // check for all languages.
+            // languages,
+          ).romanize(word),
         );
-
-        return romanizer.romanize(word);
       },
     );
   }

lib/src/romanizers/chinese.dart

@@ -76,7 +76,9 @@ class ChineseRomanizer extends Romanizer {
     return buffer.toString();
   }
 
-  static final _kanaPattern = RegExp(r'[\u3040-\u309F\u30A0-\u30FF]');
+  static final _kanaPattern = RegExp(
+    r'[\u3040-\u309F\u30A0-\u30FF\uFF66-\uFF9F]',
+  );
 
   /// Validates if the input string contains Chinese characters.
   ///

lib/src/romanizers/japanese.dart

@@ -39,18 +39,28 @@ class JapaneseRomanizer extends Romanizer {
   String romanize(String input) {
     if (input.trim().isEmpty) return input;
     if (_tokenizer != null) {
-      final tokens = _tokenizer!.tokenize(input);
-      final buffer = StringBuffer();
-      for (final token in tokens) {
-        if (token['reading'] != null &&
-            token['reading'].isNotEmpty &&
-            token['reading'] != '*') {
-          buffer.write(token['reading']);
-        } else {
-          buffer.write(token['surface_form']);
+      try {
+        final tokens = _tokenizer!.tokenize(
+          input
+          // Kurumoji has a problem with full stop "。" characters.
+          // Replace them with "."
+          .replaceAll('。', '.'),
+        );
+        final buffer = StringBuffer();
+        for (final token in tokens) {
+          if (token.isEmpty) continue;
+          if (token['reading'] != null &&
+              token['reading'].isNotEmpty &&
+              token['reading'] != '*') {
+            buffer.write(kanaKit.toRomaji(token['reading']));
+          } else {
+            buffer.write(kanaKit.toRomaji(token['surface_form']));
+          }
         }
+        return buffer.toString();
+      } catch (_) {
+        // Fallback to direct romanization if tokenization fails
       }
-      return kanaKit.toRomaji(buffer.toString());
     }
     return kanaKit.toRomaji(input);
   }