feat: Improve analysis by separating matches into chunks

Author: bdlukaa

lib/romanize.dart

@@ -110,8 +110,20 @@ class TextRomanizer {
     return romanizers.where((romanizer) => romanizer.isValid(input)).toSet();
   }
 
-  static final _separatorPattern = RegExp(
-    r'[^\p{L}\p{N}\p{M}]+',
+  static final _separatorPattern = RegExp(r'[\s\p{P}\p{S}]+', unicode: true);
+
+  // Matches chunks of text that share the same script system.
+  // Grouping Kanji/Kana ensures Japanese sentences stay together.
+  static final RegExp _scriptChunkPattern = RegExp(
+    r'('
+    r'[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]+|' // CJK (Keep together)
+    r'[\p{Script=Hangul}]+|' // Korean
+    r'[\p{Script=Arabic}]+|' // Arabic
+    r'[\p{Script=Hebrew}]+|' // Hebrew
+    r'[\p{Script=Cyrillic}]+|' // Cyrillic
+    r'[\p{Script=Latin}]+|' // Latin
+    r'[0-9]+' // ASCII Digits
+    r')',
     unicode: true,
   );
 
@@ -186,6 +198,7 @@ class TextRomanizer {
     final parts = <RomanizedText>[];
     final wordCache = <String, RomanizedText>{};
 
+    // 1. Outer Split: Handles spaces and punctuation
     input.splitMapJoin(
       _separatorPattern,
       onMatch: (Match match) {
@@ -198,18 +211,47 @@ class TextRomanizer {
         );
         return match[0]!;
       },
-      // Handle the content (words):
       onNonMatch: (String word) {
         if (word.isEmpty) return '';
-        final romanizedPart = wordCache.putIfAbsent(word, () {
-          final romanizer = detectLanguage(word);
-          return RomanizedText(
-            rawText: word,
-            language: romanizer.language,
-            romanizedText: romanizer.romanize(word),
+
+        // 2. Inner Split: Handles mixed scripts (e.g., "abc가나다")
+        // We find all "chunks" of consistent script within the word.
+        final matches = _scriptChunkPattern.allMatches(word);
+
+        // Track position to handle any un-matched gaps (symbols skipped by regex?)
+        int currentPos = 0;
+
+        for (final m in matches) {
+          // Handle gap if any (rare, but good for safety)
+          if (m.start > currentPos) {
+            final gap = word.substring(currentPos, m.start);
+            parts.add(
+              RomanizedText(rawText: gap, language: '', romanizedText: gap),
+            );
+          }
+
+          final chunk = m[0]!;
+          final romanizedPart = wordCache.putIfAbsent(chunk, () {
+            final romanizer = detectLanguage(chunk);
+            return RomanizedText(
+              rawText: chunk,
+              language: romanizer.language,
+              romanizedText: romanizer.romanize(chunk),
+            );
+          });
+          parts.add(romanizedPart);
+
+          currentPos = m.end;
+        }
+
+        // Handle trailing characters
+        if (currentPos < word.length) {
+          final tail = word.substring(currentPos);
+          parts.add(
+            RomanizedText(rawText: tail, language: '', romanizedText: tail),
           );
-        });
-        parts.add(romanizedPart);
+        }
+
         return word;
       },
     );

test/romanize_test.dart

@@ -470,6 +470,95 @@ void main() {
         final japanesePart = result.firstWhere((r) => r.rawText == 'こんにちは');
         expect(japanesePart.language, equals('japanese'));
       });
+
+      test('should split mixed Latin and Hangul (abc가나다)', () {
+        const input = 'abc가나다';
+        final result = TextRomanizer.analyze(input);
+
+        // Expect: [Latin(abc), Hangul(가나다)]
+        expect(result, hasLength(2));
+
+        expect(result[0].rawText, equals('abc'));
+        expect(result[0].language, equals('empty'));
+
+        expect(result[1].rawText, equals('가나다'));
+        expect(result[1].language, equals('korean'));
+      });
+
+      test('should split mixed Numbers and Scripts', () {
+        // "Room101" -> Room (Latin), 101 (Digits)
+        const input = 'Room101';
+        final result = TextRomanizer.analyze(input);
+
+        expect(result, hasLength(2));
+        expect(result[0].rawText, equals('Room'));
+        expect(result[1].rawText, equals('101'));
+      });
+
+      test('should NOT split natural Japanese (Kanji + Kana)', () {
+        // "日本語です" (Kanji + Hiragana) should stay as ONE chunk
+        const input = '日本語です';
+        final result = TextRomanizer.analyze(input);
+
+        expect(result, hasLength(1));
+        expect(result[0].rawText, equals('日本語です'));
+        expect(result[0].language, equals('japanese'));
+      });
+
+      test('should handle a complex snippet', () {
+        // Input: 123٤٥٦(Numbers)abc가나다(Hangul)カキク(Katakana)
+        // 123   -> Digits
+        // ٤٥٦   -> Arabic
+        // (     -> Separator
+        // Numbers -> Latin
+        // )     -> Separator
+        // abc   -> Latin
+        // 가나다 -> Korean
+        // (     -> Separator
+        // Hangul -> Latin
+        // )     -> Separator
+        // ... etc
+
+        const input = '123٤٥٦(Numbers)abc가나다';
+        final result = TextRomanizer.analyze(input);
+
+        // 1. Check "123٤٥٦" split
+        // ASCII digits [0-9] are separate from Arabic script in the regex
+        final part123 = result.firstWhere((r) => r.rawText == '123');
+        final partArabicNum = result.firstWhere((r) => r.rawText == '٤٥٦');
+
+        expect(part123, isNotNull);
+        expect(partArabicNum, isNotNull);
+        expect(partArabicNum.language, equals('arabic'));
+
+        // 2. Check "abc가나다" split
+        // Find the sequence where abc is followed immediately by 가나다
+        final indexAbc = result.indexWhere((r) => r.rawText == 'abc');
+        expect(indexAbc, isNot(-1));
+
+        final partKorean = result[indexAbc + 1];
+        expect(partKorean.rawText, equals('가나다'));
+        expect(partKorean.language, equals('korean'));
+      });
+
+      test('should split CJK vs Latin', () {
+        const input = 'Hello你好';
+        final result = TextRomanizer.analyze(input);
+
+        expect(result, hasLength(2));
+        expect(result[0].rawText, equals('Hello'));
+        expect(result[1].rawText, equals('你好'));
+      });
+
+      test('should split Cyrillic vs Latin', () {
+        const input = 'TestТест'; // Latin 'Test', Cyrillic 'Test'
+        final result = TextRomanizer.analyze(input);
+
+        expect(result, hasLength(2));
+        expect(result[0].rawText, equals('Test'));
+        expect(result[1].rawText, equals('Тест'));
+        expect(result[1].language, equals('cyrillic'));
+      });
     });
 
     group('supportedLanguages', () {