feat: TextRomanizer.analyze

Author: bdlukaa

CHANGELOG.md

@@ -1,3 +1,7 @@
+## [next]
+
+- `TextRomanizer.analyze`, which analyzes the input text and provides detailed information about detected languages and romanization results for each segment.
+
 ## 0.0.2
 
 - Japanese Kanji support added using [`kuromoji`](https://pub.dev/packages/kuromoji).

lib/romanize.dart

@@ -1,6 +1,7 @@
 library;
 
 import 'package:romanize/src/romanize_base.dart';
+import 'package:romanize/src/romanized_text.dart';
 import 'package:romanize/src/romanizers/arabic.dart';
 import 'package:romanize/src/romanizers/chinese.dart';
 import 'package:romanize/src/romanizers/cyrillic.dart';
@@ -9,6 +10,7 @@ import 'package:romanize/src/romanizers/japanese.dart';
 import 'package:romanize/src/romanizers/korean.dart';
 
 export 'src/romanize_base.dart';
+export 'src/romanized_text.dart';
 export 'src/romanizers/arabic.dart';
 export 'src/romanizers/chinese.dart';
 export 'src/romanizers/cyrillic.dart';
@@ -46,10 +48,9 @@ class TextRomanizer {
   ///
   /// The romanizers are checked in order when auto-detecting the language.
   static final Set<Romanizer> romanizers = <Romanizer>{
-    HangulRomanizer(),
-
-    CyrillicRomanizer(),
     ArabicRomanizer(),
+    CyrillicRomanizer(),
+    HangulRomanizer(),
     HebrewRomanizer(),
 
     // Note: Chinese is placed before Japanese. Pure Kanji (e.g., "東京") will
@@ -109,7 +110,10 @@ class TextRomanizer {
     return romanizers.where((romanizer) => romanizer.isValid(input)).toSet();
   }
 
-  static final _separatorPattern = RegExp(r'[\s\p{P}_()]+');
+  static final _separatorPattern = RegExp(
+    r'[^\p{L}\p{N}\p{M}]+',
+    unicode: true,
+  );
 
   /// Romanizes the input text by processing each word separately.
   ///
@@ -158,6 +162,61 @@ class TextRomanizer {
     );
   }
 
+  /// Analyzes the input text and returns a list of [RomanizedText] parts.
+  ///
+  /// This method splits the input by spaces and punctuation, romanizing each
+  /// word independently while preserving the original structure of the text.
+  /// Each word is auto-detected and romanized according to its language.
+  ///
+  /// Example:
+  /// ```dart
+  /// // Multi-language text
+  /// final result = TextRomanizer.analyze('你好 Hello 안녕');
+  /// print(result);
+  /// // [
+  /// //  RomanizedText(rawText: '你好', language: 'japanese', romanizedText: 'ni hao'),
+  /// //  RomanizedText(rawText: 'Hello', language: '', romanizedText: 'Hello'),
+  /// //  RomanizedText(rawText: '안녕', language: 'korean', romanizedText: 'annyeong'),
+  /// // ]
+  /// ```
+  ///
+  /// Uses a cache to avoid redundant language detection for repeated words.
+  /// This improves performance for long texts.
+  static List<RomanizedText> analyze(String input) {
+    final parts = <RomanizedText>[];
+    final wordCache = <String, RomanizedText>{};
+
+    input.splitMapJoin(
+      _separatorPattern,
+      onMatch: (Match match) {
+        parts.add(
+          RomanizedText(
+            rawText: match[0]!,
+            language: '',
+            romanizedText: match[0]!,
+          ),
+        );
+        return match[0]!;
+      },
+      // Handle the content (words):
+      onNonMatch: (String word) {
+        if (word.isEmpty) return '';
+        final romanizedPart = wordCache.putIfAbsent(word, () {
+          final romanizer = detectLanguage(word);
+          return RomanizedText(
+            rawText: word,
+            language: romanizer.language,
+            romanizedText: romanizer.romanize(word),
+          );
+        });
+        parts.add(romanizedPart);
+        return word;
+      },
+    );
+
+    return parts;
+  }
+
   /// Returns a [Romanizer] for the specified language.
   ///
   /// The language name is case-insensitive. For example, 'Korean', 'korean',
@@ -168,7 +227,7 @@ class TextRomanizer {
   /// Example:
   /// ```dart
   /// final romanizer = TextRomanizer.forLanguage('japanese');
-  /// final result = romanizer.romanize('こんにちは');
+  /// final result = romanizer.romanize('こんにちは'); // konnichiwa
   /// ```
   static Romanizer forLanguage(String language) {
     if (language.trim().isEmpty) {

lib/src/romanized_text.dart

@@ -0,0 +1,22 @@
+/// A class representing the romanized text along with its original form and the
+/// language used for romanization.
+///
+/// See also:
+///
+///  * [TextRomanizer.analyze], which returns a list of [RomanizedText] parts.
+class RomanizedText {
+  /// The original raw text.
+  final String rawText;
+
+  /// The romanizer used to convert the text.
+  final String language;
+
+  /// The romanized text.
+  final String romanizedText;
+
+  const RomanizedText({
+    required this.rawText,
+    required this.language,
+    required this.romanizedText,
+  });
+}

test/romanize_test.dart

@@ -383,6 +383,95 @@ void main() {
       });
     });
 
+    group('analyze', () {
+      test('should analyze multi-language text correctly', () {
+        const input = '你好 Hello 안녕';
+        final result = TextRomanizer.analyze(input);
+
+        // Expect: [Word(你好), Sep( ), Word(Hello), Sep( ), Word(안녕)]
+        expect(result, hasLength(5));
+
+        // 1. Chinese/Japanese word
+        expect(result[0].rawText, equals('你好'));
+        // Note: Language detection for short strings like '你好' might be 'chinese' or 'japanese'
+        expect(result[0].language, isIn(['chinese', 'japanese']));
+        expect(result[0].romanizedText, isNotEmpty);
+
+        // 2. Separator (Space)
+        expect(result[1].rawText, equals(' '));
+        expect(
+          result[1].language,
+          isEmpty,
+        ); // Separators have empty string language
+        expect(result[1].romanizedText, equals(' '));
+
+        // 3. English word (Unsupported/Empty)
+        expect(result[2].rawText, equals('Hello'));
+        expect(
+          result[2].language,
+          equals('empty'),
+        ); // detectLanguage returns 'empty' for unsupported
+        expect(result[2].romanizedText, equals('Hello'));
+
+        // 4. Separator (Space)
+        expect(result[3].rawText, equals(' '));
+
+        // 5. Korean word
+        expect(result[4].rawText, equals('안녕'));
+        expect(result[4].language, equals('korean'));
+        expect(result[4].romanizedText, isNotEmpty);
+      });
+
+      test('should preserve punctuation and sentence structure', () {
+        const input = 'Hello, World!';
+        final result = TextRomanizer.analyze(input);
+
+        final reconstructed = result.map((r) => r.rawText).join();
+        expect(reconstructed, equals(input));
+      });
+
+      test('should handle repeated words consistently (caching)', () {
+        const input = '안녕 & 안녕';
+        final result = TextRomanizer.analyze(input);
+
+        final firstWord = result.first;
+        final lastWord = result.last;
+
+        expect(firstWord.rawText, equals('안녕'));
+        expect(lastWord.rawText, equals('안녕'));
+
+        // The romanization should be identical
+        expect(firstWord.romanizedText, equals(lastWord.romanizedText));
+        expect(firstWord.language, equals(lastWord.language));
+      });
+
+      test('should return empty list for empty input', () {
+        final result = TextRomanizer.analyze('');
+        expect(result, isEmpty);
+      });
+
+      test('should treat whitespace-only input as separators', () {
+        const input = '   ';
+        final result = TextRomanizer.analyze(input);
+
+        expect(result, isNotEmpty);
+        expect(result.first.rawText, equals(input));
+        expect(result.first.language, isEmpty); // Separator
+      });
+
+      test('should identify specific languages correctly in a sentence', () {
+        // Korean + Punctuation + Japanese
+        const input = '안녕하세요. こんにちは。';
+        final result = TextRomanizer.analyze(input);
+
+        final koreanPart = result.firstWhere((r) => r.rawText == '안녕하세요');
+        expect(koreanPart.language, equals('korean'));
+
+        final japanesePart = result.firstWhere((r) => r.rawText == 'こんにちは');
+        expect(japanesePart.language, equals('japanese'));
+      });
+    });
+
     group('supportedLanguages', () {
       test('should return list of supported languages', () {
         final languages = TextRomanizer.supportedLanguages;