|
1 | 1 | // Lucene version compatibility level 4.8.1 |
2 | 2 | #if FEATURE_BREAKITERATOR |
3 | | -using ICU4N.Support.Text; |
4 | 3 | using ICU4N.Text; |
5 | 4 | using J2N; |
6 | 5 | using Lucene.Net.Analysis.TokenAttributes; |
7 | 6 | using Lucene.Net.Analysis.Util; |
8 | 7 | using Lucene.Net.Support; |
9 | | -using Lucene.Net.Support.Threading; |
10 | 8 | using Lucene.Net.Util; |
11 | 9 | using System; |
12 | 10 | using System.Collections.Generic; |
13 | 11 | using System.Globalization; |
14 | 12 | using System.IO; |
15 | | -using System.Text.RegularExpressions; |
16 | 13 |
|
17 | 14 | namespace Lucene.Net.Analysis.Th |
18 | 15 | { |
@@ -44,6 +41,19 @@ namespace Lucene.Net.Analysis.Th |
44 | 41 | /// <summary> |
45 | 42 | /// Tokenizer that use <see cref="BreakIterator"/> to tokenize Thai text. |
46 | 43 | /// </summary> |
| 44 | + /// <remarks> |
| 45 | + /// This is an attempt to mimic the behavior of the JDK's <c>java.Text.BreakIterator</c> approach |
| 46 | + /// to tokenizing Thai text. While it passes the Lucene tests, there may be innumerable differences |
| 47 | + /// between this implementation and the one in the JDK. |
| 48 | + /// <para/> |
| 49 | + /// Unlike the JDK, this implementation is guaranteed to be stable across all supported target frameworks. |
| 50 | + /// While it does use ICU4N's <see cref="RuleBasedBreakIterator"/>, this implementation doesn't follow |
| 51 | + /// the UAX #29 specification (http://unicode.org/reports/tr29) and is not guaranteed to behave the same as |
| 52 | + /// either the one in the JDK or in ICU4J. |
| 53 | + /// <para/> |
| 54 | + /// This implementation is provided primarily for API compatibility with Lucene. If strict Unicode compliance |
| 55 | + /// is desired, it is highly recommended to use the <see cref="Icu.Segmentation.ICUTokenizer"/> instead. |
| 56 | + /// </remarks> |
47 | 57 | public class ThaiTokenizer : SegmentingTokenizerBase |
48 | 58 | { |
49 | 59 | // LUCENENET specific - DBBI_AVAILABLE removed because ICU always has a dictionary-based BreakIterator |
|
0 commit comments