Skip to content

Commit 7bceef4

Browse files
authored
Bumped ICU4N to 60.1.0-alpha.440 (#1353)
1 parent cada5e7 commit 7bceef4

2 files changed

Lines changed: 14 additions & 4 deletions

File tree

.build/dependencies.props

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
https://github.com/apache/lucene-solr/tree/31d7ec7bbfdcd2c4cc61d9d35e962165410b65fe/lucene/analysis/icu/src/data/utr30
3535
Just make sure they are adjusted to the right version of ICU/Lucene.
3636
<ICU4NPackageVersion>[60.1,60.2)</ICU4NPackageVersion> -->
37-
<ICU4NPackageVersion>[60.1.0-alpha.438,60.1.0-alpha.446)</ICU4NPackageVersion>
37+
<ICU4NPackageVersion>[60.1.0-alpha.440,60.1.0-alpha.446)</ICU4NPackageVersion>
3838
<IKVMPackageVersion>8.7.5</IKVMPackageVersion>
3939
<IKVMMavenSdkPackageVersion>1.6.7</IKVMMavenSdkPackageVersion>
4040
<!-- J2N will break binary compatibility in 3.0.0 to fix the APIs of collection types -->

src/Lucene.Net.Analysis.Common/Analysis/Th/ThaiTokenizer.cs

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,15 @@
11
// Lucene version compatibility level 4.8.1
22
#if FEATURE_BREAKITERATOR
3-
using ICU4N.Support.Text;
43
using ICU4N.Text;
54
using J2N;
65
using Lucene.Net.Analysis.TokenAttributes;
76
using Lucene.Net.Analysis.Util;
87
using Lucene.Net.Support;
9-
using Lucene.Net.Support.Threading;
108
using Lucene.Net.Util;
119
using System;
1210
using System.Collections.Generic;
1311
using System.Globalization;
1412
using System.IO;
15-
using System.Text.RegularExpressions;
1613

1714
namespace Lucene.Net.Analysis.Th
1815
{
@@ -44,6 +41,19 @@ namespace Lucene.Net.Analysis.Th
4441
/// <summary>
4542
/// Tokenizer that use <see cref="BreakIterator"/> to tokenize Thai text.
4643
/// </summary>
44+
/// <remarks>
45+
/// This is an attempt to mimic the behavior of the JDK's <c>java.Text.BreakIterator</c> approach
46+
/// to tokenizing Thai text. While it passes the Lucene tests, there may be innumerable differences
47+
/// between this implementation and the one in the JDK.
48+
/// <para/>
49+
/// Unlike the JDK, this implementation is guaranteed to be stable across all supported target frameworks.
50+
/// While it does use ICU4N's <see cref="RuleBasedBreakIterator"/>, this implementation doesn't follow
51+
/// the UAX #29 specification (http://unicode.org/reports/tr29) and is not guaranteed to behave the same as
52+
/// either the one in the JDK or in ICU4J.
53+
/// <para/>
54+
/// This implementation is provided primarily for API compatibility with Lucene. If strict Unicode compliance
55+
/// is desired, it is highly recommended to use the <see cref="Icu.Segmentation.ICUTokenizer"/> instead.
56+
/// </remarks>
4757
public class ThaiTokenizer : SegmentingTokenizerBase
4858
{
4959
// LUCENENET specific - DBBI_AVAILABLE removed because ICU always has a dictionary-based BreakIterator

0 commit comments

Comments
 (0)