Skip to content

Commit 44a5ed8

Browse files
paulirwinclaude
paulirwin
and
claude
committed
Finish port of Lucene.Net.Collation functionality, #1343
Complete the abandoned WIP under Lucene.Net.Analysis.Common/Collation, which was gated behind an undefined FEATURE_COLLATION flag and referenced the icu.net package (never a solution dependency, so it never compiled). - Reimplement CollationKeyAnalyzer, CollationAttributeFactory, CollationKeyFilter, CollationKeyFilterFactory and CollatedTermAttributeImpl on the BCL platform collator (System.Globalization.CompareInfo / SortKey / CompareOptions), mirroring Java's java.text.Collator. Remove all FEATURE_COLLATION guards; no new dependency. - CompareInfo has no tailoring API, so custom RuleBasedCollator rules are not supported: CollationKeyFilterFactory's "custom" path throws NotSupportedException (use the ICU package for custom rulesets). - Normalize terms to NFC before generating sort keys so decomposed and precomposed input collate consistently across the NLS (.NET Framework) and ICU (.NET 5+) globalization backends. - Port the collation tests to CompareInfo. TestCollationKeySort's French ordering is backend-dependent (NLS -> EACGI, matching upstream; ICU -> ECAGI), so it is detected at runtime via CollationTestBase.FranceResult. - Re-enable the platform-collator implementation in the benchmark NewCollationAnalyzerTask (the upstream "JDK" impl, renamed to "BCL"; "jdk" kept as a legacy alias for Java .alg compatibility) and cover both impls in TestPerfTasksLogic.TestCollator. - Port Collation/package.md examples and caveats to .NET idioms. Verified: full Tests.Analysis.Common and Tests.Benchmark suites green on net10.0 and net48. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent e2e5d97 commit 44a5ed8

12 files changed

Lines changed: 400 additions & 533 deletions

File tree

‎src/Lucene.Net.Analysis.Common/Collation/CollationAttributeFactory.cs‎

Lines changed: 47 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
// Lucene version compatibility level 4.8.1
2-
#if FEATURE_COLLATION
3-
using Icu.Collation;
42
using Lucene.Net.Collation.TokenAttributes;
53
using Lucene.Net.Util;
6-
using System.Reflection;
4+
using System.Globalization;
75

86
namespace Lucene.Net.Collation
97
{
@@ -30,56 +28,75 @@ namespace Lucene.Net.Collation
3028
/// encodes the bytes as an index term.
3129
/// </para>
3230
/// <para>
33-
/// <strong>WARNING:</strong> Make sure you use exactly the same <see cref="Collator"/> at
31+
/// <strong>WARNING:</strong> Make sure you use exactly the same <see cref="CompareInfo"/> at
3432
/// index and query time -- <see cref="System.Globalization.SortKey"/>s are only comparable when produced by
35-
/// the same <see cref="Collator"/>. Since <see cref="RuleBasedCollator"/>s are not
33+
/// the same <see cref="CompareInfo"/>. Since the platform collator is not
3634
/// independently versioned, it is unsafe to search against stored
3735
/// <see cref="System.Globalization.SortKey"/>s unless the following are exactly the same (best practice is
3836
/// to store this information with the index and check that they remain the
3937
/// same at query time):
4038
/// </para>
4139
/// <list type="number">
42-
/// <item><description>JVM vendor</description></item>
43-
/// <item><description>JVM version, including patch version</description></item>
40+
/// <item><description>The .NET runtime version</description></item>
4441
/// <item><description>
45-
/// The language (and country and variant, if specified) of the Locale
46-
/// used when constructing the collator via
47-
/// <see cref="Collator.Create(System.Globalization.CultureInfo)"/>.
42+
/// The <see cref="CultureInfo"/> of the <see cref="CompareInfo"/> obtained via
43+
/// <see cref="CompareInfo.GetCompareInfo(string)"/>.
4844
/// </description></item>
4945
/// <item><description>
50-
/// The collation strength used - see <see cref="Collator.Strength"/>
46+
/// The <see cref="CompareOptions"/> used - which control the collation strength
47+
/// and Unicode normalization.
5148
/// </description></item>
5249
/// </list>
5350
/// <para>
54-
/// The <c>ICUCollationAttributeFactory</c> in the analysis-icu package
55-
/// uses ICU4J's Collator, which makes its
51+
/// The <c>ICUCollationAttributeFactory</c> in the Lucene.Net.ICU package
52+
/// uses ICU4N's Collator, which makes its
5653
/// version available, thus allowing collation to be versioned independently
57-
/// from the JVM. ICUCollationAttributeFactory is also significantly faster and
58-
/// generates significantly shorter keys than CollationAttributeFactory. See
59-
/// <a href="http://site.icu-project.org/charts/collation-icu4j-sun"
60-
/// >http://site.icu-project.org/charts/collation-icu4j-sun</a> for key
61-
/// generation timing and key length comparisons between ICU4J and
62-
/// <see cref="Collator"/> over several languages.
54+
/// from the .NET runtime. ICUCollationAttributeFactory generates
55+
/// significantly shorter keys than <see cref="CollationAttributeFactory"/> and exposes
56+
/// tailoring/custom rules that the platform collator does not support.
6357
/// </para>
6458
/// <para>
65-
/// CollationKeys generated by java.text.Collators are not compatible
66-
/// with those those generated by ICU Collators. Specifically, if you use
67-
/// CollationAttributeFactory to generate index terms, do not use
59+
/// <see cref="System.Globalization.SortKey"/>s generated by the platform collator are not compatible
60+
/// with those generated by ICU Collators. Specifically, if you use
61+
/// <see cref="CollationAttributeFactory"/> to generate index terms, do not use
6862
/// ICUCollationAttributeFactory on the query side, or vice versa.
6963
/// </para>
7064
/// </summary>
7165
public class CollationAttributeFactory : AttributeFactory
7266
{
73-
private readonly Collator collator;
67+
private readonly CompareInfo collator;
68+
private readonly CompareOptions options;
7469
private readonly AttributeFactory @delegate;
7570

7671
/// <summary>
7772
/// Create a <see cref="CollationAttributeFactory"/>, using
7873
/// <see cref="AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY"/> as the
7974
/// factory for all other attributes. </summary>
8075
/// <param name="collator"> <see cref="System.Globalization.SortKey"/> generator </param>
81-
public CollationAttributeFactory(Collator collator)
82-
: this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, collator)
76+
public CollationAttributeFactory(CompareInfo collator)
77+
: this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, collator, CompareOptions.None)
78+
{
79+
}
80+
81+
/// <summary>
82+
/// Create a <see cref="CollationAttributeFactory"/>, using
83+
/// <see cref="AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY"/> as the
84+
/// factory for all other attributes. </summary>
85+
/// <param name="collator"> <see cref="System.Globalization.SortKey"/> generator </param>
86+
/// <param name="options"> Collation options that control the collation strength and
87+
/// Unicode normalization (decomposition) of the generated sort key. </param>
88+
public CollationAttributeFactory(CompareInfo collator, CompareOptions options)
89+
: this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, collator, options)
90+
{
91+
}
92+
93+
/// <summary>
94+
/// Create a <see cref="CollationAttributeFactory"/>, using the supplied Attribute Factory
95+
/// as the factory for all other attributes. </summary>
96+
/// <param name="delegate"> Attribute Factory </param>
97+
/// <param name="collator"> <see cref="System.Globalization.SortKey"/> generator </param>
98+
public CollationAttributeFactory(AttributeFactory @delegate, CompareInfo collator)
99+
: this(@delegate, collator, CompareOptions.None)
83100
{
84101
}
85102

@@ -88,18 +105,20 @@ public CollationAttributeFactory(Collator collator)
88105
/// as the factory for all other attributes. </summary>
89106
/// <param name="delegate"> Attribute Factory </param>
90107
/// <param name="collator"> <see cref="System.Globalization.SortKey"/> generator </param>
91-
public CollationAttributeFactory(AttributeFactory @delegate, Collator collator)
108+
/// <param name="options"> Collation options that control the collation strength and
109+
/// Unicode normalization (decomposition) of the generated sort key. </param>
110+
public CollationAttributeFactory(AttributeFactory @delegate, CompareInfo collator, CompareOptions options)
92111
{
93112
this.@delegate = @delegate;
94113
this.collator = collator;
114+
this.options = options;
95115
}
96116

97117
public override Attribute CreateAttributeInstance<T>()
98118
{
99119
return typeof(T).IsAssignableFrom(typeof(CollatedTermAttributeImpl))
100-
? new CollatedTermAttributeImpl(this.collator)
120+
? new CollatedTermAttributeImpl(this.collator, this.options)
101121
: this.@delegate.CreateAttributeInstance<T>();
102122
}
103123
}
104124
}
105-
#endif

‎src/Lucene.Net.Analysis.Common/Collation/CollationKeyAnalyzer.cs‎

Lines changed: 39 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
// Lucene version compatibility level 4.8.1
2-
#if FEATURE_COLLATION
3-
using Icu.Collation;
42
using Lucene.Net.Analysis;
53
using Lucene.Net.Analysis.Core;
64
using Lucene.Net.Util;
75
using System;
6+
using System.Globalization;
87
using System.IO;
98

109
namespace Lucene.Net.Collation
@@ -37,41 +36,37 @@ namespace Lucene.Net.Collation
3736
/// it to be stored as an index term.
3837
/// </para>
3938
/// <para>
40-
/// <strong>WARNING:</strong> Make sure you use exactly the same <see cref="Collator"/> at
39+
/// <strong>WARNING:</strong> Make sure you use exactly the same <see cref="CompareInfo"/> at
4140
/// index and query time -- <see cref="System.Globalization.SortKey"/> are only comparable when produced by
42-
/// the same <see cref="Collator"/>. Since <c>java.text.RuleBasedCollators</c> are not
41+
/// the same <see cref="CompareInfo"/>. Since the platform collator is not
4342
/// independently versioned, it is unsafe to search against stored
4443
/// <see cref="System.Globalization.SortKey"/> unless the following are exactly the same (best practice is
4544
/// to store this information with the index and check that they remain the
4645
/// same at query time):
4746
/// </para>
4847
/// <list type="number">
49-
/// <item><description>JVM vendor</description></item>
50-
/// <item><description>JVM version, including patch version</description></item>
48+
/// <item><description>The .NET runtime version</description></item>
5149
/// <item><description>
52-
/// The language (and country and variant, if specified) of the Locale
53-
/// used when constructing the collator via
54-
/// <see cref="Collator.Create(System.Globalization.CultureInfo)"/>.
50+
/// The <see cref="CultureInfo"/> of the <see cref="CompareInfo"/> obtained via
51+
/// <see cref="CompareInfo.GetCompareInfo(string)"/>.
5552
/// </description></item>
5653
/// <item><description>
57-
/// The collation strength used - see <see cref="Collator.Strength"/>
54+
/// The <see cref="CompareOptions"/> used - which control the collation strength
55+
/// and Unicode normalization.
5856
/// </description></item>
5957
/// </list>
6058
/// <para>
61-
/// The <c>ICUCollationKeyAnalyzer</c> in the analysis-icu package
62-
/// uses ICU4J's Collator, which makes its
63-
/// its version available, thus allowing collation to be versioned
64-
/// independently from the JVM. ICUCollationKeyAnalyzer is also significantly
65-
/// faster and generates significantly shorter keys than CollationKeyAnalyzer.
66-
/// See <a href="http://site.icu-project.org/charts/collation-icu4j-sun"
67-
/// >http://site.icu-project.org/charts/collation-icu4j-sun</a> for key
68-
/// generation timing and key length comparisons between ICU4J and
69-
/// <see cref="Collator"/> over several languages.
59+
/// The <c>ICUCollationKeyAnalyzer</c> in the Lucene.Net.ICU package
60+
/// uses ICU4N's Collator, which makes its
61+
/// version available, thus allowing collation to be versioned
62+
/// independently from the .NET runtime. ICUCollationKeyAnalyzer
63+
/// generates significantly shorter keys than <see cref="CollationKeyAnalyzer"/> and
64+
/// exposes tailoring/custom rules that the platform collator does not support.
7065
/// </para>
7166
/// <para>
72-
/// CollationKeys generated by <see cref="Collator"/> are not compatible
73-
/// with those those generated by ICU Collators. Specifically, if you use
74-
/// CollationKeyAnalyzer to generate index terms, do not use
67+
/// <see cref="System.Globalization.SortKey"/>s generated by the platform collator are not compatible
68+
/// with those generated by ICU Collators. Specifically, if you use
69+
/// <see cref="CollationKeyAnalyzer"/> to generate index terms, do not use
7570
/// ICUCollationKeyAnalyzer on the query side, or vice versa.
7671
/// </para>
7772
/// <para>You must specify the required <see cref="LuceneVersion"/>
@@ -84,7 +79,8 @@ namespace Lucene.Net.Collation
8479
/// </summary>
8580
public sealed class CollationKeyAnalyzer : Analyzer
8681
{
87-
private readonly Collator collator;
82+
private readonly CompareInfo collator;
83+
private readonly CompareOptions options;
8884
private readonly CollationAttributeFactory factory;
8985
private readonly LuceneVersion matchVersion;
9086

@@ -93,20 +89,33 @@ public sealed class CollationKeyAnalyzer : Analyzer
9389
/// </summary>
9490
/// <param name="matchVersion"> See <see cref="CollationKeyAnalyzer"/> </param>
9591
/// <param name="collator"> <see cref="System.Globalization.SortKey"/> generator </param>
96-
public CollationKeyAnalyzer(LuceneVersion matchVersion, Collator collator)
92+
public CollationKeyAnalyzer(LuceneVersion matchVersion, CompareInfo collator)
93+
: this(matchVersion, collator, CompareOptions.None)
94+
{
95+
}
96+
97+
/// <summary>
98+
/// Create a new <see cref="CollationKeyAnalyzer"/>, using the specified collator.
99+
/// </summary>
100+
/// <param name="matchVersion"> See <see cref="CollationKeyAnalyzer"/> </param>
101+
/// <param name="collator"> <see cref="System.Globalization.SortKey"/> generator </param>
102+
/// <param name="options"> Collation options that control the collation strength and
103+
/// Unicode normalization (decomposition) of the generated sort key. </param>
104+
public CollationKeyAnalyzer(LuceneVersion matchVersion, CompareInfo collator, CompareOptions options)
97105
{
98106
this.matchVersion = matchVersion;
99107
this.collator = collator;
100-
this.factory = new CollationAttributeFactory(collator);
108+
this.options = options;
109+
this.factory = new CollationAttributeFactory(collator, options);
101110
}
102111

103-
[Obsolete("Use <seealso cref=\"CollationKeyAnalyzer#CollationKeyAnalyzer(LuceneVersion, Collator)\"/> and specify a version instead. This ctor will be removed in Lucene 5.0")]
104-
public CollationKeyAnalyzer(Collator collator)
105-
: this(LuceneVersion.LUCENE_31, collator)
112+
[Obsolete("Use CollationKeyAnalyzer.CollationKeyAnalyzer(LuceneVersion, CompareInfo) and specify a version instead. This ctor will be removed in Lucene 5.0")]
113+
public CollationKeyAnalyzer(CompareInfo collator)
114+
: this(LuceneVersion.LUCENE_31, collator, CompareOptions.None)
106115
{
107116
}
108117

109-
protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
118+
protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
110119
{
111120
#pragma warning disable 612, 618
112121
if (this.matchVersion.OnOrAfter(LuceneVersion.LUCENE_40))
@@ -120,10 +129,9 @@ protected override TokenStreamComponents CreateComponents(string fieldName, Text
120129
var tokenizer = new KeywordTokenizer(reader);
121130
return new TokenStreamComponents(tokenizer,
122131
#pragma warning disable 612, 618
123-
new CollationKeyFilter(tokenizer, this.collator));
132+
new CollationKeyFilter(tokenizer, this.collator, this.options));
124133
#pragma warning restore 612, 618
125134
}
126135
}
127136
}
128137
}
129-
#endif

0 commit comments

Comments
 (0)