Skip to content

Commit fb6fdb7

Browse files
paulirwinclaude
andcommitted
Fix per-process randomized string hashing in test-framework codec/similarity selection
MockRandomPostingsFormat, RandomCodec, and RandomSimilarityProvider chose a per-file/per-field int-stream factory, postings/docvalues format, and similarity by indexing into a collection with `string.GetHashCode()`. On .NET, string hash codes are randomized per process, so a choice made when writing an index (e.g. in the forked child process of TestIndexWriterOnJRECrash) differed from the choice made when reading it back in another process, producing spurious "codec header mismatch" / CheckIndex failures. Java is unaffected because String.hashCode() is deterministic across JVMs. Use J2N's CharSequenceComparer.Ordinal.GetHashCode(), which is deterministic and produces the same value as Java's String.hashCode(), keeping writer and reader in sync. This was a pre-existing test-framework bug (reproduces on master). Verified: TestIndexWriterOnJRECrash.TestNRTThreads_Mem (previously a deterministic failure under seed 0x03c0703dd1d74190:0xfc8ded704ed908f9) now passes on fixed and random seeds; RandomCodec- and similarity-heavy suites remain green. Confirmed against a native build of upstream Lucene 4.8.1, which passes this test. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent b403674 commit fb6fdb7

3 files changed

Lines changed: 27 additions & 7 deletions

File tree

src/Lucene.Net.TestFramework/Codecs/MockRandom/MockRandomPostingsFormat.cs

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
using J2N.Text;
12
using Lucene.Net.Codecs.BlockTerms;
23
using Lucene.Net.Codecs.Lucene41;
34
using Lucene.Net.Codecs.Memory;
@@ -97,7 +98,12 @@ public override Int32IndexInput OpenInput(Directory dir, string fileName, IOCont
9798
{
9899
// Must only use extension, because IW.addIndexes can
99100
// rename segment!
100-
Int32StreamFactory f = delegates[(Math.Abs(salt ^ GetExtension(fileName).GetHashCode())) % delegates.Count];
101+
// LUCENENET specific: use J2N's CharSequenceComparer.Ordinal.GetHashCode(), which produces
102+
// the same value as Java's String.hashCode(). The .NET string.GetHashCode() is randomized
103+
// per-process, so the factory chosen when writing (e.g. in a forked child process, as in
104+
// TestIndexWriterOnJRECrash) would differ from the one chosen when reading in another process,
105+
// causing "codec header mismatch" failures. A deterministic hash keeps writer and reader in sync.
106+
Int32StreamFactory f = delegates[(Math.Abs(salt ^ CharSequenceComparer.Ordinal.GetHashCode(GetExtension(fileName)))) % delegates.Count];
101107
if (LuceneTestCase.Verbose)
102108
{
103109
Console.WriteLine("MockRandomCodec: read using int factory " + f + " from fileName=" + fileName);
@@ -107,7 +113,8 @@ public override Int32IndexInput OpenInput(Directory dir, string fileName, IOCont
107113

108114
public override Int32IndexOutput CreateOutput(Directory dir, string fileName, IOContext context)
109115
{
110-
Int32StreamFactory f = delegates[(Math.Abs(salt ^ GetExtension(fileName).GetHashCode())) % delegates.Count];
116+
// LUCENENET specific: deterministic (Java-parity) hash; see the note in OpenInput().
117+
Int32StreamFactory f = delegates[(Math.Abs(salt ^ CharSequenceComparer.Ordinal.GetHashCode(GetExtension(fileName)))) % delegates.Count];
111118
if (LuceneTestCase.Verbose)
112119
{
113120
Console.WriteLine("MockRandomCodec: write using int factory " + f + " to fileName=" + fileName);

src/Lucene.Net.TestFramework/Index/RandomCodec.cs

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
using J2N.Text;
12
using Lucene.Net.Codecs;
23
using Lucene.Net.Codecs.Asserting;
34
using Lucene.Net.Codecs.Bloom;
@@ -84,11 +85,16 @@ public override PostingsFormat GetPostingsFormatForField(string name)
8485
{
8586
if (!previousMappings.TryGetValue(name, out PostingsFormat codec) || codec is null)
8687
{
87-
codec = formats[Math.Abs(perFieldSeed ^ name.GetHashCode()) % formats.Count];
88+
// LUCENENET specific: use J2N's CharSequenceComparer.Ordinal.GetHashCode(), which produces
89+
// the same value as Java's String.hashCode(). The .NET string.GetHashCode() is randomized
90+
// per-process, so a field's format chosen when writing (e.g. in a forked child process, as
91+
// in TestIndexWriterOnJRECrash) would differ from the one chosen when reading in another
92+
// process, causing "codec header mismatch" failures. A deterministic hash keeps them in sync.
93+
codec = formats[Math.Abs(perFieldSeed ^ CharSequenceComparer.Ordinal.GetHashCode(name)) % formats.Count];
8894
if (codec is SimpleTextPostingsFormat && perFieldSeed % 5 != 0)
8995
{
9096
// make simpletext rarer, choose again
91-
codec = formats[Math.Abs(perFieldSeed ^ name.ToUpperInvariant().GetHashCode()) % formats.Count];
97+
codec = formats[Math.Abs(perFieldSeed ^ CharSequenceComparer.Ordinal.GetHashCode(name.ToUpperInvariant())) % formats.Count];
9298
}
9399
previousMappings[name] = codec;
94100
// Safety:
@@ -107,11 +113,12 @@ public override DocValuesFormat GetDocValuesFormatForField(string name)
107113
{
108114
if (!previousDVMappings.TryGetValue(name, out DocValuesFormat codec) || codec is null)
109115
{
110-
codec = dvFormats[Math.Abs(perFieldSeed ^ name.GetHashCode()) % dvFormats.Count];
116+
// LUCENENET specific: deterministic (Java-parity) hash; see the note in GetPostingsFormatForField().
117+
codec = dvFormats[Math.Abs(perFieldSeed ^ CharSequenceComparer.Ordinal.GetHashCode(name)) % dvFormats.Count];
111118
if (codec is SimpleTextDocValuesFormat && perFieldSeed % 5 != 0)
112119
{
113120
// make simpletext rarer, choose again
114-
codec = dvFormats[Math.Abs(perFieldSeed ^ name.ToUpperInvariant().GetHashCode()) % dvFormats.Count];
121+
codec = dvFormats[Math.Abs(perFieldSeed ^ CharSequenceComparer.Ordinal.GetHashCode(name.ToUpperInvariant())) % dvFormats.Count];
115122
}
116123
previousDVMappings[name] = codec;
117124
// Safety:

src/Lucene.Net.TestFramework/Search/RandomSimilarityProvider.cs

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using J2N.Collections.Generic.Extensions;
2+
using J2N.Text;
23
using Lucene.Net.Diagnostics;
34
using Lucene.Net.Search.Similarities;
45
using Lucene.Net.Support.Threading;
@@ -78,7 +79,12 @@ public override Similarity Get(string field)
7879
if (Debugging.AssertsEnabled) Debugging.Assert(field != null);
7980
if (!previousMappings.TryGetValue(field, out Similarity sim) || sim is null)
8081
{
81-
sim = knownSims[Math.Max(0, Math.Abs(perFieldSeed ^ field.GetHashCode())) % knownSims.Count];
82+
// LUCENENET specific: use J2N's CharSequenceComparer.Ordinal.GetHashCode(), which produces
83+
// the same value as Java's String.hashCode(). The .NET string.GetHashCode() is randomized
84+
// per-process, so the per-field similarity chosen in one process would differ from another,
85+
// breaking reproducibility across processes (e.g. the forked TestIndexWriterOnJRECrash) and
86+
// diverging from Java. A deterministic hash keeps the choice stable.
87+
sim = knownSims[Math.Max(0, Math.Abs(perFieldSeed ^ CharSequenceComparer.Ordinal.GetHashCode(field))) % knownSims.Count];
8288
previousMappings[field] = sim;
8389
}
8490
return sim;

0 commit comments

Comments
 (0)