Skip to content

Commit 2e8eca8

Browse files
paulirwinpaulirwin
committed
Port MaybeChangeLiveIndexWriterConfig changes from 4.8.1
1 parent e7f31c4 commit 2e8eca8

6 files changed

Lines changed: 253 additions & 26 deletions

File tree

‎src/Lucene.Net.TestFramework/Index/BaseIndexFileFormatTestCase.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,10 +108,11 @@ public virtual void TestMergeStability()
108108
{
109109
using Directory dir = NewDirectory();
110110
// do not use newMergePolicy that might return a MockMergePolicy that ignores the no-CFS ratio
111+
// do not use RIW which will change things up!
111112
MergePolicy mp = NewTieredMergePolicy();
112113
mp.NoCFSRatio = 0;
113114
var cfg = (new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random))).SetUseCompoundFile(false).SetMergePolicy(mp);
114-
using (var w = new RandomIndexWriter(Random, dir, cfg))
115+
using (var w = new IndexWriter(dir, cfg))
115116
{
116117
var numDocs = AtLeast(500);
117118
for (var i = 0; i < numDocs; ++i)
@@ -129,7 +130,7 @@ public virtual void TestMergeStability()
129130
mp.NoCFSRatio = 0;
130131
cfg = (new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random))).SetUseCompoundFile(false).SetMergePolicy(mp);
131132

132-
using (var w = new RandomIndexWriter(Random, dir2, cfg))
133+
using (var w = new IndexWriter(dir2, cfg))
133134
{
134135
w.AddIndexes(reader);
135136
w.Commit();

‎src/Lucene.Net.TestFramework/Index/RandomIndexWriter.cs‎

Lines changed: 53 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -126,11 +126,13 @@ public RandomIndexWriter(Random r, Directory dir, IndexWriterConfig c)
126126
/// <seealso cref="IndexWriter.AddDocument(IEnumerable{IIndexableField})"/>
127127
public virtual void AddDocument(IEnumerable<IIndexableField> doc)
128128
{
129+
LuceneTestCase.MaybeChangeLiveIndexWriterConfig(r, IndexWriter.Config);
129130
AddDocument(doc, IndexWriter.Analyzer);
130131
}
131132

132133
public virtual void AddDocument(IEnumerable<IIndexableField> doc, Analyzer a)
133134
{
135+
LuceneTestCase.MaybeChangeLiveIndexWriterConfig(r, IndexWriter.Config);
134136
if (r.Next(5) == 3)
135137
{
136138
// TODO: maybe, we should simply buffer up added docs
@@ -206,6 +208,7 @@ public void Dispose()
206208

207209
private void MaybeCommit()
208210
{
211+
LuceneTestCase.MaybeChangeLiveIndexWriterConfig(r, IndexWriter.Config);
209212
if (docCount++ == flushAt)
210213
{
211214
if (LuceneTestCase.Verbose)
@@ -224,12 +227,14 @@ private void MaybeCommit()
224227

225228
public virtual void AddDocuments(IEnumerable<IEnumerable<IIndexableField>> docs)
226229
{
230+
LuceneTestCase.MaybeChangeLiveIndexWriterConfig(r, IndexWriter.Config);
227231
IndexWriter.AddDocuments(docs);
228232
MaybeCommit();
229233
}
230234

231235
public virtual void UpdateDocuments(Term delTerm, IEnumerable<IEnumerable<IIndexableField>> docs)
232236
{
237+
LuceneTestCase.MaybeChangeLiveIndexWriterConfig(r, IndexWriter.Config);
233238
IndexWriter.UpdateDocuments(delTerm, docs);
234239
MaybeCommit();
235240
}
@@ -239,6 +244,7 @@ public virtual void UpdateDocuments(Term delTerm, IEnumerable<IEnumerable<IIndex
239244
/// <see cref="IndexWriter.UpdateDocument(Term, IEnumerable{IIndexableField})"/>
240245
public virtual void UpdateDocument(Term t, IEnumerable<IIndexableField> doc)
241246
{
247+
LuceneTestCase.MaybeChangeLiveIndexWriterConfig(r, IndexWriter.Config);
242248
if (r.Next(5) == 3)
243249
{
244250
IndexWriter.UpdateDocuments(t, new EnumerableAnonymousClass2(doc));
@@ -306,25 +312,46 @@ public void Dispose()
306312
}
307313

308314
public virtual void AddIndexes(params Directory[] dirs)
309-
=> IndexWriter.AddIndexes(dirs);
315+
{
316+
LuceneTestCase.MaybeChangeLiveIndexWriterConfig(r, IndexWriter.Config);
317+
IndexWriter.AddIndexes(dirs);
318+
}
310319

311320
public virtual void AddIndexes(params IndexReader[] readers)
312-
=> IndexWriter.AddIndexes(readers);
321+
{
322+
LuceneTestCase.MaybeChangeLiveIndexWriterConfig(r, IndexWriter.Config);
323+
IndexWriter.AddIndexes(readers);
324+
}
313325

314326
public virtual void UpdateNumericDocValue(Term term, string field, long? value)
315-
=> IndexWriter.UpdateNumericDocValue(term, field, value);
327+
{
328+
LuceneTestCase.MaybeChangeLiveIndexWriterConfig(r, IndexWriter.Config);
329+
IndexWriter.UpdateNumericDocValue(term, field, value);
330+
}
316331

317332
public virtual void UpdateBinaryDocValue(Term term, string field, BytesRef value)
318-
=> IndexWriter.UpdateBinaryDocValue(term, field, value);
333+
{
334+
LuceneTestCase.MaybeChangeLiveIndexWriterConfig(r, IndexWriter.Config);
335+
IndexWriter.UpdateBinaryDocValue(term, field, value);
336+
}
319337

320338
public virtual void DeleteDocuments(Term term)
321-
=> IndexWriter.DeleteDocuments(term);
339+
{
340+
LuceneTestCase.MaybeChangeLiveIndexWriterConfig(r, IndexWriter.Config);
341+
IndexWriter.DeleteDocuments(term);
342+
}
322343

323344
public virtual void DeleteDocuments(Query q)
324-
=> IndexWriter.DeleteDocuments(q);
345+
{
346+
LuceneTestCase.MaybeChangeLiveIndexWriterConfig(r, IndexWriter.Config);
347+
IndexWriter.DeleteDocuments(q);
348+
}
325349

326350
public virtual void Commit()
327-
=> IndexWriter.Commit();
351+
{
352+
LuceneTestCase.MaybeChangeLiveIndexWriterConfig(r, IndexWriter.Config);
353+
IndexWriter.Commit();
354+
}
328355

329356
public virtual int NumDocs
330357
=> IndexWriter.NumDocs;
@@ -336,16 +363,25 @@ public virtual void DeleteAll()
336363
=> IndexWriter.DeleteAll();
337364

338365
public virtual DirectoryReader GetReader()
339-
=> GetReader(true);
366+
{
367+
LuceneTestCase.MaybeChangeLiveIndexWriterConfig(r, IndexWriter.Config);
368+
return GetReader(true);
369+
}
340370

341371
private bool doRandomForceMerge = true;
342372
private bool doRandomForceMergeAssert = true;
343373

344374
public virtual void ForceMergeDeletes(bool doWait)
345-
=> IndexWriter.ForceMergeDeletes(doWait);
375+
{
376+
LuceneTestCase.MaybeChangeLiveIndexWriterConfig(r, IndexWriter.Config);
377+
IndexWriter.ForceMergeDeletes(doWait);
378+
}
346379

347380
public virtual void ForceMergeDeletes()
348-
=> IndexWriter.ForceMergeDeletes();
381+
{
382+
LuceneTestCase.MaybeChangeLiveIndexWriterConfig(r, IndexWriter.Config);
383+
IndexWriter.ForceMergeDeletes();
384+
}
349385

350386
public virtual bool DoRandomForceMerge
351387
{
@@ -391,6 +427,7 @@ private void _DoRandomForceMerge() // LUCENENET specific - added leading undersc
391427

392428
public virtual DirectoryReader GetReader(bool applyDeletions)
393429
{
430+
LuceneTestCase.MaybeChangeLiveIndexWriterConfig(r, IndexWriter.Config);
394431
getReaderCalled = true;
395432
if (r.Next(20) == 2)
396433
{
@@ -447,6 +484,11 @@ protected virtual void Dispose(bool disposing)
447484
{
448485
if (disposing)
449486
{
487+
if (!IndexWriter.IsClosed)
488+
{
489+
LuceneTestCase.MaybeChangeLiveIndexWriterConfig(r, IndexWriter.Config);
490+
}
491+
450492
// if someone isn't using getReader() API, we want to be sure to
451493
// forceMerge since presumably they might open a reader on the dir.
452494
if (getReaderCalled == false && r.Next(8) == 2)
@@ -465,6 +507,7 @@ protected virtual void Dispose(bool disposing)
465507
/// <seealso cref="IndexWriter.ForceMerge(int)"/>
466508
public virtual void ForceMerge(int maxSegmentCount)
467509
{
510+
LuceneTestCase.MaybeChangeLiveIndexWriterConfig(r, IndexWriter.Config);
468511
IndexWriter.ForceMerge(maxSegmentCount);
469512
}
470513

‎src/Lucene.Net.TestFramework/Util/LuceneTestCase.cs‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1598,6 +1598,170 @@ public static MergePolicy NewLogMergePolicy(int mergeFactor)
15981598
return logmp;
15991599
}
16001600

1601+
// if you want it in LiveIndexWriterConfig: it must and will be tested here.
1602+
public static void MaybeChangeLiveIndexWriterConfig(Random r, LiveIndexWriterConfig c)
1603+
{
1604+
bool didChange = false;
1605+
1606+
if (Rarely(r))
1607+
{
1608+
// change flush parameters:
1609+
// this is complicated because the api requires you "invoke setters in a magical order!"
1610+
// LUCENE-5661: workaround for race conditions in the API
1611+
UninterruptableMonitor.Enter(c);
1612+
try
1613+
{
1614+
bool flushByRam = r.NextBoolean();
1615+
if (flushByRam)
1616+
{
1617+
c.SetRAMBufferSizeMB(TestUtil.NextInt32(r, 1, 10));
1618+
c.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
1619+
}
1620+
else
1621+
{
1622+
if (Rarely(r))
1623+
{
1624+
// crazy value
1625+
c.SetMaxBufferedDocs(TestUtil.NextInt32(r, 2, 15));
1626+
}
1627+
else
1628+
{
1629+
// reasonable value
1630+
c.SetMaxBufferedDocs(TestUtil.NextInt32(r, 16, 1000));
1631+
}
1632+
1633+
c.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
1634+
}
1635+
}
1636+
finally
1637+
{
1638+
UninterruptableMonitor.Exit(c);
1639+
}
1640+
1641+
didChange = true;
1642+
}
1643+
1644+
if (Rarely(r))
1645+
{
1646+
// change buffered deletes parameters
1647+
bool limitBufferedDeletes = r.NextBoolean();
1648+
if (limitBufferedDeletes)
1649+
{
1650+
c.SetMaxBufferedDeleteTerms(TestUtil.NextInt32(r, 1, 1000));
1651+
}
1652+
else
1653+
{
1654+
c.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
1655+
}
1656+
1657+
didChange = true;
1658+
}
1659+
1660+
if (Rarely(r))
1661+
{
1662+
// change warmer parameters
1663+
if (r.NextBoolean())
1664+
{
1665+
c.SetMergedSegmentWarmer(new SimpleMergedSegmentWarmer(c.InfoStream));
1666+
}
1667+
else
1668+
{
1669+
c.SetMergedSegmentWarmer(null);
1670+
}
1671+
1672+
didChange = true;
1673+
}
1674+
1675+
if (Rarely(r))
1676+
{
1677+
// change CFS flush parameters
1678+
c.SetUseCompoundFile(r.NextBoolean());
1679+
didChange = true;
1680+
}
1681+
1682+
if (Rarely(r))
1683+
{
1684+
// change merge integrity check parameters
1685+
c.SetCheckIntegrityAtMerge(r.NextBoolean());
1686+
didChange = true;
1687+
}
1688+
1689+
if (Rarely(r))
1690+
{
1691+
// change CMS merge parameters
1692+
IMergeScheduler ms = c.MergeScheduler;
1693+
if (ms is ConcurrentMergeScheduler cms)
1694+
{
1695+
int maxThreadCount = TestUtil.NextInt32(r, 1, 4);
1696+
int maxMergeCount = TestUtil.NextInt32(r, maxThreadCount, maxThreadCount + 4);
1697+
cms.SetMaxMergesAndThreads(maxMergeCount, maxThreadCount);
1698+
}
1699+
1700+
didChange = true;
1701+
}
1702+
1703+
if (Rarely(r))
1704+
{
1705+
MergePolicy mp = c.MergePolicy;
1706+
ConfigureRandom(r, mp);
1707+
if (mp is LogMergePolicy logmp)
1708+
{
1709+
logmp.CalibrateSizeByDeletes = r.NextBoolean();
1710+
if (Rarely(r))
1711+
{
1712+
logmp.MergeFactor = TestUtil.NextInt32(r, 2, 9);
1713+
}
1714+
else
1715+
{
1716+
logmp.MergeFactor = TestUtil.NextInt32(r, 10, 50);
1717+
}
1718+
}
1719+
else if (mp is TieredMergePolicy tmp)
1720+
{
1721+
if (Rarely(r))
1722+
{
1723+
tmp.MaxMergeAtOnce = TestUtil.NextInt32(r, 2, 9);
1724+
tmp.MaxMergeAtOnceExplicit = TestUtil.NextInt32(r, 2, 9);
1725+
}
1726+
else
1727+
{
1728+
tmp.MaxMergeAtOnce = TestUtil.NextInt32(r, 10, 50);
1729+
tmp.MaxMergeAtOnceExplicit = TestUtil.NextInt32(r, 10, 50);
1730+
}
1731+
1732+
if (Rarely(r))
1733+
{
1734+
tmp.MaxMergedSegmentMB = 0.2 + r.NextDouble() * 2.0;
1735+
}
1736+
else
1737+
{
1738+
tmp.MaxMergedSegmentMB = r.NextDouble() * 100;
1739+
}
1740+
1741+
tmp.FloorSegmentMB = 0.2 + r.NextDouble() * 2.0;
1742+
tmp.ForceMergeDeletesPctAllowed = 0.0 + r.NextDouble() * 30.0;
1743+
if (Rarely(r))
1744+
{
1745+
tmp.SegmentsPerTier = TestUtil.NextInt32(r, 2, 20);
1746+
}
1747+
else
1748+
{
1749+
tmp.SegmentsPerTier = TestUtil.NextInt32(r, 10, 50);
1750+
}
1751+
1752+
ConfigureRandom(r, tmp);
1753+
tmp.ReclaimDeletesWeight = r.NextDouble() * 4;
1754+
}
1755+
1756+
didChange = true;
1757+
}
1758+
1759+
if (Verbose && didChange)
1760+
{
1761+
Console.WriteLine($"NOTE: LuceneTestCase: randomly changed IWC's live settings to:\n{c}");
1762+
}
1763+
}
1764+
16011765
/// <summary>
16021766
/// Returns a new <see cref="Directory"/> instance. Use this when the test does not
16031767
/// care about the specific <see cref="Directory"/> implementation (most tests).

‎src/Lucene.Net.Tests/Index/TestConcurrentMergeScheduler.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -408,7 +408,7 @@ public virtual void TestTotalBytesSize()
408408
// no
409409
iwc.SetCodec(TestUtil.AlwaysPostingsFormat(new Lucene41PostingsFormat()));
410410
}
411-
RandomIndexWriter w = new RandomIndexWriter(Random, d, iwc);
411+
IndexWriter w = new IndexWriter(d, iwc);
412412
for (int i = 0; i < 1000; i++)
413413
{
414414
Document doc = new Document();
@@ -420,7 +420,7 @@ public virtual void TestTotalBytesSize()
420420
w.DeleteDocuments(new Term("id", "" + Random.Next(i + 1)));
421421
}
422422
}
423-
Assert.IsTrue(((TrackingCMS)w.IndexWriter.Config.MergeScheduler).totMergedBytes != 0);
423+
Assert.IsTrue(((TrackingCMS)w.Config.MergeScheduler).totMergedBytes != 0);
424424
w.Dispose();
425425
d.Dispose();
426426
}

‎src/Lucene.Net.Tests/Index/TestTieredMergePolicy.cs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -170,8 +170,7 @@ public virtual void TestForceMergeDeletesMaxSegSize()
170170
tmp.ForceMergeDeletesPctAllowed = 0.0;
171171
conf.SetMergePolicy(tmp);
172172

173-
RandomIndexWriter w = new RandomIndexWriter(Random, dir, conf);
174-
w.DoRandomForceMerge = false;
173+
IndexWriter w = new IndexWriter(dir, conf);
175174

176175
int numDocs = AtLeast(200);
177176
for (int i = 0; i < numDocs; i++)

0 commit comments

Comments
 (0)