Fix random TestRandomChains failures by excluding broken-offsets producers, #1072 (#1348)

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: paulirwin

src/Lucene.Net.Analysis.Common/Analysis/Compound/HyphenationCompoundWordTokenFilter.cs

@@ -244,7 +244,11 @@ protected override void Decompose()
                             m_tokens.Enqueue(new CompoundToken(this, start, partLength));
                         }
                     }
-                    else if (m_dictionary.Contains(m_termAtt.Buffer, start, partLength - 1))
+                    // LUCENENET specific - guard against a negative length (partLength == 0).
+                    // Upstream Java relies on CharArrayMap silently treating a negative length as
+                    // "not found", but Lucene.NET's CharArrayMap validates its arguments and throws
+                    // ArgumentOutOfRangeException. See the BOGUS/BROKEN/FUNKY/WACKO note above.
+                    else if (partLength - 1 >= 0 && m_dictionary.Contains(m_termAtt.Buffer, start, partLength - 1))
                     {
                         // check the dictionary again with a word that is one character
                         // shorter

src/Lucene.Net.Analysis.Kuromoji/JapaneseKatakanaStemFilter.cs

@@ -51,9 +51,11 @@ public sealed class JapaneseKatakanaStemFilter : TokenFilter
         public JapaneseKatakanaStemFilter(TokenStream input, int minimumLength)
             : base(input)
         {
-            // LUCENENET: Added guard clause
-            if (minimumLength < 0)
-                throw new ArgumentOutOfRangeException(nameof(minimumLength), "Minimum length must be a non-negative integer.");
+            // LUCENENET: Backport of the LUCENE-10352 guard clause. Enforcing minimumLength >= 1
+            // prevents a zero-length term (e.g. the single empty token a KeywordTokenizer emits for
+            // empty input) from reaching the term[length - 1] read in Stem() with length == 0.
+            if (minimumLength < 1)
+                throw new ArgumentOutOfRangeException(nameof(minimumLength), "minimumLength must be >=1");
 
             this.minimumKatakanaLength = minimumLength;
             this.termAttr = AddAttribute<ICharTermAttribute>();

src/Lucene.Net.Tests.Analysis.Common/Analysis/Compound/TestCompoundWordTokenFilter.cs

@@ -4,6 +4,7 @@
 using Lucene.Net.Analysis.Core;
 using Lucene.Net.Analysis.TokenAttributes;
 using Lucene.Net.Analysis.Util;
+using Lucene.Net.Attributes;
 using Lucene.Net.Util;
 using NUnit.Framework;
 using System.IO;
@@ -266,6 +267,41 @@ public virtual void TestRandomStrings()
             CheckRandomData(Random, b, 1000 * RandomMultiplier);
         }
 
+        /// <summary>
+        /// LUCENENET-specific regression test for <a href="https://github.com/apache/lucenenet/issues/1072">#1072</a>.
+        /// <para/>
+        /// When a term has leading non-letter characters (which the hyphenator counts via
+        /// <c>iIgnoreAtBeginning</c>) the resulting hyphenation points can contain two equal values,
+        /// so <see cref="HyphenationCompoundWordTokenFilter"/>'s <c>Decompose()</c> computes a
+        /// <c>partLength</c> of <c>0</c>. With <c>minSubwordSize == 0</c> that zero-length part is not
+        /// filtered out, and the genitive-'s fallback then probes the dictionary with a length of
+        /// <c>partLength - 1 == -1</c>. Upstream Java tolerates the negative length (its CharArrayMap
+        /// silently treats it as "not found"), but Lucene.NET's CharArrayMap validates its arguments
+        /// and throws <see cref="System.ArgumentOutOfRangeException"/>. The filter must not pass a
+        /// negative length downstream. The input <c>"...rindfleisch"</c> yields hyphenation points
+        /// <c>[0, 7, 11, 11]</c>, deterministically reproducing the crash that
+        /// <c>TestRandomChains</c> hit under seed <c>0xc0ab8518c470366f:0x6ed44cef961049a2</c>.
+        /// </summary>
+        [Test]
+        [LuceneNetSpecific] // Issue #1072
+        public virtual void TestZeroLengthSubwordDoesNotThrow()
+        {
+            CharArraySet dict = makeDictionary("rind", "fleisch");
+
+            using var @is = this.GetType().getResourceAsStream("da_UTF8.xml");
+            HyphenationTree hyphenator = HyphenationCompoundWordTokenFilter.GetHyphenationTree(@is);
+
+            // minSubwordSize == 0 is the configuration that lets a zero-length part through.
+            HyphenationCompoundWordTokenFilter tf = new HyphenationCompoundWordTokenFilter(TEST_VERSION_CURRENT,
+                new MockTokenizer(new StringReader("...rindfleisch"), MockTokenizer.WHITESPACE, false),
+                hyphenator, dict, CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE, 0, 100, false);
+
+            // Before the fix this threw ArgumentOutOfRangeException while decomposing. The leading
+            // "..." shifts the matched subword offsets, so no subword survives the dictionary check;
+            // the only required behavior is that decomposition completes without throwing.
+            AssertTokenStreamContents(tf, new string[] { "...rindfleisch" });
+        }
+
         [Test]
         public virtual void TestEmptyTerm()
         {

src/Lucene.Net.Tests.Analysis.Common/Analysis/Core/TestRandomChains.cs

@@ -151,6 +151,16 @@ static TestRandomChains()
                     foreach (ConstructorInfo ctor in c.GetConstructors())
                     {
                         brokenOffsetsConstructors[ctor] = ALWAYS;
+
+                        // LUCENENET specific (#1072): Also fully exclude these broken-offsets producers
+                        // from the random chains, not just relax offset validation. brokenOffsetsConstructors
+                        // only flips offsetsAreCorrect=false (which suppresses ValidatingTokenFilter's checks),
+                        // but the offending filter still runs and can feed backwards offsets into a downstream
+                        // word-combiner (e.g. ShingleFilter), which then throws from OffsetAttribute.SetOffset.
+                        // Upstream Lucene resolved this in 9.1 (LUCENE-10352) by excluding these classes via the
+                        // @IgnoreRandomChains annotation. Until that annotation is backported, exclude them here.
+                        // TODO: Remove this when LUCENE-10352 (IgnoreRandomChains) is backported.
+                        brokenConstructors[ctor] = ALWAYS;
                     }
                 }
             }
@@ -1169,8 +1179,8 @@ private TokenFilterSpec NewFilterChain(Random random, Tokenizer tokenizer, bool
                         // hack: MockGraph/MockLookahead has assertions that will trip if they follow
                         // an offsets violator. so we can't use them after e.g. wikipediatokenizer
                         if (!spec.offsetsAreCorrect &&
-                            (ctor.DeclaringType.Equals(typeof(MockGraphTokenFilter)))
-                                || ctor.DeclaringType.Equals(typeof(MockRandomLookaheadTokenFilter)))
+                            (ctor.DeclaringType.Equals(typeof(MockGraphTokenFilter))
+                                || ctor.DeclaringType.Equals(typeof(MockRandomLookaheadTokenFilter))))
                         {
                             continue;
                         }
@@ -1284,7 +1294,6 @@ private static TEnum RandomEnum<TEnum>(Random random)
         }
 
         [Test]
-        [AwaitsFix(BugUrl = "https://github.com/apache/lucenenet/issues/271#issuecomment-973005744")] // LUCENENET TODO: this test occasionally fails
         public void TestRandomChains_()
         {
             int numIterations = AtLeast(20);
@@ -1311,7 +1320,6 @@ public void TestRandomChains_()
 
         // we might regret this decision...
         [Test]
-        [AwaitsFix(BugUrl = "https://github.com/apache/lucenenet/issues/271#issuecomment-973005744")] // LUCENENET TODO: this test occasionally fails
         public void TestRandomChainsWithLargeStrings()
         {
             int numIterations = AtLeast(20);

src/Lucene.Net.Tests.Analysis.Kuromoji/TestJapaneseKatakanaStemFilter.cs

@@ -1,6 +1,7 @@
 using Lucene.Net.Analysis.Core;
 using Lucene.Net.Analysis.Miscellaneous;
 using Lucene.Net.Analysis.Util;
+using Lucene.Net.Attributes;
 using NUnit.Framework;
 using System;
 
@@ -96,5 +97,25 @@ public void TestEmptyTerm()
 
             CheckOneTerm(a, "", "");
         }
+
+        /// <summary>
+        /// LUCENENET-specific regression test for <a href="https://github.com/apache/lucenenet/issues/1072">#1072</a>.
+        /// <para/>
+        /// Verifies the LUCENE-10352 guard: the constructor must reject a <c>minimumLength</c> less than
+        /// <c>1</c>. A <c>minimumLength</c> of <c>0</c> would let a zero-length token (such as the single empty
+        /// token a <see cref="KeywordTokenizer"/> emits for empty input) skip the length check in <c>Stem()</c>,
+        /// after which the <c>term[length - 1]</c> access reads <c>term[-1]</c> and throws. This surfaced as a
+        /// residual <c>TestRandomChains</c> failure reported on PR #1348 (e.g. seed
+        /// <c>0x951afd480395f9b7:0x63fc16274945f1a3</c>).
+        /// </summary>
+        [Test]
+        [LuceneNetSpecific] // Issue #1072
+        public void TestMinimumLengthLessThanOneThrows()
+        {
+            Tokenizer tokenizer = new KeywordTokenizer(new System.IO.StringReader(""));
+
+            Assert.Throws<ArgumentOutOfRangeException>(() => new JapaneseKatakanaStemFilter(tokenizer, minimumLength: 0));
+            Assert.Throws<ArgumentOutOfRangeException>(() => new JapaneseKatakanaStemFilter(tokenizer, minimumLength: -1));
+        }
     }
 }