Skip to content

Commit be94cfd

Browse files
paulirwinclaude
paulirwin
and
claude
authored
Fix ArgumentOutOfRangeException in IrishLowerCaseFilter span slicing, apache#1150 (apache#1286)
* Fix ArgumentOutOfRangeException in IrishLowerCaseFilter span slicing (apache#1150) IrishLowerCaseFilter was incorrectly calculating the span length for the lowercase operation. When the n-/t-prothesis path runs (inserting a hyphen after 'n' or 't'), idx becomes 2 and the span length should be (chLen - idx), not chLen. This caused out-of-bounds span creation. The fix calculates spanLen = chLen - idx and uses this corrected length for both the source and destination spans, as well as the spare buffer allocation. Added a regression test using a custom TightBufferTokenStream that writes directly into CharTermAttribute.termBuffer to bypass the Oversize over-allocation all public tokenizer paths use, producing a buffer sized to exactly the post-prothesis term length. This reliably triggers the ArgumentOutOfRangeException on the buggy code. Fixes apache#1150 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * Add prothesis correctness tests for IrishLowerCaseFilter (apache#1150) Tests all n-/t- prothesis combinations across plain and accented (fada) upper vowels to guard against regressions in output correctness. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * Address Copilot feedback --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent b905052 commit be94cfd

2 files changed

Lines changed: 105 additions & 3 deletions

File tree

‎src/Lucene.Net.Analysis.Common/Analysis/Ga/IrishLowerCaseFilter.cs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,10 @@ public override bool IncrementToken()
6565
}
6666

6767
// LUCENENET: Reduce allocations by using the stack and spans
68-
var source = new ReadOnlySpan<char>(chArray, idx, chLen);
69-
var destination = chArray.AsSpan(idx, chLen);
70-
var spare = chLen * sizeof(char) <= Constants.MaxStackByteLimit ? stackalloc char[chLen] : new char[chLen];
68+
int spanLen = chLen - idx;
69+
var source = new ReadOnlySpan<char>(chArray, idx, spanLen);
70+
var destination = chArray.AsSpan(idx, spanLen);
71+
var spare = spanLen * sizeof(char) <= Constants.MaxStackByteLimit ? stackalloc char[spanLen] : new char[spanLen];
7172
source.ToLower(spare, culture); // LUCENENET specific - use Irish culture when lowercasing
7273
spare.CopyTo(destination);
7374

‎src/Lucene.Net.Tests.Analysis.Common/Analysis/Ga/TestIrishLowerCaseFilter.cs‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
// Lucene version compatibility level 4.8.1
22
using Lucene.Net.Analysis.Core;
3+
using Lucene.Net.Analysis.TokenAttributes;
4+
using Lucene.Net.Attributes;
35
using NUnit.Framework;
46
using System.IO;
57

@@ -49,5 +51,104 @@ public virtual void TestEmptyTerm()
4951
});
5052
CheckOneTerm(a, "", "");
5153
}
54+
55+
/// <summary>
56+
/// Test that prothesis output is correct across all upper vowels and fadas,
57+
/// and for both n- and t- prefixes.
58+
/// </summary>
59+
[Test, LuceneNetSpecific] // Issue #1150
60+
public virtual void TestProthesisCorrectness()
61+
{
62+
Analyzer a = Analyzer.NewAnonymous(createComponents: (fieldName, reader) =>
63+
{
64+
Tokenizer tokenizer = new KeywordTokenizer(reader);
65+
return new TokenStreamComponents(tokenizer, new IrishLowerCaseFilter(tokenizer));
66+
});
67+
68+
// Plain upper vowels
69+
CheckOneTerm(a, "nAthair", "n-athair");
70+
CheckOneTerm(a, "nEan", "n-ean");
71+
CheckOneTerm(a, "nIasc", "n-iasc");
72+
CheckOneTerm(a, "nOiche", "n-oiche");
73+
CheckOneTerm(a, "nUll", "n-ull");
74+
CheckOneTerm(a, "tAthair", "t-athair");
75+
CheckOneTerm(a, "tEan", "t-ean");
76+
CheckOneTerm(a, "tIasc", "t-iasc");
77+
CheckOneTerm(a, "tOiche", "t-oiche");
78+
CheckOneTerm(a, "tUll", "t-ull");
79+
80+
// Accented vowels (fadas)
81+
CheckOneTerm(a, "nÁit", "n-áit");
82+
CheckOneTerm(a, "nÉan", "n-éan");
83+
CheckOneTerm(a, "nÍoc", "n-íoc");
84+
CheckOneTerm(a, "nÓg", "n-óg");
85+
CheckOneTerm(a, "nÚll", "n-úll");
86+
CheckOneTerm(a, "tÁit", "t-áit");
87+
CheckOneTerm(a, "tÉan", "t-éan");
88+
CheckOneTerm(a, "tÍoc", "t-íoc");
89+
CheckOneTerm(a, "tÓg", "t-óg");
90+
CheckOneTerm(a, "tÚll", "t-úll");
91+
}
92+
93+
/// <summary>
94+
/// Regression test for issue #1150: ArgumentOutOfRangeException in IrishLowerCaseFilter.
95+
///
96+
/// We confirm this is fixed by using a custom TokenStream that writes directly into the
97+
/// internal CharTermAttribute.termBuffer field, bypassing the Oversize over-allocation that
98+
/// all public paths go through, so the buffer is sized to exactly the post-prothesis
99+
/// term length. Any out-of-range access would throw an exception and fail the test.
100+
/// </summary>
101+
[Test, LuceneNetSpecific] // Issue #1150
102+
public virtual void TestProthesis_SpanBoundary()
103+
{
104+
// "nAthair" has length 7; after prothesis it becomes "n-athair" (length 8).
105+
// We pre-size the buffer to exactly 8 so that we can detect out-of-range access.
106+
const string input = "nAthair";
107+
const string expected = "n-athair";
108+
109+
var source = new TightBufferTokenStream(input);
110+
var filter = new IrishLowerCaseFilter(source);
111+
AssertTokenStreamContents(filter, new[] { expected });
112+
}
113+
114+
/// <summary>
115+
/// A TokenStream that emits a single token with the backing buffer sized to exactly
116+
/// the post-prothesis term length (term.Length + 1), bypassing Oversize over-allocation.
117+
/// This forces the tight-buffer condition that could detect span length bugs in
118+
/// IrishLowerCaseFilter.
119+
/// </summary>
120+
private sealed class TightBufferTokenStream : TokenStream
121+
{
122+
private readonly string _term;
123+
private bool _done;
124+
private readonly CharTermAttribute _termAtt;
125+
126+
public TightBufferTokenStream(string term)
127+
{
128+
_term = term;
129+
_termAtt = (CharTermAttribute)AddAttribute<ICharTermAttribute>();
130+
}
131+
132+
public override bool IncrementToken()
133+
{
134+
if (_done) return false;
135+
_done = true;
136+
ClearAttributes();
137+
// Set the backing buffer to exactly postProthesisLen chars — no Oversize slack.
138+
// IrishLowerCaseFilter.IncrementToken calls ResizeBuffer(chLen+1) which is a no-op
139+
// when the buffer is already that size, leaving the span with no room for idx=2.
140+
int postProthesisLen = _term.Length + 1;
141+
_termAtt.termBuffer = new char[postProthesisLen];
142+
_term.CopyTo(0, _termAtt.termBuffer, 0, _term.Length);
143+
_termAtt.Length = _term.Length;
144+
return true;
145+
}
146+
147+
public override void Reset()
148+
{
149+
base.Reset();
150+
_done = false;
151+
}
152+
}
52153
}
53154
}

0 commit comments

Comments
 (0)