Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,10 @@ public override bool IncrementToken()
}

// LUCENENET: Reduce allocations by using the stack and spans
var source = new ReadOnlySpan<char>(chArray, idx, chLen);
var destination = chArray.AsSpan(idx, chLen);
var spare = chLen * sizeof(char) <= Constants.MaxStackByteLimit ? stackalloc char[chLen] : new char[chLen];
int spanLen = chLen - idx;
var source = new ReadOnlySpan<char>(chArray, idx, spanLen);
var destination = chArray.AsSpan(idx, spanLen);
var spare = spanLen * sizeof(char) <= Constants.MaxStackByteLimit ? stackalloc char[spanLen] : new char[spanLen];
source.ToLower(spare, culture); // LUCENENET specific - use Irish culture when lowercasing
spare.CopyTo(destination);

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
// Lucene version compatibility level 4.8.1
using Lucene.Net.Analysis.Core;
using Lucene.Net.Analysis.TokenAttributes;
using Lucene.Net.Attributes;
using NUnit.Framework;
using System.IO;

Expand Down Expand Up @@ -49,5 +51,111 @@ public virtual void TestEmptyTerm()
});
CheckOneTerm(a, "", "");
}

/// <summary>
/// Test that prothesis output is correct across all upper vowels and fadas,
/// and for both n- and t- prefixes.
/// </summary>
[Test, LuceneNetSpecific] // Issue #1150
public virtual void TestProthesisCorrectness()
{
Analyzer a = Analyzer.NewAnonymous(createComponents: (fieldName, reader) =>
{
Tokenizer tokenizer = new KeywordTokenizer(reader);
return new TokenStreamComponents(tokenizer, new IrishLowerCaseFilter(tokenizer));
});

// Plain upper vowels
CheckOneTerm(a, "nAthair", "n-athair");
CheckOneTerm(a, "nEan", "n-ean");
CheckOneTerm(a, "nIasc", "n-iasc");
CheckOneTerm(a, "nOiche", "n-oiche");
CheckOneTerm(a, "nUll", "n-ull");
CheckOneTerm(a, "tAthair", "t-athair");
CheckOneTerm(a, "tEan", "t-ean");
CheckOneTerm(a, "tIasc", "t-iasc");
CheckOneTerm(a, "tOiche", "t-oiche");
CheckOneTerm(a, "tUll", "t-ull");

// Accented vowels (fadas)
CheckOneTerm(a, "nÁit", "n-áit");
CheckOneTerm(a, "nÉan", "n-éan");
CheckOneTerm(a, "nÍoc", "n-íoc");
CheckOneTerm(a, "nÓg", "n-óg");
CheckOneTerm(a, "nÚll", "n-úll");
CheckOneTerm(a, "tÁit", "t-áit");
CheckOneTerm(a, "tÉan", "t-éan");
CheckOneTerm(a, "tÍoc", "t-íoc");
CheckOneTerm(a, "tÓg", "t-óg");
CheckOneTerm(a, "tÚll", "t-úll");
}

/// <summary>
/// Regression test for issue #1150: ArgumentOutOfRangeException in IrishLowerCaseFilter.
///
/// The bug: when the n-/t-prothesis path runs, idx is set to 2 but the span length was
/// left as chLen (full term length) instead of chLen - idx. This creates a span that reads
/// chArray[idx .. idx+chLen], which exceeds the buffer when the buffer is sized just large
/// enough for chLen chars.
///
/// We reproduce by using a custom TokenStream that writes directly into the internal
/// CharTermAttributeImpl.termBuffer field, bypassing the Oversize over-allocation that
Comment thread
paulirwin marked this conversation as resolved.
Outdated
/// all public paths go through, so the buffer is sized to exactly the post-prothesis
/// term length. The buggy code then asks for chArray[2..2+chLen] which is 2 chars past
/// the end.
/// </summary>
[Test, LuceneNetSpecific] // Issue #1150
public virtual void TestProthesis_SpanBoundaryThrows()
{
// "nAthair" has length 7; after prothesis it becomes "n-athair" (length 8).
// We pre-size the buffer to exactly 8 so that chArray[2..2+8] = chArray[2..10]
// exceeds the buffer and throws ArgumentOutOfRangeException on the buggy code.
const string input = "nAthair";
const string expected = "n-athair";

var source = new TightBufferTokenStream(input);
var filter = new IrishLowerCaseFilter(source);
AssertTokenStreamContents(filter, new[] { expected });
Comment thread
paulirwin marked this conversation as resolved.
Outdated
}

/// <summary>
/// A TokenStream that emits a single token with the backing buffer sized to exactly
/// the post-prothesis term length (term.Length + 1), bypassing Oversize over-allocation.
/// This forces the tight-buffer condition that triggers the span length bug in
/// IrishLowerCaseFilter when idx=2 and spanLen is incorrectly computed as chLen.
/// </summary>
private sealed class TightBufferTokenStream : TokenStream
{
private readonly string _term;
private bool _done;
private readonly CharTermAttribute _termAtt;

public TightBufferTokenStream(string term) : base()
{
_term = term;
_termAtt = (CharTermAttribute)AddAttribute<ICharTermAttribute>();
}

public override bool IncrementToken()
{
if (_done) return false;
_done = true;
ClearAttributes();
// Set the backing buffer to exactly postProthesisLen chars — no Oversize slack.
// IrishLowerCaseFilter.IncrementToken calls ResizeBuffer(chLen+1) which is a no-op
// when the buffer is already that size, leaving the span with no room for idx=2.
int postProthesisLen = _term.Length + 1;
_termAtt.termBuffer = new char[postProthesisLen];
_term.CopyTo(0, _termAtt.termBuffer, 0, _term.Length);
_termAtt.Length = _term.Length;
return true;
}

public override void Reset()
{
base.Reset();
_done = false;
}
}
}
}
Loading