Skip to content

Commit e67da5b

Browse files
paulirwinclaude
paulirwin
and
claude
authored
Performance improvements for DataInput/DataOutput and bit-shifting logic, apache#1279 (apache#1345)
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 3428473 commit e67da5b

29 files changed

Lines changed: 1362 additions & 311 deletions

‎src/Lucene.Net.Analysis.Common/Analysis/Hunspell/Dictionary.cs‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
using Lucene.Net.Util.Automaton;
1010
using Lucene.Net.Util.Fst;
1111
using System;
12+
using System.Buffers.Binary;
1213
using System.Collections.Generic;
1314
using System.Globalization;
1415
using System.IO;
@@ -1148,9 +1149,14 @@ internal static void EncodeFlags(BytesRef b, char[] flags)
11481149
int upto = b.Offset;
11491150
for (int i = 0; i < flags.Length; i++)
11501151
{
1151-
int flag = flags[i];
1152-
b.Bytes[upto++] = (byte)((flag >> 8) & 0xff);
1153-
b.Bytes[upto++] = (byte)(flag & 0xff);
1152+
ushort flag = flags[i];
1153+
1154+
// b.Bytes[upto++] = (byte)((flag >> 8) & 0xff);
1155+
// b.Bytes[upto++] = (byte)(flag & 0xff);
1156+
1157+
// LUCENENET: Use BinaryPrimitives for JIT-intrinsics opportunity
1158+
BinaryPrimitives.WriteUInt16BigEndian(b.Bytes.AsSpan(upto, sizeof(ushort)), flag);
1159+
upto += sizeof(ushort);
11541160
}
11551161
}
11561162

‎src/Lucene.Net.Analysis.Common/Analysis/Payloads/PayloadHelper.cs‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// Lucene version compatibility level 4.8.1
22

33
using System;
4+
using System.Buffers.Binary;
45

56
namespace Lucene.Net.Analysis.Payloads
67
{
@@ -55,10 +56,14 @@ public static byte[] EncodeInt32(int payload)
5556
/// </summary>
5657
public static byte[] EncodeInt32(int payload, byte[] data, int offset)
5758
{
58-
data[offset] = (byte)(payload >> 24);
59-
data[offset + 1] = (byte)(payload >> 16);
60-
data[offset + 2] = (byte)(payload >> 8);
61-
data[offset + 3] = (byte)payload;
59+
// data[offset] = (byte)(payload >> 24);
60+
// data[offset + 1] = (byte)(payload >> 16);
61+
// data[offset + 2] = (byte)(payload >> 8);
62+
// data[offset + 3] = (byte)payload;
63+
// return data;
64+
65+
// LUCENENET: Use BinaryPrimitives for JIT-intrinsics opportunity
66+
BinaryPrimitives.WriteInt32BigEndian(data.AsSpan(offset, sizeof(int)), payload);
6267
return data;
6368
}
6469

@@ -94,7 +99,10 @@ public static float DecodeSingle(ReadOnlySpan<byte> bytes, int offset)
9499
/// </summary>
95100
public static int DecodeInt32(ReadOnlySpan<byte> bytes, int offset)
96101
{
97-
return ((bytes[offset] & 0xFF) << 24) | ((bytes[offset + 1] & 0xFF) << 16) | ((bytes[offset + 2] & 0xFF) << 8) | (bytes[offset + 3] & 0xFF);
102+
// return ((bytes[offset] & 0xFF) << 24) | ((bytes[offset + 1] & 0xFF) << 16) | ((bytes[offset + 2] & 0xFF) << 8) | (bytes[offset + 3] & 0xFF);
103+
104+
// LUCENENET: Use BinaryPrimitives for JIT-intrinsics opportunity
105+
return BinaryPrimitives.ReadInt32BigEndian(bytes.Slice(offset, sizeof(int)));
98106
}
99107
}
100108
}

‎src/Lucene.Net.Facet/FacetsConfig.cs‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
// Lucene version compatibility level 4.8.1
22
using Lucene.Net.Diagnostics;
3+
using Lucene.Net.Index;
34
using Lucene.Net.Support;
45
using Lucene.Net.Support.Threading;
56
using System;
7+
using System.Buffers.Binary;
68
using System.Collections.Concurrent;
79
using System.Collections.Generic;
810
using System.Runtime.CompilerServices;
@@ -283,7 +285,7 @@ private static void CheckSeen(ISet<string> seenDims, string dim)
283285
/// (<see cref="FacetField"/> or <see cref="AssociationFacetField"/>).
284286
///
285287
/// <para>
286-
/// <b>NOTE:</b> you should add the returned document to <see cref="Index.IndexWriter"/>, not the
288+
/// <b>NOTE:</b> you should add the returned document to <see cref="IndexWriter"/>, not the
287289
/// input one!
288290
/// </para>
289291
/// </summary>
@@ -297,7 +299,7 @@ public virtual Document Build(Document doc)
297299
/// Translates any added <see cref="FacetField"/>s into normal fields for indexing.
298300
///
299301
/// <para>
300-
/// <b>NOTE:</b> you should add the returned document to <see cref="Index.IndexWriter"/>, not the
302+
/// <b>NOTE:</b> you should add the returned document to <see cref="IndexWriter"/>, not the
301303
/// input one!
302304
/// </para>
303305
/// </summary>
@@ -530,11 +532,17 @@ private static void ProcessAssocFacetFields(ITaxonomyWriter taxoWriter, IDiction
530532
{
531533
bytes = ArrayUtil.Grow(bytes, upto + 4);
532534
}
533-
// big-endian:
534-
bytes[upto++] = (byte)(ordinal >> 24);
535-
bytes[upto++] = (byte)(ordinal >> 16);
536-
bytes[upto++] = (byte)(ordinal >> 8);
537-
bytes[upto++] = (byte)ordinal;
535+
536+
// // big-endian:
537+
// bytes[upto++] = (byte)(ordinal >> 24);
538+
// bytes[upto++] = (byte)(ordinal >> 16);
539+
// bytes[upto++] = (byte)(ordinal >> 8);
540+
// bytes[upto++] = (byte)ordinal;
541+
542+
// LUCENENET: Use BinaryPrimitives for JIT-intrinsics opportunity
543+
BinaryPrimitives.WriteInt32BigEndian(bytes.AsSpan(upto, sizeof(int)), ordinal);
544+
upto += sizeof(int);
545+
538546
if (upto + field.Assoc.Length > bytes.Length)
539547
{
540548
bytes = ArrayUtil.Grow(bytes, upto + field.Assoc.Length);

‎src/Lucene.Net.Facet/Taxonomy/IntAssociationFacetField.cs‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
// Lucene version compatibility level 4.8.1
22
using Lucene.Net.Support;
3+
using System;
4+
using System.Buffers.Binary;
35

46
namespace Lucene.Net.Facet.Taxonomy
57
{
@@ -52,11 +54,15 @@ public Int32AssociationFacetField(int assoc, string dim, params string[] path)
5254
public static BytesRef Int32ToBytesRef(int v)
5355
{
5456
byte[] bytes = new byte[4];
55-
// big-endian:
56-
bytes[0] = (byte)(v >> 24);
57-
bytes[1] = (byte)(v >> 16);
58-
bytes[2] = (byte)(v >> 8);
59-
bytes[3] = (byte)v;
57+
// // big-endian:
58+
// bytes[0] = (byte)(v >> 24);
59+
// bytes[1] = (byte)(v >> 16);
60+
// bytes[2] = (byte)(v >> 8);
61+
// bytes[3] = (byte)v;
62+
63+
// LUCENENET: Use BinaryPrimitives for JIT-intrinsics opportunity
64+
BinaryPrimitives.WriteInt32BigEndian(bytes, v);
65+
6066
return new BytesRef(bytes);
6167
}
6268

@@ -67,8 +73,11 @@ public static BytesRef Int32ToBytesRef(int v)
6773
/// </summary>
6874
public static int BytesRefToInt32(BytesRef b)
6975
{
70-
return ((b.Bytes[b.Offset] & 0xFF) << 24) | ((b.Bytes[b.Offset + 1] & 0xFF) << 16) |
71-
((b.Bytes[b.Offset + 2] & 0xFF) << 8) | (b.Bytes[b.Offset + 3] & 0xFF);
76+
// return ((b.Bytes[b.Offset] & 0xFF) << 24) | ((b.Bytes[b.Offset + 1] & 0xFF) << 16) |
77+
// ((b.Bytes[b.Offset + 2] & 0xFF) << 8) | (b.Bytes[b.Offset + 3] & 0xFF);
78+
79+
// LUCENENET: Use BinaryPrimitives for JIT-intrinsics opportunity
80+
return BinaryPrimitives.ReadInt32BigEndian(b.AsSpan(0, sizeof(int))); // AsSpan implicitly handles b.Offset
7281
}
7382

7483
public override string ToString()

‎src/Lucene.Net.Facet/Taxonomy/TaxonomyFacetSumFloatAssociations.cs‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
// Lucene version compatibility level 4.8.1
2+
3+
using System;
4+
using System.Buffers.Binary;
25
using System.Collections.Generic;
36

47
namespace Lucene.Net.Facet.Taxonomy
@@ -81,12 +84,22 @@ private void SumValues(IList<FacetsCollector.MatchingDocs> matchingDocs)
8184
int offset = scratch.Offset;
8285
while (offset < end)
8386
{
84-
int ord = ((bytes[offset] & 0xFF) << 24) | ((bytes[offset + 1] & 0xFF) << 16) |
85-
((bytes[offset + 2] & 0xFF) << 8) | (bytes[offset + 3] & 0xFF);
86-
offset += 4;
87-
int value = ((bytes[offset] & 0xFF) << 24) | ((bytes[offset + 1] & 0xFF) << 16) |
88-
((bytes[offset + 2] & 0xFF) << 8) | (bytes[offset + 3] & 0xFF);
89-
offset += 4;
87+
// int ord = ((bytes[offset] & 0xFF) << 24) | ((bytes[offset + 1] & 0xFF) << 16) |
88+
// ((bytes[offset + 2] & 0xFF) << 8) | (bytes[offset + 3] & 0xFF);
89+
// offset += 4;
90+
91+
// LUCENENET: Use BinaryPrimitives for JIT-intrinsics opportunity
92+
int ord = BinaryPrimitives.ReadInt32BigEndian(bytes.AsSpan(offset, sizeof(int)));
93+
offset += sizeof(int);
94+
95+
// int value = ((bytes[offset] & 0xFF) << 24) | ((bytes[offset + 1] & 0xFF) << 16) |
96+
// ((bytes[offset + 2] & 0xFF) << 8) | (bytes[offset + 3] & 0xFF);
97+
// offset += 4;
98+
99+
// LUCENENET: Use BinaryPrimitives for JIT-intrinsics opportunity
100+
int value = BinaryPrimitives.ReadInt32BigEndian(bytes.AsSpan(offset, sizeof(int)));
101+
offset += sizeof(int);
102+
90103
m_values[ord] += J2N.BitConversion.Int32BitsToSingle(value);
91104
}
92105
}

‎src/Lucene.Net.Facet/Taxonomy/TaxonomyFacetSumIntAssociations.cs‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
// Lucene version compatibility level 4.8.1
2+
3+
using System;
4+
using System.Buffers.Binary;
25
using System.Collections.Generic;
36

47
namespace Lucene.Net.Facet.Taxonomy
@@ -81,12 +84,22 @@ private void SumValues(IList<FacetsCollector.MatchingDocs> matchingDocs)
8184
int offset = scratch.Offset;
8285
while (offset < end)
8386
{
84-
int ord = ((bytes[offset] & 0xFF) << 24) | ((bytes[offset + 1] & 0xFF) << 16) |
85-
((bytes[offset + 2] & 0xFF) << 8) | (bytes[offset + 3] & 0xFF);
86-
offset += 4;
87-
int value = ((bytes[offset] & 0xFF) << 24) | ((bytes[offset + 1] & 0xFF) << 16) |
88-
((bytes[offset + 2] & 0xFF) << 8) | (bytes[offset + 3] & 0xFF);
89-
offset += 4;
87+
// int ord = ((bytes[offset] & 0xFF) << 24) | ((bytes[offset + 1] & 0xFF) << 16) |
88+
// ((bytes[offset + 2] & 0xFF) << 8) | (bytes[offset + 3] & 0xFF);
89+
// offset += 4;
90+
91+
// LUCENENET: Use BinaryPrimitives for JIT-intrinsics opportunity
92+
int ord = BinaryPrimitives.ReadInt32BigEndian(bytes.AsSpan(offset, sizeof(int)));
93+
offset += sizeof(int);
94+
95+
// int value = ((bytes[offset] & 0xFF) << 24) | ((bytes[offset + 1] & 0xFF) << 16) |
96+
// ((bytes[offset + 2] & 0xFF) << 8) | (bytes[offset + 3] & 0xFF);
97+
// offset += 4;
98+
99+
// LUCENENET: Use BinaryPrimitives for JIT-intrinsics opportunity
100+
int value = BinaryPrimitives.ReadInt32BigEndian(bytes.AsSpan(offset, sizeof(int)));
101+
offset += sizeof(int);
102+
90103
m_values[ord] += value;
91104
}
92105
}

‎src/Lucene.Net.Tests.Analysis.Common/Analysis/Hunspell/TestDictionary.cs‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
// Lucene version compatibility level 4.10.4
22
using J2N.Text;
3+
using Lucene.Net.Attributes;
34
using Lucene.Net.Util;
45
using Lucene.Net.Util.Fst;
56
using NUnit.Framework;
@@ -281,5 +282,40 @@ public virtual void TestFlagWithCrazyWhitespace()
281282
assertNotNull(Dictionary.GetFlagParsingStrategy("FLAG\tUTF-8"));
282283
assertNotNull(Dictionary.GetFlagParsingStrategy("FLAG UTF-8"));
283284
}
285+
286+
// LUCENENET specific: direct round-trip for EncodeFlags/DecodeFlags. EncodeFlags was
287+
// rewritten to use BinaryPrimitives.WriteUInt16BigEndian; this guards the big-endian
288+
// char layout (including chars above 0x7FFF, which must not sign-extend) against
289+
// regressions. The existing tests only exercise DecodeFlags off dictionary data.
290+
[Test]
291+
[LuceneNetSpecific]
292+
public virtual void TestEncodeDecodeFlags_RoundTrips()
293+
{
294+
char[][] cases =
295+
{
296+
new char[] { }, // empty
297+
new char[] { (char)0 },
298+
new char[] { 'A' },
299+
new char[] { 'ÿ' }, // low byte set
300+
new char[] { 'Ā' }, // high byte set
301+
new char[] { '耀' }, // top bit set - must not sign-extend
302+
new char[] { '￿' }, // all bits set
303+
new char[] { 'a', 'b', 'c', 'd' },
304+
new char[] { 'ሴ', 'ꯍ', (char)0, '￿' },
305+
};
306+
307+
foreach (char[] flags in cases)
308+
{
309+
BytesRef b = new BytesRef();
310+
Dictionary.EncodeFlags(b, flags);
311+
assertEquals(flags.Length * 2, b.Length);
312+
char[] decoded = Dictionary.DecodeFlags(b);
313+
assertEquals(flags.Length, decoded.Length);
314+
for (int i = 0; i < flags.Length; i++)
315+
{
316+
assertEquals("flag[" + i + "]", flags[i], decoded[i]);
317+
}
318+
}
319+
}
284320
}
285321
}

0 commit comments

Comments
 (0)