Skip to content

Commit 272064b

Browse files
xoofxxoofx
committed
Improve Perf for ARM64 parent compression
1 parent 5141ff2 commit 272064b

3 files changed

Lines changed: 140 additions & 7 deletions

File tree

‎doc/benchmarks.md‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -288,6 +288,51 @@ Job=ShortRun IterationCount=3 LaunchCount=1 WarmupCount=3
288288
| 'Blake3 default' | 10000000 | 907,587.73 ns | 45,363.456 ns | 2,486.524 ns | 1.00 | 8,441 B | - | NA |
289289
| 'Blake3 native' | 10000000 | 756,075.81 ns | 41,412.564 ns | 2,269.962 ns | 0.83 | 360 B | - | NA |
290290

291+
```
292+
BenchmarkDotNet v0.15.8, macOS Tahoe 26.5.1 (25F80) [Darwin 25.5.0]
293+
Apple M4 Pro, 1 CPU, 14 logical and 14 physical cores
294+
.NET SDK 10.0.201
295+
[Host] : .NET 10.0.5 (10.0.5, 10.0.526.15411), Arm64 RyuJIT armv8.0-a
296+
ShortRun : .NET 10.0.5 (10.0.5, 10.0.526.15411), Arm64 RyuJIT armv8.0-a
297+
298+
Job=ShortRun IterationCount=3 LaunchCount=1 WarmupCount=3
299+
```
300+
301+
| Method | N | Mean | Error | StdDev | Ratio | Allocated | Alloc Ratio |
302+
|----------------- |--------- |----------------:|---------------:|--------------:|------:|----------:|------------:|
303+
| 'Blake3 default' | 4 | 49.87 ns | 1.231 ns | 0.067 ns | 1.00 | - | NA |
304+
| 'Blake3 native' | 4 | 52.63 ns | 1.330 ns | 0.073 ns | 1.06 | - | NA |
305+
| | | | | | | | |
306+
| 'Blake3 default' | 100 | 102.62 ns | 3.250 ns | 0.178 ns | 1.00 | - | NA |
307+
| 'Blake3 native' | 100 | 91.25 ns | 2.549 ns | 0.140 ns | 0.89 | - | NA |
308+
| | | | | | | | |
309+
| 'Blake3 default' | 1000 | 836.55 ns | 20.618 ns | 1.130 ns | 1.00 | - | NA |
310+
| 'Blake3 native' | 1000 | 700.29 ns | 22.154 ns | 1.214 ns | 0.84 | - | NA |
311+
| | | | | | | | |
312+
| 'Blake3 default' | 10000 | 5,218.23 ns | 110.609 ns | 6.063 ns | 1.00 | - | NA |
313+
| 'Blake3 native' | 10000 | 4,514.60 ns | 38.689 ns | 2.121 ns | 0.87 | - | NA |
314+
| | | | | | | | |
315+
| 'Blake3 default' | 65536 | 28,487.52 ns | 821.890 ns | 45.051 ns | 1.00 | - | NA |
316+
| 'Blake3 native' | 65536 | 25,077.70 ns | 136.511 ns | 7.483 ns | 0.88 | - | NA |
317+
| | | | | | | | |
318+
| 'Blake3 default' | 100000 | 44,171.43 ns | 1,142.003 ns | 62.597 ns | 1.00 | - | NA |
319+
| 'Blake3 native' | 100000 | 38,953.93 ns | 189.971 ns | 10.413 ns | 0.88 | - | NA |
320+
| | | | | | | | |
321+
| 'Blake3 default' | 131072 | 57,069.08 ns | 2,428.221 ns | 133.099 ns | 1.00 | - | NA |
322+
| 'Blake3 native' | 131072 | 50,187.54 ns | 441.632 ns | 24.207 ns | 0.88 | - | NA |
323+
| | | | | | | | |
324+
| 'Blake3 default' | 262144 | 114,543.10 ns | 7,969.881 ns | 436.856 ns | 1.00 | - | NA |
325+
| 'Blake3 native' | 262144 | 100,528.36 ns | 1,200.951 ns | 65.828 ns | 0.88 | - | NA |
326+
| | | | | | | | |
327+
| 'Blake3 default' | 524288 | 228,890.88 ns | 6,392.852 ns | 350.414 ns | 1.00 | - | NA |
328+
| 'Blake3 native' | 524288 | 201,074.93 ns | 1,575.382 ns | 86.352 ns | 0.88 | - | NA |
329+
| | | | | | | | |
330+
| 'Blake3 default' | 1000000 | 440,723.65 ns | 9,942.807 ns | 544.999 ns | 1.00 | - | NA |
331+
| 'Blake3 native' | 1000000 | 384,006.50 ns | 12,111.836 ns | 663.891 ns | 0.87 | - | NA |
332+
| | | | | | | | |
333+
| 'Blake3 default' | 10000000 | 4,428,962.46 ns | 300,739.435 ns | 16,484.543 ns | 1.00 | - | NA |
334+
| 'Blake3 native' | 10000000 | 3,846,564.83 ns | 25,030.885 ns | 1,372.027 ns | 0.87 | - | NA |
335+
291336

292337
## ExternalHashBenchmarks
293338

‎src/Blake3/Blake3ManagedCore.Arm64.cs‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,19 @@ private static void StoreTransposed4(Span<Vector128<uint>> chainingValues, Span<
211211
second3.StoreUnsafe(ref outputWord, 28);
212212
}
213213

214+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
215+
private static void StoreTransposed4(ref Vector128<uint> chainingValue, Span<uint> output, int count)
216+
{
217+
ref var outputWord = ref MemoryMarshal.GetReference(output);
218+
Transpose4(ref chainingValue);
219+
Transpose4(ref Unsafe.Add(ref chainingValue, 4));
220+
for (nuint lane = 0; lane < (nuint)count; lane++)
221+
{
222+
Unsafe.Add(ref chainingValue, lane).StoreUnsafe(ref outputWord, lane * 8);
223+
Unsafe.Add(ref chainingValue, lane + 4).StoreUnsafe(ref outputWord, (lane * 8) + 4);
224+
}
225+
}
226+
214227
[MethodImpl(MethodImplOptions.AggressiveInlining)]
215228
private static void Transpose4(
216229
ref Vector128<uint> row0,
@@ -228,6 +241,20 @@ private static void Transpose4(
228241
row3 = AdvSimd.Arm64.ZipHigh(row01High.AsUInt64(), row23High.AsUInt64()).AsUInt32();
229242
}
230243

244+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
245+
private static void Transpose4(ref Vector128<uint> first)
246+
{
247+
var row0 = first;
248+
var row1 = Unsafe.Add(ref first, 1);
249+
var row2 = Unsafe.Add(ref first, 2);
250+
var row3 = Unsafe.Add(ref first, 3);
251+
Transpose4(ref row0, ref row1, ref row2, ref row3);
252+
first = row0;
253+
Unsafe.Add(ref first, 1) = row1;
254+
Unsafe.Add(ref first, 2) = row2;
255+
Unsafe.Add(ref first, 3) = row3;
256+
}
257+
231258
[SkipLocalsInit]
232259
private static int Hash4ChunksArm64(
233260
ReadOnlySpan<byte> input,
@@ -279,6 +306,55 @@ private static int Hash4ChunksArm64(
279306
return Degree;
280307
}
281308

309+
[SkipLocalsInit]
310+
private static bool TryCompress4ParentsArm64(
311+
Span<uint> childChainingValues,
312+
int parentCount,
313+
ReadOnlySpan<uint> keyWords,
314+
uint flags)
315+
{
316+
if (!AdvSimd.Arm64.IsSupported || parentCount is < 1 or > 4)
317+
{
318+
return false;
319+
}
320+
321+
Span<Vector128<uint>> chainingValues = stackalloc Vector128<uint>[8];
322+
Span<Vector128<uint>> message = stackalloc Vector128<uint>[16];
323+
ref var chainingValue = ref MemoryMarshal.GetReference(chainingValues);
324+
ref var messageWord = ref MemoryMarshal.GetReference(message);
325+
ref var childWord = ref MemoryMarshal.GetReference(childChainingValues);
326+
for (var index = 0; index < 8; index++)
327+
{
328+
Unsafe.Add(ref chainingValue, index) = Vector128.Create(keyWords[index]);
329+
}
330+
331+
for (nuint parent = (nuint)parentCount; parent < 4; parent++)
332+
{
333+
Unsafe.Add(ref messageWord, parent) = Vector128<uint>.Zero;
334+
Unsafe.Add(ref messageWord, parent + 4) = Vector128<uint>.Zero;
335+
Unsafe.Add(ref messageWord, parent + 8) = Vector128<uint>.Zero;
336+
Unsafe.Add(ref messageWord, parent + 12) = Vector128<uint>.Zero;
337+
}
338+
339+
for (nuint parent = 0; parent < (nuint)parentCount; parent++)
340+
{
341+
var childOffset = parent * 16;
342+
Unsafe.Add(ref messageWord, parent) = Vector128.LoadUnsafe(ref childWord, childOffset);
343+
Unsafe.Add(ref messageWord, parent + 4) = Vector128.LoadUnsafe(ref childWord, childOffset + 4);
344+
Unsafe.Add(ref messageWord, parent + 8) = Vector128.LoadUnsafe(ref childWord, childOffset + 8);
345+
Unsafe.Add(ref messageWord, parent + 12) = Vector128.LoadUnsafe(ref childWord, childOffset + 12);
346+
}
347+
348+
Transpose4(ref messageWord);
349+
Transpose4(ref Unsafe.Add(ref messageWord, 4));
350+
Transpose4(ref Unsafe.Add(ref messageWord, 8));
351+
Transpose4(ref Unsafe.Add(ref messageWord, 12));
352+
var zero = Vector128<uint>.Zero;
353+
Compress4Arm64(ref chainingValue, ref messageWord, in zero, in zero, flags | Parent);
354+
StoreTransposed4(ref chainingValue, childChainingValues, parentCount);
355+
return true;
356+
}
357+
282358
[SkipLocalsInit]
283359
private static void Compress4Arm64(
284360
ref Vector128<uint> chainingValue,

‎src/Blake3/Blake3ManagedCore.cs‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -897,13 +897,17 @@ internal static void ReduceChunkChainingValues(
897897
while (chunkCount > 2)
898898
{
899899
var parentCount = chunkCount / 2;
900-
if (parentCount <= Vector<uint>.Count)
900+
if (parentCount > 4 ||
901+
!TryCompress4ParentsArm64(chainingValues, parentCount, keyWords, flags))
901902
{
902-
CompressParentsVector(chainingValues, parentCount, keyWords, flags);
903-
}
904-
else
905-
{
906-
CompressParentsScalar(chainingValues, parentCount, keyWords, flags);
903+
if (parentCount <= Vector<uint>.Count)
904+
{
905+
CompressParentsVector(chainingValues, parentCount, keyWords, flags);
906+
}
907+
else
908+
{
909+
CompressParentsScalar(chainingValues, parentCount, keyWords, flags);
910+
}
907911
}
908912

909913
chunkCount = parentCount;
@@ -1326,8 +1330,16 @@ private static int CompressParentsParallel(
13261330
var parentCount = childCount / 2;
13271331
if (parentCount < 9 || !TryCompress16Parents(childChainingValues, parentCount, keyWords, flags))
13281332
{
1329-
var parentsCompressed = parentCount >= 2 &&
1333+
var parentsCompressed = TryCompress4ParentsArm64(
1334+
childChainingValues,
1335+
parentCount,
1336+
keyWords,
1337+
flags);
1338+
if (!parentsCompressed)
1339+
{
1340+
parentsCompressed = parentCount >= 2 &&
13301341
TryCompress8Parents(childChainingValues, parentCount, keyWords, flags);
1342+
}
13311343
if (!parentsCompressed)
13321344
{
13331345
parentsCompressed = TryCompressParentsX86(childChainingValues, parentCount, keyWords, flags);

0 commit comments

Comments
 (0)