Skip to content

Commit c698884

Browse files
Xor-elXor-el
authored
Add AArch64 NEON implementation for XXH3 (#82)
1 parent ac3e2f5 commit c698884

4 files changed

Lines changed: 249 additions & 1 deletion

File tree

‎HashLib/src/Hash64/HlpXXHash3Dispatch.pas‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,7 @@ procedure XXH3_InitSecret_Avx2(ACustomSecret: Pointer;
173173

174174
{$ENDIF HASHLIB_X86_64_ASM}
175175

176-
{$IFDEF HASHLIB_X86_SIMD}
176+
{$IF DEFINED(HASHLIB_X86_SIMD) OR DEFINED(HASHLIB_ARM_SIMD)}
177177

178178
type
179179
TXXH3_Accumulate512Proc = procedure(AAcc, AInput, ASecret: Pointer);
@@ -188,6 +188,10 @@ procedure XXH3_Accumulate_Loop(AAcc: Pointer; AInput: Pointer;
188188
PByte(ASecret) + N * XXH_SECRET_CONSUME_RATE);
189189
end;
190190

191+
{$ENDIF}
192+
193+
{$IFDEF HASHLIB_X86_SIMD}
194+
191195
procedure XXH3_Accumulate_Sse2(AAcc: Pointer; AInput: Pointer;
192196
ASecret: Pointer; ANbStripes: Int32);
193197
begin
@@ -206,6 +210,41 @@ procedure XXH3_Accumulate_Avx2(AAcc: Pointer; AInput: Pointer;
206210

207211
{$ENDIF HASHLIB_X86_64_ASM}
208212

213+
// =============================================================================
214+
// SIMD implementations: AArch64 NEON
215+
// =============================================================================
216+
217+
{$IFDEF HASHLIB_AARCH64_ASM}
218+
219+
procedure XXH3_Accumulate512_Neon(AAcc: Pointer; AInput: Pointer;
220+
ASecret: Pointer);
221+
{$I ..\Include\Simd\Common\SimdProc3Begin_aarch64.inc}
222+
{$I ..\Include\Simd\XXH3\XXH3Acc512Neon_aarch64.inc}
223+
end;
224+
225+
procedure XXH3_ScrambleAcc_Neon(AAcc: Pointer; ASecret: Pointer);
226+
{$I ..\Include\Simd\Common\SimdProc2Begin_aarch64.inc}
227+
{$I ..\Include\Simd\XXH3\XXH3ScrambleNeon_aarch64.inc}
228+
end;
229+
230+
procedure XXH3_InitSecret_Neon(ACustomSecret: Pointer;
231+
ADefaultSecret: Pointer; ASeed: UInt64);
232+
{$I ..\Include\Simd\Common\SimdProc3Begin_aarch64.inc}
233+
{$I ..\Include\Simd\XXH3\XXH3InitSecretNeon_aarch64.inc}
234+
end;
235+
236+
{$IFDEF HASHLIB_ARM_SIMD}
237+
238+
procedure XXH3_Accumulate_Neon(AAcc: Pointer; AInput: Pointer;
239+
ASecret: Pointer; ANbStripes: Int32);
240+
begin
241+
XXH3_Accumulate_Loop(AAcc, AInput, ASecret, ANbStripes, @XXH3_Accumulate512_Neon);
242+
end;
243+
244+
{$ENDIF HASHLIB_ARM_SIMD}
245+
246+
{$ENDIF HASHLIB_AARCH64_ASM}
247+
209248
// =============================================================================
210249
// Dispatch initialization
211250
// =============================================================================
@@ -245,6 +284,17 @@ procedure InitDispatch();
245284
end;
246285
end;
247286
{$ENDIF}
287+
{$IFDEF HASHLIB_AARCH64_ASM}
288+
case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of
289+
TArmSimdLevel.NEON:
290+
begin
291+
XXH3_Accumulate512 := @XXH3_Accumulate512_Neon;
292+
XXH3_Accumulate := @XXH3_Accumulate_Neon;
293+
XXH3_ScrambleAcc := @XXH3_ScrambleAcc_Neon;
294+
XXH3_InitSecret := @XXH3_InitSecret_Neon;
295+
end;
296+
end;
297+
{$ENDIF}
248298
end;
249299

250300
initialization

‎HashLib/src/Include/Simd/XXH3/XXH3Acc512Neon_aarch64.inc‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
// XXH3 accumulate_512 AArch64 NEON implementation (one 64-byte stripe).
2+
// Expects AAPCS64: x0 = acc ptr (8 x UInt64, 16-byte aligned), x1 = input ptr,
3+
// x2 = secret ptr.
4+
// Touches only caller-saved registers (x0-x2, v0-v7, v16-v19); no AAPCS64
5+
// callee-saved register is used, so the 'nostackframe' leaf needs no prologue.
6+
// Reference (golden): HashLib XXH3Acc512Sse2_x86_64.inc (semantic parity);
7+
// Cyan4973/xxHash xxhash.h XXH3_accumulate_512_neon with XXH3_NEON_LANES=8
8+
// (2 batch iterations, no scalar tail).
9+
//
10+
// Instruction encodings (for assembler compatibility):
11+
// FPC 3.2.2's inline assembler cannot encode AArch64 vector mnemonics
12+
// (subscripted vector regs require FPC 3.3+, and ldr/str/ldp with qN crash the
13+
// 3.2.2 assembler). Each such instruction is therefore emitted as its raw 32-bit
14+
// '.long' opcode, with the equivalent mnemonic in the trailing comment; only
15+
// neg/ret stay as text.
16+
//
17+
// Register map:
18+
// v0-v3 = acc[0..3] (4 x uint64x2)
19+
// v4-v5 = data_vec_1/2 (input)
20+
// v6-v7 = key_vec / uzp temps (data_key_lo, data_key_hi)
21+
// v16-v17 = data_swap_1/2 (vext input by 8 bytes)
22+
// v18-v19 = sum_1/sum_2 (umlal / umlal2 widen-multiply-add into swap)
23+
// --- batch i=0: acc[0..1], input[0x0..], secret[0x0..] ---
24+
.long 0x3dc00024 // ldr q4, [x1, #0x0]
25+
.long 0x3dc00425 // ldr q5, [x1, #0x10]
26+
.long 0x3dc00046 // ldr q6, [x2, #0x0]
27+
.long 0x3dc00447 // ldr q7, [x2, #0x10]
28+
.long 0x6e044090 // ext v16.16b, v4.16b, v4.16b, #8
29+
.long 0x6e0540b1 // ext v17.16b, v5.16b, v5.16b, #8
30+
.long 0x6e261c84 // eor v4.16b, v4.16b, v6.16b
31+
.long 0x6e271ca5 // eor v5.16b, v5.16b, v7.16b
32+
.long 0x4e871886 // uzp1 v6.4s, v4.4s, v5.4s
33+
.long 0x4e875887 // uzp2 v7.4s, v4.4s, v5.4s
34+
.long 0x2ea780d0 // umlal v16.2d, v6.2s, v7.2s
35+
.long 0x6ea780d1 // umlal2 v17.2d, v6.4s, v7.4s
36+
.long 0x3dc00000 // ldr q0, [x0, #0x0]
37+
.long 0x3dc00401 // ldr q1, [x0, #0x10]
38+
.long 0x4ef08400 // add v0.2d, v0.2d, v16.2d
39+
.long 0x4ef18421 // add v1.2d, v1.2d, v17.2d
40+
.long 0x3d800000 // str q0, [x0, #0x0]
41+
.long 0x3d800401 // str q1, [x0, #0x10]
42+
// --- batch i=2: acc[2..3], input[0x20..], secret[0x20..] ---
43+
.long 0x3dc00824 // ldr q4, [x1, #0x20]
44+
.long 0x3dc00c25 // ldr q5, [x1, #0x30]
45+
.long 0x3dc00846 // ldr q6, [x2, #0x20]
46+
.long 0x3dc00c47 // ldr q7, [x2, #0x30]
47+
.long 0x6e044090 // ext v16.16b, v4.16b, v4.16b, #8
48+
.long 0x6e0540b1 // ext v17.16b, v5.16b, v5.16b, #8
49+
.long 0x6e261c84 // eor v4.16b, v4.16b, v6.16b
50+
.long 0x6e271ca5 // eor v5.16b, v5.16b, v7.16b
51+
.long 0x4e871886 // uzp1 v6.4s, v4.4s, v5.4s
52+
.long 0x4e875887 // uzp2 v7.4s, v4.4s, v5.4s
53+
.long 0x2ea780d0 // umlal v16.2d, v6.2s, v7.2s
54+
.long 0x6ea780d1 // umlal2 v17.2d, v6.4s, v7.4s
55+
.long 0x3dc00802 // ldr q2, [x0, #0x20]
56+
.long 0x3dc00c03 // ldr q3, [x0, #0x30]
57+
.long 0x4ef08442 // add v2.2d, v2.2d, v16.2d
58+
.long 0x4ef18463 // add v3.2d, v3.2d, v17.2d
59+
.long 0x3d800802 // str q2, [x0, #0x20]
60+
.long 0x3d800c03 // str q3, [x0, #0x30]
61+
ret

‎HashLib/src/Include/Simd/XXH3/XXH3InitSecretNeon_aarch64.inc‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
// XXH3 initCustomSecret AArch64 NEON implementation (192-byte custom secret).
2+
// Expects AAPCS64: x0 = customSecret ptr, x1 = defaultSecret ptr, x2 = seed (UInt64).
3+
// Touches caller-saved x2-x3, v0, v16-v17; leaf, no stack frame.
4+
// Reference (golden): HashLib XXH3InitSecretSse2_x86_64.inc (12 x 16-byte blocks:
5+
// lo qword += seed, hi qword -= seed). No upstream NEON equivalent.
6+
//
7+
// Instruction encodings (for assembler compatibility):
8+
// FPC 3.2.2's inline assembler cannot encode AArch64 vector mnemonics
9+
// (subscripted vector regs require FPC 3.3+, and ldr/str/ldp with qN crash the
10+
// 3.2.2 assembler). Each such instruction is therefore emitted as its raw 32-bit
11+
// '.long' opcode, with the equivalent mnemonic in the trailing comment; only
12+
// neg/ret stay as text.
13+
//
14+
// Register map:
15+
// x3 = negated seed (-seed)
16+
// v0 = vSeed broadcast vector [seed, -seed]
17+
// v16-v17 = fmov staging for zip1 pack
18+
// v2 = defaultSecret block load / add temp
19+
neg x3, x2
20+
.long 0x9e670050 // fmov d16, x2
21+
.long 0x9e670071 // fmov d17, x3
22+
.long 0x4ed13a00 // zip1 v0.2d, v16.2d, v17.2d
23+
// --- block 0: defaultSecret[0x0..] ---
24+
.long 0x3dc00022 // ldr q2, [x1, #0x0]
25+
.long 0x4ee08442 // add v2.2d, v2.2d, v0.2d
26+
.long 0x3d800002 // str q2, [x0, #0x0]
27+
// --- block 1: defaultSecret[0x10..] ---
28+
.long 0x3dc00422 // ldr q2, [x1, #0x10]
29+
.long 0x4ee08442 // add v2.2d, v2.2d, v0.2d
30+
.long 0x3d800402 // str q2, [x0, #0x10]
31+
// --- block 2: defaultSecret[0x20..] ---
32+
.long 0x3dc00822 // ldr q2, [x1, #0x20]
33+
.long 0x4ee08442 // add v2.2d, v2.2d, v0.2d
34+
.long 0x3d800802 // str q2, [x0, #0x20]
35+
// --- block 3: defaultSecret[0x30..] ---
36+
.long 0x3dc00c22 // ldr q2, [x1, #0x30]
37+
.long 0x4ee08442 // add v2.2d, v2.2d, v0.2d
38+
.long 0x3d800c02 // str q2, [x0, #0x30]
39+
// --- block 4: defaultSecret[0x40..] ---
40+
.long 0x3dc01022 // ldr q2, [x1, #0x40]
41+
.long 0x4ee08442 // add v2.2d, v2.2d, v0.2d
42+
.long 0x3d801002 // str q2, [x0, #0x40]
43+
// --- block 5: defaultSecret[0x50..] ---
44+
.long 0x3dc01422 // ldr q2, [x1, #0x50]
45+
.long 0x4ee08442 // add v2.2d, v2.2d, v0.2d
46+
.long 0x3d801402 // str q2, [x0, #0x50]
47+
// --- block 6: defaultSecret[0x60..] ---
48+
.long 0x3dc01822 // ldr q2, [x1, #0x60]
49+
.long 0x4ee08442 // add v2.2d, v2.2d, v0.2d
50+
.long 0x3d801802 // str q2, [x0, #0x60]
51+
// --- block 7: defaultSecret[0x70..] ---
52+
.long 0x3dc01c22 // ldr q2, [x1, #0x70]
53+
.long 0x4ee08442 // add v2.2d, v2.2d, v0.2d
54+
.long 0x3d801c02 // str q2, [x0, #0x70]
55+
// --- block 8: defaultSecret[0x80..] ---
56+
.long 0x3dc02022 // ldr q2, [x1, #0x80]
57+
.long 0x4ee08442 // add v2.2d, v2.2d, v0.2d
58+
.long 0x3d802002 // str q2, [x0, #0x80]
59+
// --- block 9: defaultSecret[0x90..] ---
60+
.long 0x3dc02422 // ldr q2, [x1, #0x90]
61+
.long 0x4ee08442 // add v2.2d, v2.2d, v0.2d
62+
.long 0x3d802402 // str q2, [x0, #0x90]
63+
// --- block 10: defaultSecret[0xa0..] ---
64+
.long 0x3dc02822 // ldr q2, [x1, #0xa0]
65+
.long 0x4ee08442 // add v2.2d, v2.2d, v0.2d
66+
.long 0x3d802802 // str q2, [x0, #0xa0]
67+
// --- block 11: defaultSecret[0xb0..] ---
68+
.long 0x3dc02c22 // ldr q2, [x1, #0xb0]
69+
.long 0x4ee08442 // add v2.2d, v2.2d, v0.2d
70+
.long 0x3d802c02 // str q2, [x0, #0xb0]
71+
ret

‎HashLib/src/Include/Simd/XXH3/XXH3ScrambleNeon_aarch64.inc‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
// XXH3 scrambleAcc AArch64 NEON implementation (64-byte secret slice).
2+
// Expects AAPCS64: x0 = acc ptr (8 x UInt64, 16-byte aligned), x1 = secret ptr.
3+
// Touches only caller-saved registers (x0-x1, w10, v0-v5); leaf, no stack frame.
4+
// Reference (golden): HashLib XXH3ScrambleSse2_x86_64.inc;
5+
// Cyan4973/xxHash xxhash.h XXH3_scrambleAcc_neon (4 x uint64x2 chunks).
6+
//
7+
// Instruction encodings (for assembler compatibility):
8+
// FPC 3.2.2's inline assembler cannot encode AArch64 vector mnemonics
9+
// (subscripted vector regs require FPC 3.3+, and ldr/str/ldp with qN crash the
10+
// 3.2.2 assembler). Each such instruction is therefore emitted as its raw 32-bit
11+
// '.long' opcode, with the equivalent mnemonic in the trailing comment; only
12+
// mov/ret stay as text.
13+
//
14+
// Register map:
15+
// w10 = XXH_PRIME32_1 constant loader
16+
// v4 = kPrimeHi ({0, PRIME32_1, 0, PRIME32_1} as u32x4)
17+
// v5 = kPrimeLo (PRIME32_1 broadcast to u32x2)
18+
// v3 = prod_hi (vmul u32) / final chunk result
19+
// v1 = shifted acc, secret key, or data_key_lo (xtn)
20+
.long 0x528f362a // movz w10, #0x79b1, lsl #0
21+
.long 0x72b3c6ea // movk w10, #0x9e37, lsl #16
22+
.long 0x0e040d45 // dup v5.2s, w10
23+
.long 0x6e241c84 // eor v4.16b, v4.16b, v4.16b
24+
.long 0x4e0c1d44 // ins v4.s[1], w10
25+
.long 0x4e1c1d44 // ins v4.s[3], w10
26+
// --- chunk 0: acc[0..1], secret[0x0..] ---
27+
.long 0x3dc00000 // ldr q0, [x0, #0x0]
28+
.long 0x6f510401 // ushr v1.2d, v0.2d, #47
29+
.long 0x6e211c00 // eor v0.16b, v0.16b, v1.16b
30+
.long 0x3dc00021 // ldr q1, [x1, #0x0]
31+
.long 0x6e211c00 // eor v0.16b, v0.16b, v1.16b
32+
.long 0x4ea69c03 // mul v3.4s, v0.4s, v4.4s
33+
.long 0x0ea12801 // xtn v1.2s, v0.2d
34+
.long 0x2ea78023 // umlal v3.2d, v1.2s, v5.2s
35+
.long 0x3d800003 // str q3, [x0, #0x0]
36+
// --- chunk 1: acc[2..3], secret[0x10..] ---
37+
.long 0x3dc00400 // ldr q0, [x0, #0x10]
38+
.long 0x6f510401 // ushr v1.2d, v0.2d, #47
39+
.long 0x6e211c00 // eor v0.16b, v0.16b, v1.16b
40+
.long 0x3dc00421 // ldr q1, [x1, #0x10]
41+
.long 0x6e211c00 // eor v0.16b, v0.16b, v1.16b
42+
.long 0x4ea69c03 // mul v3.4s, v0.4s, v4.4s
43+
.long 0x0ea12801 // xtn v1.2s, v0.2d
44+
.long 0x2ea78023 // umlal v3.2d, v1.2s, v5.2s
45+
.long 0x3d800403 // str q3, [x0, #0x10]
46+
// --- chunk 2: acc[4..5], secret[0x20..] ---
47+
.long 0x3dc00800 // ldr q0, [x0, #0x20]
48+
.long 0x6f510401 // ushr v1.2d, v0.2d, #47
49+
.long 0x6e211c00 // eor v0.16b, v0.16b, v1.16b
50+
.long 0x3dc00821 // ldr q1, [x1, #0x20]
51+
.long 0x6e211c00 // eor v0.16b, v0.16b, v1.16b
52+
.long 0x4ea69c03 // mul v3.4s, v0.4s, v4.4s
53+
.long 0x0ea12801 // xtn v1.2s, v0.2d
54+
.long 0x2ea78023 // umlal v3.2d, v1.2s, v5.2s
55+
.long 0x3d800803 // str q3, [x0, #0x20]
56+
// --- chunk 3: acc[6..7], secret[0x30..] ---
57+
.long 0x3dc00c00 // ldr q0, [x0, #0x30]
58+
.long 0x6f510401 // ushr v1.2d, v0.2d, #47
59+
.long 0x6e211c00 // eor v0.16b, v0.16b, v1.16b
60+
.long 0x3dc00c21 // ldr q1, [x1, #0x30]
61+
.long 0x6e211c00 // eor v0.16b, v0.16b, v1.16b
62+
.long 0x4ea69c03 // mul v3.4s, v0.4s, v4.4s
63+
.long 0x0ea12801 // xtn v1.2s, v0.2d
64+
.long 0x2ea78023 // umlal v3.2d, v1.2s, v5.2s
65+
.long 0x3d800c03 // str q3, [x0, #0x30]
66+
ret

0 commit comments

Comments
 (0)