-
-
Notifications
You must be signed in to change notification settings - Fork 92
Expand file tree
/
Copy pathArgon2FillBlockAvx2_x86_64.inc
More file actions
323 lines (265 loc) · 16.4 KB
/
Copy pathArgon2FillBlockAvx2_x86_64.inc
File metadata and controls
323 lines (265 loc) · 16.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
// AVX2 implementation of Argon2 FillBlock (BlaMka round function).
// Rotations: rot32 via vpshufd, rot24/16 via vpshufb against the resident
// masks, rot63 via shift+or (not byte-aligned). Diagonalize/Undiagonalize
// via vpermq (cross-lane 64-bit permute). vzeroupper required before return.
// ABI (after HlpSimdProc5Begin_x86_64.inc): rcx = Left ptr, rdx = Right ptr,
// r8 = Current ptr, r9 = WithXor (0 or 1), r10 = byte-rotation masks ptr
// (ARGON2_ROT_MASKS: rot24 at +0, rot16 at +32; read unaligned, so the
// Pascal const needs no special alignment; consumed by the entry mask
// loads, r10 is scratch afterwards).
// Each pointer addresses 128 QWords (1024 bytes). ARight and ACurrent may
// alias; R_buf buffering handles this. r8/r9 survive all loops.
// Register map (G rounds): ymm0 = A(v0..v3), ymm1 = B(v4..v7),
// ymm2 = C(v8..v11), ymm3 = D(v12..v15), ymm4-ymm5 = temps,
// ymm10 = rot24 mask, ymm11 = rot16 mask.
// Frame (sub rsp, 2184):
// [rsp+0..127] = ymm6-ymm9 save area (4 x 32 bytes)
// [rsp+128..1151] = R_buf (1024 bytes)
// [rsp+1152..2175] = Z_buf (1024 bytes)
// [rsp+2176..2183] = alignment padding
// Saves: ymm6-ymm11 (MS x64 non-volatile); uses ymm0-ymm11.
// AVX/AVX2 instructions are db-encoded for broad assembler compatibility.
// Reference: official Argon2 AVX2 (blamka-round-opt.h).
{$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc}
// Load byte-rotation masks (resident for the whole kernel)
db $C4, $41, $7E, $6F, $12 // vmovdqu ymm10, yword [r10]
db $C4, $41, $7E, $6F, $5A, $20 // vmovdqu ymm11, yword [r10 + $20]
sub rsp, 2056
// =========================================================================
// Step 1: Compute R_buf = Left XOR Right, store at [rsp+128]
// =========================================================================
lea rax, [rsp]
xor r10, r10
@xor_loop:
db $C4, $A1, $7E, $6F, $04, $11 // vmovdqu ymm0, yword [rcx + r10]
db $C4, $A1, $7D, $EF, $04, $12 // vpxor ymm0, ymm0, yword [rdx + r10]
db $C4, $A1, $7E, $7F, $04, $10 // vmovdqu yword [rax + r10], ymm0
add r10, 32
cmp r10, 1024
jb @xor_loop
// =========================================================================
// Step 2: Copy R_buf to Z_buf at [rsp+1152]
// =========================================================================
lea r11, [rsp + 1024]
xor r10, r10
@copy_loop:
db $C4, $A1, $7E, $6F, $04, $10 // vmovdqu ymm0, yword [rax + r10]
db $C4, $81, $7E, $7F, $04, $13 // vmovdqu yword [r11 + r10], ymm0
add r10, 32
cmp r10, 1024
jb @copy_loop
// =========================================================================
// Step 3: Column rounds on Z_buf (8 iterations, 16 QWords = 128 bytes each)
// =========================================================================
// Each iteration: load 4 ywords (128 bytes) into ymm0-3 (A,B,C,D),
// apply BLAKE2_ROUND_NOMSG (columns + diagonals), store back.
xor r10, r10
@col_loop:
db $C4, $81, $7E, $6F, $04, $13 // vmovdqu ymm0, yword [r11 + r10]
db $C4, $81, $7E, $6F, $4C, $13, $20 // vmovdqu ymm1, yword [r11 + r10 + $20]
db $C4, $81, $7E, $6F, $54, $13, $40 // vmovdqu ymm2, yword [r11 + r10 + $40]
db $C4, $81, $7E, $6F, $5C, $13, $60 // vmovdqu ymm3, yword [r11 + r10 + $60]
// ----- Column G: G(A,B,C,D) across all 4 lanes -----
// fBlaMka(a, b): a = a + b + 2*lo32(a)*lo32(b)
db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, 1
db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1
db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4
// d = rotr64(d ^ a, 32)
db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1
// fBlaMka(c, d)
db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, 1
db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3
db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4
// b = rotr64(b ^ c, 24)
db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
db $C4, $C2, $75, $00, $CA // vpshufb ymm1, ymm1, ymm10
// fBlaMka(a, b)
db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, 1
db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1
db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4
// d = rotr64(d ^ a, 16)
db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
db $C4, $C2, $65, $00, $DB // vpshufb ymm3, ymm3, ymm11
// fBlaMka(c, d)
db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, 1
db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3
db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4
// b = rotr64(b ^ c, 63)
db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
db $C5, $DD, $73, $D1, $3F // vpsrlq ymm4, ymm1, 63
db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1
db $C5, $F5, $EB, $CC // vpor ymm1, ymm1, ymm4
// ----- Diagonalize -----
// B: rotate left by 1 QWord position
db $C4, $E3, $FD, $00, $C9, $39 // vpermq ymm1, ymm1, $39
// C: swap halves (rotate by 2)
db $C4, $E3, $FD, $00, $D2, $4E // vpermq ymm2, ymm2, $4E
// D: rotate right by 1 QWord position
db $C4, $E3, $FD, $00, $DB, $93 // vpermq ymm3, ymm3, $93
// ----- Diagonal G -----
db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, 1
db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1
db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4
db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1
db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, 1
db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3
db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4
db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
db $C4, $C2, $75, $00, $CA // vpshufb ymm1, ymm1, ymm10
db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, 1
db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1
db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4
db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
db $C4, $C2, $65, $00, $DB // vpshufb ymm3, ymm3, ymm11
db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, 1
db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3
db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4
db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
db $C5, $DD, $73, $D1, $3F // vpsrlq ymm4, ymm1, 63
db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1
db $C5, $F5, $EB, $CC // vpor ymm1, ymm1, ymm4
// ----- Undiagonalize -----
db $C4, $E3, $FD, $00, $C9, $93 // vpermq ymm1, ymm1, $93
db $C4, $E3, $FD, $00, $D2, $4E // vpermq ymm2, ymm2, $4E
db $C4, $E3, $FD, $00, $DB, $39 // vpermq ymm3, ymm3, $39
// ----- Store back to Z_buf -----
db $C4, $81, $7E, $7F, $04, $13 // vmovdqu yword [r11 + r10], ymm0
db $C4, $81, $7E, $7F, $4C, $13, $20 // vmovdqu yword [r11 + r10 + $20], ymm1
db $C4, $81, $7E, $7F, $54, $13, $40 // vmovdqu yword [r11 + r10 + $40], ymm2
db $C4, $81, $7E, $7F, $5C, $13, $60 // vmovdqu yword [r11 + r10 + $60], ymm3
add r10, 128
cmp r10, 1024
jb @col_loop
// =========================================================================
// Step 4: Row rounds on Z_buf (8 iterations, stride-16 QWord access)
// =========================================================================
// Row i: Z[2i,2i+1, 2i+16,2i+17, 2i+32,2i+33, 2i+48,2i+49,
// 2i+64,2i+65, 2i+80,2i+81, 2i+96,2i+97, 2i+112,2i+113]
// Load via vmovdqu xmm + vinserti128 to gather non-contiguous pairs.
// Stride between pairs = 128 bytes.
// ymm0 = A = (Z[2i..2i+1], Z[2i+16..2i+17])
// ymm1 = B = (Z[2i+32..2i+33], Z[2i+48..2i+49])
// ymm2 = C = (Z[2i+64..2i+65], Z[2i+80..2i+81])
// ymm3 = D = (Z[2i+96..2i+97], Z[2i+112..2i+113])
xor r10, r10
@row_loop:
db $C4, $81, $7A, $6F, $04, $13 // vmovdqu xmm0, oword [r11 + r10]
db $C4, $83, $7D, $38, $84, $13, $80, $00, $00, $00, $01 // vinserti128 ymm0, ymm0, oword [r11 + r10 + 128], 1
db $C4, $81, $7A, $6F, $8C, $13, $00, $01, $00, $00 // vmovdqu xmm1, oword [r11 + r10 + 256]
db $C4, $83, $75, $38, $8C, $13, $80, $01, $00, $00, $01 // vinserti128 ymm1, ymm1, oword [r11 + r10 + 384], 1
db $C4, $81, $7A, $6F, $94, $13, $00, $02, $00, $00 // vmovdqu xmm2, oword [r11 + r10 + 512]
db $C4, $83, $6D, $38, $94, $13, $80, $02, $00, $00, $01 // vinserti128 ymm2, ymm2, oword [r11 + r10 + 640], 1
db $C4, $81, $7A, $6F, $9C, $13, $00, $03, $00, $00 // vmovdqu xmm3, oword [r11 + r10 + 768]
db $C4, $83, $65, $38, $9C, $13, $80, $03, $00, $00, $01 // vinserti128 ymm3, ymm3, oword [r11 + r10 + 896], 1
// ----- Column G -----
db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, 1
db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1
db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4
db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1
db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, 1
db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3
db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4
db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
db $C4, $C2, $75, $00, $CA // vpshufb ymm1, ymm1, ymm10
db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, 1
db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1
db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4
db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
db $C4, $C2, $65, $00, $DB // vpshufb ymm3, ymm3, ymm11
db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, 1
db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3
db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4
db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
db $C5, $DD, $73, $D1, $3F // vpsrlq ymm4, ymm1, 63
db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1
db $C5, $F5, $EB, $CC // vpor ymm1, ymm1, ymm4
// ----- Diagonalize -----
db $C4, $E3, $FD, $00, $C9, $39 // vpermq ymm1, ymm1, $39
db $C4, $E3, $FD, $00, $D2, $4E // vpermq ymm2, ymm2, $4E
db $C4, $E3, $FD, $00, $DB, $93 // vpermq ymm3, ymm3, $93
// ----- Diagonal G -----
db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, 1
db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1
db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4
db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1
db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, 1
db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3
db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4
db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
db $C4, $C2, $75, $00, $CA // vpshufb ymm1, ymm1, ymm10
db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, 1
db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1
db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4
db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
db $C4, $C2, $65, $00, $DB // vpshufb ymm3, ymm3, ymm11
db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, 1
db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3
db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4
db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
db $C5, $DD, $73, $D1, $3F // vpsrlq ymm4, ymm1, 63
db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1
db $C5, $F5, $EB, $CC // vpor ymm1, ymm1, ymm4
// ----- Undiagonalize -----
db $C4, $E3, $FD, $00, $C9, $93 // vpermq ymm1, ymm1, $93
db $C4, $E3, $FD, $00, $D2, $4E // vpermq ymm2, ymm2, $4E
db $C4, $E3, $FD, $00, $DB, $39 // vpermq ymm3, ymm3, $39
// ----- Store back to Z_buf (scatter to stride offsets) -----
db $C4, $81, $7A, $7F, $04, $13 // vmovdqu oword [r11 + r10], xmm0
db $C4, $83, $7D, $39, $84, $13, $80, $00, $00, $00, $01 // vextracti128 oword [r11 + r10 + 128], ymm0, 1
db $C4, $81, $7A, $7F, $8C, $13, $00, $01, $00, $00 // vmovdqu oword [r11 + r10 + 256], xmm1
db $C4, $83, $7D, $39, $8C, $13, $80, $01, $00, $00, $01 // vextracti128 oword [r11 + r10 + 384], ymm1, 1
db $C4, $81, $7A, $7F, $94, $13, $00, $02, $00, $00 // vmovdqu oword [r11 + r10 + 512], xmm2
db $C4, $83, $7D, $39, $94, $13, $80, $02, $00, $00, $01 // vextracti128 oword [r11 + r10 + 640], ymm2, 1
db $C4, $81, $7A, $7F, $9C, $13, $00, $03, $00, $00 // vmovdqu oword [r11 + r10 + 768], xmm3
db $C4, $83, $7D, $39, $9C, $13, $80, $03, $00, $00, $01 // vextracti128 oword [r11 + r10 + 896], ymm3, 1
add r10, 16
cmp r10, 128
jb @row_loop
// =========================================================================
// Step 5: Final XOR - Current = R_buf XOR Z_buf [XOR Current]
// =========================================================================
test r9, r9
jnz @final_with_xor
xor r10, r10
@final_noxor_loop:
db $C4, $A1, $7E, $6F, $04, $10 // vmovdqu ymm0, yword [rax + r10]
db $C4, $81, $7D, $EF, $04, $13 // vpxor ymm0, ymm0, yword [r11 + r10]
db $C4, $81, $7E, $7F, $04, $10 // vmovdqu yword [r8 + r10], ymm0
add r10, 32
cmp r10, 1024
jb @final_noxor_loop
jmp @epilogue
@final_with_xor:
xor r10, r10
@final_xor_loop:
db $C4, $A1, $7E, $6F, $04, $10 // vmovdqu ymm0, yword [rax + r10]
db $C4, $81, $7D, $EF, $04, $13 // vpxor ymm0, ymm0, yword [r11 + r10]
db $C4, $81, $7D, $EF, $04, $10 // vpxor ymm0, ymm0, yword [r8 + r10]
db $C4, $81, $7E, $7F, $04, $10 // vmovdqu yword [r8 + r10], ymm0
add r10, 32
cmp r10, 1024
jb @final_xor_loop
@epilogue:
add rsp, 2056
db $C5, $F8, $77 // vzeroupper
{$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc}