-
-
Notifications
You must be signed in to change notification settings - Fork 92
Expand file tree
/
Copy pathArgon2FillBlockAvx2_i386.inc
More file actions
276 lines (259 loc) · 15.7 KB
/
Copy pathArgon2FillBlockAvx2_i386.inc
File metadata and controls
276 lines (259 loc) · 15.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
// AVX2 implementation of Argon2 FillBlock (BlaMka round function; IA-32).
// 256-bit ymm integer ops work in 32-bit mode; same algorithm and buffer
// layout as Argon2FillBlockAvx2_x86_64.inc, redesigned for the IA-32
// register budget:
// - the byte-rotation masks are VEX memory operands (rot24 at [masks],
// rot16 at [masks+$20]; read unaligned, so the Pascal const needs no
// special alignment) - the kernel touches only ymm0-ymm4;
// - R_buf/Z_buf sit at [esp]/[esp+1024] and are addressed [esp+edx+disp]
// (edx = loop offset) - no buffer pointer registers are needed;
// - Left/Right are dead after step 1's XOR.
// Diagonalize/Undiagonalize via vpermq. vzeroupper required before return.
// ABI (after HlpSimdProc5Begin_i386.inc): ebx = Left ptr, esi = Right ptr,
// edi = Current ptr, eax = WithXor (0 or 1; live until the final step),
// ecx = byte-rotation masks ptr (ARGON2_ROT_MASKS; live throughout).
// Each pointer addresses 128 QWords (1024 bytes). ARight and ACurrent may
// alias; R_buf buffering handles this.
// Frame (sub esp, 2048): [esp+0..1023] R_buf, [esp+1024..2047] Z_buf.
// Saves: none (uses ymm0-ymm4 only; all volatile on IA-32).
// AVX/AVX2 instructions are db-encoded for broad assembler compatibility.
// Reference: official Argon2 AVX2 (blamka-round-opt.h), HashLib
// Argon2FillBlockAvx2_x86_64.inc.
sub esp, $800
// Step 1: R_buf = Left XOR Right at [esp] (Left/Right dead afterwards)
xor edx, edx
@xor_loop:
db $C5, $FE, $6F, $04, $13 // vmovdqu ymm0, yword [ebx + edx*1]
db $C5, $FD, $EF, $04, $16 // vpxor ymm0, ymm0, yword [esi + edx*1]
db $C5, $FE, $7F, $04, $14 // vmovdqu yword [esp + edx*1], ymm0
add edx, $20
cmp edx, $400
jb @xor_loop
// Step 2: copy R_buf to Z_buf at [esp+1024]
xor edx, edx
@copy_loop:
db $C5, $FE, $6F, $04, $14 // vmovdqu ymm0, yword [esp + edx*1]
db $C5, $FE, $7F, $84, $14, $00, $04, $00, $00// vmovdqu yword [esp + edx*1 + $400], ymm0
add edx, $20
cmp edx, $400
jb @copy_loop
// Step 3: column rounds on Z_buf (8 iterations, 128 bytes each)
xor edx, edx
@col_loop:
db $C5, $FE, $6F, $84, $14, $00, $04, $00, $00// vmovdqu ymm0, yword [esp + edx*1 + $400]
db $C5, $FE, $6F, $8C, $14, $20, $04, $00, $00// vmovdqu ymm1, yword [esp + edx*1 + $420]
db $C5, $FE, $6F, $94, $14, $40, $04, $00, $00// vmovdqu ymm2, yword [esp + edx*1 + $440]
db $C5, $FE, $6F, $9C, $14, $60, $04, $00, $00// vmovdqu ymm3, yword [esp + edx*1 + $460]
// ----- Column G -----
// fBlaMka(0, 1): a = a + b + 2*lo32(a)*lo32(b)
db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1
db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1
db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4
// d = rotr64(d ^ a, 32)
db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1
// fBlaMka(2, 3): a = a + b + 2*lo32(a)*lo32(b)
db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1
db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3
db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4
// b = rotr64(b ^ c, 24)
db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
db $C4, $E2, $75, $00, $09 // vpshufb ymm1, ymm1, yword [ecx]
// fBlaMka(0, 1): a = a + b + 2*lo32(a)*lo32(b)
db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1
db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1
db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4
// d = rotr64(d ^ a, 16)
db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
db $C4, $E2, $65, $00, $59, $20 // vpshufb ymm3, ymm3, yword [ecx + $20]
// fBlaMka(2, 3): a = a + b + 2*lo32(a)*lo32(b)
db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1
db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3
db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4
// b = rotr64(b ^ c, 63)
db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
db $C5, $DD, $73, $D1, $3F // vpsrlq ymm4, ymm1, $3F
db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1
db $C5, $F5, $EB, $CC // vpor ymm1, ymm1, ymm4
// ----- Diagonalize -----
db $C4, $E3, $FD, $00, $C9, $39 // vpermq ymm1, ymm1, $39
db $C4, $E3, $FD, $00, $D2, $4E // vpermq ymm2, ymm2, $4E
db $C4, $E3, $FD, $00, $DB, $93 // vpermq ymm3, ymm3, $93
// ----- Diagonal G -----
// fBlaMka(0, 1): a = a + b + 2*lo32(a)*lo32(b)
db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1
db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1
db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4
// d = rotr64(d ^ a, 32)
db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1
// fBlaMka(2, 3): a = a + b + 2*lo32(a)*lo32(b)
db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1
db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3
db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4
// b = rotr64(b ^ c, 24)
db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
db $C4, $E2, $75, $00, $09 // vpshufb ymm1, ymm1, yword [ecx]
// fBlaMka(0, 1): a = a + b + 2*lo32(a)*lo32(b)
db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1
db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1
db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4
// d = rotr64(d ^ a, 16)
db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
db $C4, $E2, $65, $00, $59, $20 // vpshufb ymm3, ymm3, yword [ecx + $20]
// fBlaMka(2, 3): a = a + b + 2*lo32(a)*lo32(b)
db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1
db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3
db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4
// b = rotr64(b ^ c, 63)
db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
db $C5, $DD, $73, $D1, $3F // vpsrlq ymm4, ymm1, $3F
db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1
db $C5, $F5, $EB, $CC // vpor ymm1, ymm1, ymm4
// ----- Undiagonalize -----
db $C4, $E3, $FD, $00, $C9, $93 // vpermq ymm1, ymm1, $93
db $C4, $E3, $FD, $00, $D2, $4E // vpermq ymm2, ymm2, $4E
db $C4, $E3, $FD, $00, $DB, $39 // vpermq ymm3, ymm3, $39
db $C5, $FE, $7F, $84, $14, $00, $04, $00, $00// vmovdqu yword [esp + edx*1 + $400], ymm0
db $C5, $FE, $7F, $8C, $14, $20, $04, $00, $00// vmovdqu yword [esp + edx*1 + $420], ymm1
db $C5, $FE, $7F, $94, $14, $40, $04, $00, $00// vmovdqu yword [esp + edx*1 + $440], ymm2
db $C5, $FE, $7F, $9C, $14, $60, $04, $00, $00// vmovdqu yword [esp + edx*1 + $460], ymm3
add edx, $80
cmp edx, $400
jb @col_loop
// Step 4: row rounds on Z_buf (8 iterations, stride-16-QWord gathers)
xor edx, edx
@row_loop:
db $C5, $FA, $6F, $84, $14, $00, $04, $00, $00// vmovdqu xmm0, oword [esp + edx*1 + $400]
db $C4, $E3, $7D, $38, $84, $14, $80, $04, $00, $00, $01// vinserti128 ymm0, ymm0, oword [esp + edx*1 + $480], $1
db $C5, $FA, $6F, $8C, $14, $00, $05, $00, $00// vmovdqu xmm1, oword [esp + edx*1 + $500]
db $C4, $E3, $75, $38, $8C, $14, $80, $05, $00, $00, $01// vinserti128 ymm1, ymm1, oword [esp + edx*1 + $580], $1
db $C5, $FA, $6F, $94, $14, $00, $06, $00, $00// vmovdqu xmm2, oword [esp + edx*1 + $600]
db $C4, $E3, $6D, $38, $94, $14, $80, $06, $00, $00, $01// vinserti128 ymm2, ymm2, oword [esp + edx*1 + $680], $1
db $C5, $FA, $6F, $9C, $14, $00, $07, $00, $00// vmovdqu xmm3, oword [esp + edx*1 + $700]
db $C4, $E3, $65, $38, $9C, $14, $80, $07, $00, $00, $01// vinserti128 ymm3, ymm3, oword [esp + edx*1 + $780], $1
// ----- Column G -----
// fBlaMka(0, 1): a = a + b + 2*lo32(a)*lo32(b)
db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1
db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1
db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4
// d = rotr64(d ^ a, 32)
db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1
// fBlaMka(2, 3): a = a + b + 2*lo32(a)*lo32(b)
db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1
db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3
db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4
// b = rotr64(b ^ c, 24)
db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
db $C4, $E2, $75, $00, $09 // vpshufb ymm1, ymm1, yword [ecx]
// fBlaMka(0, 1): a = a + b + 2*lo32(a)*lo32(b)
db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1
db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1
db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4
// d = rotr64(d ^ a, 16)
db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
db $C4, $E2, $65, $00, $59, $20 // vpshufb ymm3, ymm3, yword [ecx + $20]
// fBlaMka(2, 3): a = a + b + 2*lo32(a)*lo32(b)
db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1
db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3
db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4
// b = rotr64(b ^ c, 63)
db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
db $C5, $DD, $73, $D1, $3F // vpsrlq ymm4, ymm1, $3F
db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1
db $C5, $F5, $EB, $CC // vpor ymm1, ymm1, ymm4
// ----- Diagonalize -----
db $C4, $E3, $FD, $00, $C9, $39 // vpermq ymm1, ymm1, $39
db $C4, $E3, $FD, $00, $D2, $4E // vpermq ymm2, ymm2, $4E
db $C4, $E3, $FD, $00, $DB, $93 // vpermq ymm3, ymm3, $93
// ----- Diagonal G -----
// fBlaMka(0, 1): a = a + b + 2*lo32(a)*lo32(b)
db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1
db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1
db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4
// d = rotr64(d ^ a, 32)
db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1
// fBlaMka(2, 3): a = a + b + 2*lo32(a)*lo32(b)
db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1
db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3
db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4
// b = rotr64(b ^ c, 24)
db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
db $C4, $E2, $75, $00, $09 // vpshufb ymm1, ymm1, yword [ecx]
// fBlaMka(0, 1): a = a + b + 2*lo32(a)*lo32(b)
db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1
db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1
db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4
// d = rotr64(d ^ a, 16)
db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
db $C4, $E2, $65, $00, $59, $20 // vpshufb ymm3, ymm3, yword [ecx + $20]
// fBlaMka(2, 3): a = a + b + 2*lo32(a)*lo32(b)
db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3
db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1
db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3
db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4
// b = rotr64(b ^ c, 63)
db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
db $C5, $DD, $73, $D1, $3F // vpsrlq ymm4, ymm1, $3F
db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1
db $C5, $F5, $EB, $CC // vpor ymm1, ymm1, ymm4
// ----- Undiagonalize -----
db $C4, $E3, $FD, $00, $C9, $93 // vpermq ymm1, ymm1, $93
db $C4, $E3, $FD, $00, $D2, $4E // vpermq ymm2, ymm2, $4E
db $C4, $E3, $FD, $00, $DB, $39 // vpermq ymm3, ymm3, $39
db $C5, $FA, $7F, $84, $14, $00, $04, $00, $00// vmovdqu oword [esp + edx*1 + $400], xmm0
db $C4, $E3, $7D, $39, $84, $14, $80, $04, $00, $00, $01// vextracti128 oword [esp + edx*1 + $480], ymm0, $1
db $C5, $FA, $7F, $8C, $14, $00, $05, $00, $00// vmovdqu oword [esp + edx*1 + $500], xmm1
db $C4, $E3, $7D, $39, $8C, $14, $80, $05, $00, $00, $01// vextracti128 oword [esp + edx*1 + $580], ymm1, $1
db $C5, $FA, $7F, $94, $14, $00, $06, $00, $00// vmovdqu oword [esp + edx*1 + $600], xmm2
db $C4, $E3, $7D, $39, $94, $14, $80, $06, $00, $00, $01// vextracti128 oword [esp + edx*1 + $680], ymm2, $1
db $C5, $FA, $7F, $9C, $14, $00, $07, $00, $00// vmovdqu oword [esp + edx*1 + $700], xmm3
db $C4, $E3, $7D, $39, $9C, $14, $80, $07, $00, $00, $01// vextracti128 oword [esp + edx*1 + $780], ymm3, $1
add edx, $10
cmp edx, $80
jb @row_loop
// Step 5: Current = R_buf XOR Z_buf [XOR Current]
test eax, eax
jnz @final_with_xor
xor edx, edx
@final_noxor_loop:
db $C5, $FE, $6F, $04, $14 // vmovdqu ymm0, yword [esp + edx*1]
db $C5, $FD, $EF, $84, $14, $00, $04, $00, $00// vpxor ymm0, ymm0, yword [esp + edx*1 + $400]
db $C5, $FE, $7F, $04, $17 // vmovdqu yword [edi + edx*1], ymm0
add edx, $20
cmp edx, $400
jb @final_noxor_loop
jmp @epilogue
@final_with_xor:
xor edx, edx
@final_xor_loop:
db $C5, $FE, $6F, $04, $14 // vmovdqu ymm0, yword [esp + edx*1]
db $C5, $FD, $EF, $84, $14, $00, $04, $00, $00// vpxor ymm0, ymm0, yword [esp + edx*1 + $400]
db $C5, $FD, $EF, $04, $17 // vpxor ymm0, ymm0, yword [edi + edx*1]
db $C5, $FE, $7F, $04, $17 // vmovdqu yword [edi + edx*1], ymm0
add edx, $20
cmp edx, $400
jb @final_xor_loop
@epilogue:
add esp, $800
db $C5, $F8, $77 // vzeroupper
pop edi
pop esi
pop ebx