ML-KEM assembly: AVX512F and AVX512VBMI.

New assembly for ML-KEM for Intel x64 machines that support AVX512 extensions.
Update how much is added to the stack of others.

Author: SparkiDev

wolfcrypt/src/aes_gcm_asm.S

@@ -10190,7 +10190,7 @@ _AES_GCM_init_avx1:
         movq	%rdx, %r10
         movl	%ecx, %r11d
         movq	24(%rsp), %rax
-        subq	$16, %rsp
+        subq	$24, %rsp
         vpxor	%xmm4, %xmm4, %xmm4
         movl	%r11d, %edx
         cmpl	$12, %edx
@@ -10483,7 +10483,7 @@ L_AES_GCM_init_avx1_iv_done:
         vpaddd	L_avx1_aes_gcm_one(%rip), %xmm4, %xmm4
         vmovdqa	%xmm5, (%r8)
         vmovdqa	%xmm4, (%r9)
-        addq	$16, %rsp
+        addq	$24, %rsp
         popq	%r13
         popq	%r12
         repz retq
@@ -11811,7 +11811,7 @@ _AES_GCM_decrypt_update_avx1:
         movq	%rcx, %r11
         movq	32(%rsp), %rax
         movq	40(%rsp), %r12
-        subq	$0xa8, %rsp
+        subq	$0xb0, %rsp
         vmovdqa	(%r9), %xmm6
         vmovdqa	(%rax), %xmm5
         vpsrlq	$63, %xmm5, %xmm9
@@ -12393,7 +12393,7 @@ L_AES_GCM_decrypt_update_avx1_last_block_done:
 L_AES_GCM_decrypt_update_avx1_done_dec:
         vmovdqa	%xmm6, (%r9)
         vzeroupper
-        addq	$0xa8, %rsp
+        addq	$0xb0, %rsp
         popq	%r14
         popq	%r12
         popq	%r13
@@ -13379,7 +13379,7 @@ _AES_GCM_encrypt_avx2:
         movl	64(%rsp), %r14d
         movq	72(%rsp), %rsi
         movl	80(%rsp), %r9d
-        subq	$0xb8, %rsp
+        subq	$0xc0, %rsp
         vpxor	%xmm4, %xmm4, %xmm4
         vpxor	%xmm6, %xmm6, %xmm6
         movl	%ebx, %edx
@@ -14825,7 +14825,7 @@ L_AES_GCM_encrypt_avx2_store_tag_16:
         vmovdqu	%xmm0, (%r15)
 L_AES_GCM_encrypt_avx2_store_tag_done:
         vzeroupper
-        addq	$0xb8, %rsp
+        addq	$0xc0, %rsp
         popq	%r14
         popq	%rbx
         popq	%r15
@@ -15999,7 +15999,7 @@ _AES_GCM_init_avx2:
         movq	%rdx, %r10
         movl	%ecx, %r11d
         movq	24(%rsp), %rax
-        subq	$16, %rsp
+        subq	$24, %rsp
         vpxor	%xmm4, %xmm4, %xmm4
         movl	%r11d, %edx
         cmpl	$12, %edx
@@ -16248,7 +16248,7 @@ L_AES_GCM_init_avx2_iv_done:
         vmovdqu	%xmm5, (%r8)
         vmovdqu	%xmm4, (%r9)
         vzeroupper
-        addq	$16, %rsp
+        addq	$24, %rsp
         popq	%r12
         popq	%rbx
         repz retq
@@ -16451,7 +16451,7 @@ _AES_GCM_encrypt_update_avx2:
         movq	%rcx, %r11
         movq	32(%rsp), %rax
         movq	40(%rsp), %r12
-        subq	$0x98, %rsp
+        subq	$0xa0, %rsp
         vmovdqu	(%r9), %xmm6
         vmovdqu	(%rax), %xmm5
         vmovdqu	(%r12), %xmm4
@@ -17282,7 +17282,7 @@ L_AES_GCM_encrypt_update_avx2_done_enc:
         vmovdqu	%xmm6, (%r9)
         vmovdqu	%xmm4, (%r12)
         vzeroupper
-        addq	$0x98, %rsp
+        addq	$0xa0, %rsp
         popq	%r14
         popq	%r13
         popq	%r12
@@ -17303,7 +17303,7 @@ AES_GCM_encrypt_final_avx2:
 _AES_GCM_encrypt_final_avx2:
 #endif /* __APPLE__ */
         movq	8(%rsp), %rax
-        subq	$16, %rsp
+        subq	$24, %rsp
         vmovdqu	(%rdi), %xmm4
         vmovdqu	(%r9), %xmm5
         vmovdqu	(%rax), %xmm6
@@ -17357,7 +17357,7 @@ L_AES_GCM_encrypt_final_avx2_store_tag_16:
         vmovdqu	%xmm0, (%rsi)
 L_AES_GCM_encrypt_final_avx2_store_tag_done:
         vzeroupper
-        addq	$16, %rsp
+        addq	$24, %rsp
         repz retq
 #ifndef __APPLE__
 .size	AES_GCM_encrypt_final_avx2,.-AES_GCM_encrypt_final_avx2
@@ -17381,7 +17381,7 @@ _AES_GCM_decrypt_update_avx2:
         movq	%rcx, %r11
         movq	32(%rsp), %rax
         movq	40(%rsp), %r12
-        subq	$0xa8, %rsp
+        subq	$0xb0, %rsp
         vmovdqu	(%r9), %xmm6
         vmovdqu	(%rax), %xmm5
         vmovdqu	(%r12), %xmm4
@@ -17865,7 +17865,7 @@ L_AES_GCM_decrypt_update_avx2_done_dec:
         vmovdqu	%xmm6, (%r9)
         vmovdqu	%xmm4, (%r12)
         vzeroupper
-        addq	$0xa8, %rsp
+        addq	$0xb0, %rsp
         popq	%r14
         popq	%r12
         popq	%r13
@@ -20407,7 +20407,7 @@ _AES_GCM_decrypt_vaes:
         movq	80(%rsp), %r15
         movl	88(%rsp), %r10d
         movq	96(%rsp), %rbp
-        subq	$0x250, %rsp
+        subq	$0x258, %rsp
         vpxor	%xmm5, %xmm5, %xmm5
         vpxor	%xmm15, %xmm15, %xmm15
         cmpl	$12, %ebx
@@ -22328,7 +22328,7 @@ L_AES_GCM_decrypt_vaes_cmp_tag_16:
 L_AES_GCM_decrypt_vaes_cmp_tag_done:
         movl	%ebx, (%rbp)
         vzeroupper
-        addq	$0x250, %rsp
+        addq	$0x258, %rsp
         popq	%rbp
         popq	%r15
         popq	%r14
@@ -22357,7 +22357,7 @@ _AES_GCM_init_vaes:
         movq	%rdx, %r10
         movl	%ecx, %r11d
         movq	24(%rsp), %rax
-        subq	$16, %rsp
+        subq	$24, %rsp
         vpxor	%xmm4, %xmm4, %xmm4
         movl	%r11d, %edx
         cmpl	$12, %edx
@@ -22650,7 +22650,7 @@ L_AES_GCM_init_vaes_iv_done:
         vpaddd	L_vaes_aes_gcm_one(%rip), %xmm4, %xmm4
         vmovdqa	%xmm5, (%r8)
         vmovdqa	%xmm4, (%r9)
-        addq	$16, %rsp
+        addq	$24, %rsp
         popq	%r13
         popq	%r12
         repz retq
@@ -22671,7 +22671,7 @@ _AES_GCM_aad_update_vaes:
 #endif /* __APPLE__ */
         movq	%rdx, %r8
         movq	%rcx, %r9
-        subq	$0x200, %rsp
+        subq	$0x208, %rsp
         vmovdqa	(%r8), %xmm15
         vmovdqa	(%r9), %xmm6
         xorl	%edx, %edx
@@ -23185,7 +23185,7 @@ L_AES_GCM_aad_update_vaes_tail:
 L_AES_GCM_aad_update_vaes_done:
         vmovdqa	%xmm15, (%r8)
         vzeroupper
-        addq	$0x200, %rsp
+        addq	$0x208, %rsp
         repz retq
 #ifndef __APPLE__
 .size	AES_GCM_aad_update_vaes,.-AES_GCM_aad_update_vaes
@@ -29305,7 +29305,7 @@ _AES_GCM_decrypt_avx512:
         movq	80(%rsp), %r15
         movl	88(%rsp), %r10d
         movq	96(%rsp), %rbp
-        subq	$0x460, %rsp
+        subq	$0x468, %rsp
         vpxor	%xmm4, %xmm4, %xmm4
         vpxor	%xmm6, %xmm6, %xmm6
         cmpl	$12, %ebx
@@ -31770,7 +31770,7 @@ L_AES_GCM_decrypt_avx512_cmp_tag_16:
 L_AES_GCM_decrypt_avx512_cmp_tag_done:
         movl	%ebx, (%rbp)
         vzeroupper
-        addq	$0x460, %rsp
+        addq	$0x468, %rsp
         popq	%rbp
         popq	%r15
         popq	%r14
@@ -31799,7 +31799,7 @@ _AES_GCM_init_avx512:
         movq	%rdx, %r10
         movl	%ecx, %r11d
         movq	24(%rsp), %rax
-        subq	$16, %rsp
+        subq	$24, %rsp
         vpxor	%xmm4, %xmm4, %xmm4
         movl	%r11d, %edx
         cmpl	$12, %edx
@@ -32092,7 +32092,7 @@ L_AES_GCM_init_avx512_iv_done:
         vpaddd	L_avx512_aes_gcm_one(%rip), %xmm4, %xmm4
         vmovdqa	%xmm5, (%r8)
         vmovdqa	%xmm4, (%r9)
-        addq	$16, %rsp
+        addq	$24, %rsp
         popq	%r13
         popq	%r12
         repz retq
@@ -32113,7 +32113,7 @@ _AES_GCM_aad_update_avx512:
 #endif /* __APPLE__ */
         movq	%rdx, %r8
         movq	%rcx, %r9
-        subq	$0x100, %rsp
+        subq	$0x108, %rsp
         vmovdqa	(%r8), %xmm6
         vmovdqa	(%r9), %xmm7
         xorl	%edx, %edx
@@ -32526,7 +32526,7 @@ L_AES_GCM_aad_update_avx512_tail:
 L_AES_GCM_aad_update_avx512_done:
         vmovdqa	%xmm6, (%r8)
         vzeroupper
-        addq	$0x100, %rsp
+        addq	$0x108, %rsp
         repz retq
 #ifndef __APPLE__
 .size	AES_GCM_aad_update_avx512,.-AES_GCM_aad_update_avx512
@@ -32669,7 +32669,7 @@ _AES_GCM_encrypt_update_avx512:
         movq	%rcx, %r11
         movq	56(%rsp), %rax
         movq	64(%rsp), %r12
-        subq	$0x410, %rsp
+        subq	$0x418, %rsp
         vmovdqa	(%r9), %xmm6
         vmovdqa	(%rax), %xmm5
         vpsrlq	$63, %xmm5, %xmm9
@@ -34389,7 +34389,7 @@ L_AES_GCM_encrypt_update_avx512_last_block_done:
 L_AES_GCM_encrypt_update_avx512_done_enc:
         vmovdqa	%xmm6, (%r9)
         vzeroupper
-        addq	$0x410, %rsp
+        addq	$0x418, %rsp
         popq	%rbp
         popq	%rbx
         popq	%r15