Use optimal microkernel on Cortex X3

PiperOrigin-RevId: 729004520

Author: alankelly

cmake/gen/aarch64_microkernels.cmake

@@ -28,7 +28,6 @@ SET(PROD_AARCH64_ASM_MICROKERNEL_SRCS
   src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-cortex-a75-prfm.S
   src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-cortex-a75.S
   src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-ld128-2.S
-  src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-ld128-acc2.S
   src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-ld128-acc4.S
   src/f32-gemm/gen/f32-gemm-4x2-minmax-asm-aarch64-neonfma-ld128.S
   src/f32-gemm/gen/f32-gemm-4x8-minmax-asm-aarch64-neonfma-cortex-a53-prfm.S
@@ -44,6 +43,7 @@ SET(PROD_AARCH64_ASM_MICROKERNEL_SRCS
   src/f32-gemm/gen/f32-gemm-6x8-minmax-asm-aarch64-neonfma-cortex-a75-prfm.S
   src/f32-gemm/gen/f32-gemm-6x8-minmax-asm-aarch64-neonfma-ld128.S
   src/f32-gemm/gen/f32-gemm-7x8-minmax-asm-aarch64-neonfma-ld128-2.S
+  src/f32-gemm/gen/f32-gemm-8x8-minmax-asm-aarch64-neonfma-ld128-2.S
   src/f32-igemm/f32-igemm-4x8-minmax-asm-aarch64-neonfma-cortex-a55.S
   src/f32-igemm/f32-igemm-6x8-minmax-asm-aarch64-neonfma-cortex-a55.S
   src/f32-igemm/f32-igemm-6x8-minmax-asm-aarch64-neonfma-cortex-a73.S
@@ -134,6 +134,7 @@ SET(NON_PROD_AARCH64_ASM_MICROKERNEL_SRCS
   src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-ld64-prfm.S
   src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-ld64.S
   src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-ld128-acc2-prfm.S
+  src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-ld128-acc2.S
   src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-ld128-acc4-prfm.S
   src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-ld128-prfm.S
   src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-ld128.S
@@ -183,7 +184,6 @@ SET(NON_PROD_AARCH64_ASM_MICROKERNEL_SRCS
   src/f32-gemm/gen/f32-gemm-7x8-minmax-asm-aarch64-neonfma-ld64-2.S
   src/f32-gemm/gen/f32-gemm-8x8-minmax-asm-aarch64-neonfma-ld32-2.S
   src/f32-gemm/gen/f32-gemm-8x8-minmax-asm-aarch64-neonfma-ld64-2.S
-  src/f32-gemm/gen/f32-gemm-8x8-minmax-asm-aarch64-neonfma-ld128-2.S
   src/f32-gemm/gen/f32-gemm-goi-1x8-minmax-asm-aarch64-neonfma-ld128-prfm.S
   src/f32-gemm/gen/f32-gemm-goi-1x8-minmax-asm-aarch64-neonfma-ld128.S
   src/f32-gemm/gen/f32-gemm-goi-4x8-minmax-asm-aarch64-neonfma-ld128.S

cmake/gen/rvv_microkernels.cmake

@@ -54,10 +54,10 @@ SET(PROD_RVV_MICROKERNEL_SRCS
   src/f32-vrsqrt/gen/f32-vrsqrt-rvv-rsqrt-u4v.c
   src/qs8-dwconv/gen/qs8-dwconv-9p8vc-minmax-fp32-rvv.c
   src/qs8-dwconv/gen/qs8-dwconv-25p8vc-minmax-fp32-rvv.c
+  src/qs8-f32-vcvt/gen/qs8-f32-vcvt-rvv-u2v.c
   src/qs8-qc8w-dwconv/gen/qs8-qc8w-dwconv-3p8vc-minmax-fp32-rvv.c
   src/qs8-qc8w-dwconv/gen/qs8-qc8w-dwconv-9p8vc-minmax-fp32-rvv.c
   src/qs8-qc8w-dwconv/gen/qs8-qc8w-dwconv-25p8vc-minmax-fp32-rvv.c
-  src/qs8-f32-vcvt/gen/qs8-f32-vcvt-rvv-u2v.c
   src/qs8-vlrelu/gen/qs8-vlrelu-rvv-u2v.c
   src/qs8-vmul/gen/qs8-vmul-minmax-f32-rvv-u2v.c
   src/qs8-vmulc/gen/qs8-vmulc-minmax-f32-rvv-u2v.c

gen/aarch64_microkernels.bzl

@@ -24,7 +24,6 @@ PROD_AARCH64_ASM_MICROKERNEL_SRCS = [
     "src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-cortex-a75-prfm.S",
     "src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-cortex-a75.S",
     "src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-ld128-2.S",
-    "src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-ld128-acc2.S",
     "src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-ld128-acc4.S",
     "src/f32-gemm/gen/f32-gemm-4x2-minmax-asm-aarch64-neonfma-ld128.S",
     "src/f32-gemm/gen/f32-gemm-4x8-minmax-asm-aarch64-neonfma-cortex-a53-prfm.S",
@@ -40,6 +39,7 @@ PROD_AARCH64_ASM_MICROKERNEL_SRCS = [
     "src/f32-gemm/gen/f32-gemm-6x8-minmax-asm-aarch64-neonfma-cortex-a75-prfm.S",
     "src/f32-gemm/gen/f32-gemm-6x8-minmax-asm-aarch64-neonfma-ld128.S",
     "src/f32-gemm/gen/f32-gemm-7x8-minmax-asm-aarch64-neonfma-ld128-2.S",
+    "src/f32-gemm/gen/f32-gemm-8x8-minmax-asm-aarch64-neonfma-ld128-2.S",
     "src/f32-igemm/f32-igemm-4x8-minmax-asm-aarch64-neonfma-cortex-a55.S",
     "src/f32-igemm/f32-igemm-6x8-minmax-asm-aarch64-neonfma-cortex-a55.S",
     "src/f32-igemm/f32-igemm-6x8-minmax-asm-aarch64-neonfma-cortex-a73.S",
@@ -131,6 +131,7 @@ NON_PROD_AARCH64_ASM_MICROKERNEL_SRCS = [
     "src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-ld64-prfm.S",
     "src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-ld64.S",
     "src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-ld128-acc2-prfm.S",
+    "src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-ld128-acc2.S",
     "src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-ld128-acc4-prfm.S",
     "src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-ld128-prfm.S",
     "src/f32-gemm/gen/f32-gemm-1x8-minmax-asm-aarch64-neonfma-ld128.S",
@@ -180,7 +181,6 @@ NON_PROD_AARCH64_ASM_MICROKERNEL_SRCS = [
     "src/f32-gemm/gen/f32-gemm-7x8-minmax-asm-aarch64-neonfma-ld64-2.S",
     "src/f32-gemm/gen/f32-gemm-8x8-minmax-asm-aarch64-neonfma-ld32-2.S",
     "src/f32-gemm/gen/f32-gemm-8x8-minmax-asm-aarch64-neonfma-ld64-2.S",
-    "src/f32-gemm/gen/f32-gemm-8x8-minmax-asm-aarch64-neonfma-ld128-2.S",
     "src/f32-gemm/gen/f32-gemm-goi-1x8-minmax-asm-aarch64-neonfma-ld128-prfm.S",
     "src/f32-gemm/gen/f32-gemm-goi-1x8-minmax-asm-aarch64-neonfma-ld128.S",
     "src/f32-gemm/gen/f32-gemm-goi-4x8-minmax-asm-aarch64-neonfma-ld128.S",

gen/rvv_microkernels.bzl

@@ -48,10 +48,17 @@ PROD_RVV_MICROKERNEL_SRCS = [
     "src/f32-vrnd/gen/f32-vrndu-rvv-u4v.c",
     "src/f32-vrnd/gen/f32-vrndz-rvv-u4v.c",
     "src/f32-vrsqrt/gen/f32-vrsqrt-rvv-rsqrt-u4v.c",
+    "src/qs8-dwconv/gen/qs8-dwconv-9p8vc-minmax-fp32-rvv.c",
+    "src/qs8-dwconv/gen/qs8-dwconv-25p8vc-minmax-fp32-rvv.c",
     "src/qs8-f32-vcvt/gen/qs8-f32-vcvt-rvv-u2v.c",
+    "src/qs8-qc8w-dwconv/gen/qs8-qc8w-dwconv-3p8vc-minmax-fp32-rvv.c",
+    "src/qs8-qc8w-dwconv/gen/qs8-qc8w-dwconv-9p8vc-minmax-fp32-rvv.c",
+    "src/qs8-qc8w-dwconv/gen/qs8-qc8w-dwconv-25p8vc-minmax-fp32-rvv.c",
     "src/qs8-vlrelu/gen/qs8-vlrelu-rvv-u2v.c",
     "src/qs8-vmul/gen/qs8-vmul-minmax-f32-rvv-u2v.c",
     "src/qs8-vmulc/gen/qs8-vmulc-minmax-f32-rvv-u2v.c",
+    "src/qu8-dwconv/gen/qu8-dwconv-9p8vc-minmax-fp32-rvv.c",
+    "src/qu8-dwconv/gen/qu8-dwconv-25p8vc-minmax-fp32-rvv.c",
     "src/qu8-f32-vcvt/gen/qu8-f32-vcvt-rvv-u2v.c",
     "src/qu8-vlrelu/gen/qu8-vlrelu-rvv-u2v.c",
     "src/qu8-vmul/gen/qu8-vmul-minmax-f32-rvv-u2v.c",

src/configs/gemm-config.c

@@ -591,12 +591,12 @@ static void init_f32_gemm_config(void) {
           case cpuinfo_uarch_cortex_x3:
           case cpuinfo_uarch_neoverse_v2:
             // TODO(fbarchard): Implement asm with indexed inputs
-            f32_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(1)] = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_fn) xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch64_neonfma_ld128_acc2);
-            f32_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(6)] = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_fn) xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_lane_ld128);
+            f32_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(1)] = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_fn) xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch64_neonfma_ld128_2);
+            f32_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(8)] = xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_fn) xnn_f32_gemm_minmax_ukernel_8x8__asm_aarch64_neonfma_ld128_2);
             f32_gemm_config.init.f32 = xnn_init_f32_minmax_scalar_params;
             f32_gemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;
             f32_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;
-            f32_gemm_config.mr = 6;
+            f32_gemm_config.mr = 8;
             f32_gemm_config.nr = 8;
             break;
           case cpuinfo_uarch_cortex_a78: