[CK] [FlyDSL] Retune Kimi-K3 a16w4 MoE tile geometry

aiter/configs/model_configs/kimik3_a16w4_tuned_fmoe.csv

@@ -0,0 +1,33 @@
+gfx,cu_num,token,model_dim,inter_dim,expert,topk,act_type,dtype,q_dtype_a,q_dtype_w,q_type,use_g1u1,doweight_stage1,block_m,ksplit,us1,kernelName1,err1,us2,kernelName2,err2,us,run_1stage,xbf16,flat,tflops,bw,_tag
+gfx950,256,1,3584,512,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,15.2128,flydsl_moe1_abf16_wfp4_bf16_t32x32x256_xcd1_kw2,0.0%,11.9685,flydsl_moe2_abf16_wfp4_bf16_t32x128x128_atomic_bnt2,0.4%,27.1813,0,0,0,6.48,181467.25,
+gfx950,256,2,3584,512,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,21.4146,flydsl_moe1_abf16_wfp4_bf16_t32x64x256_xcd1_kw2,0.0%,14.2472,flydsl_moe2_abf16_wfp4_bf16_t32x128x128_atomic_bnt2_xcd4,0.4%,35.6618,0,0,0,9.88,138314.11,
+gfx950,256,3,3584,512,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,29.1549,flydsl_moe1_abf16_wfp4_bf16_t32x64x256_kw2,0.0%,17.4007,flydsl_moe2_abf16_wfp4_bf16_t32x128x128_atomic,0.6%,46.5556,0,0,0,11.35,105949.54,
+gfx950,256,4,3584,512,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,30.7522,flydsl_moe1_abf16_wfp4_bf16_t32x64x256_xcd4_kw2,0.0%,21.4556,flydsl_moe2_abf16_wfp4_bf16_t32x128x128_atomic,0.6%,52.2078,0,0,0,13.5,94479.35,
+gfx950,256,8,3584,512,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,43.412,flydsl_moe1_abf16_wfp4_bf16_t32x128x256_xcd1,0.0%,31.4221,flydsl_moe2_abf16_wfp4_bf16_t32x128x128_atomic,0.5%,74.8341,0,0,0,18.83,65914.02,
+gfx950,256,16,3584,512,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,84.2642,flydsl_moe1_abf16_wfp4_bf16_t32x128x256_xcd4,0.0%,49.2721,flydsl_moe2_abf16_wfp4_bf16_t32x128x128_atomic_bnt2,0.4%,133.5363,0,0,0,21.11,36939.25,
+gfx950,256,32,3584,512,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,146.0252,flydsl_moe1_abf16_wfp4_bf16_t32x128x256_xcd1,0.0%,78.5357,flydsl_moe2_abf16_wfp4_bf16_t32x128x128_atomic_bnt2,0.5%,224.5609,0,0,0,25.1,21967.14,
+gfx950,256,64,3584,512,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,207.6637,flydsl_moe1_abf16_wfp4_bf16_t32x128x256_xcd1,0.0%,119.4062,flydsl_moe2_abf16_wfp4_bf16_t32x128x128_atomic_bnt2_xcd4,0.5%,327.0699,0,0,0,34.47,15083.68,
+gfx950,256,128,3584,512,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,269.4591,flydsl_moe1_abf16_wfp4_bf16_t32x128x128_xcd4,0.0%,158.7188,flydsl_moe2_abf16_wfp4_bf16_t32x128x128_atomic,0.5%,428.1779,0,0,0,52.66,11524.03,
+gfx950,256,256,3584,512,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,294.0032,flydsl_moe1_abf16_wfp4_bf16_t32x128x128_xcd1,0.0%,176.7689,flydsl_moe2_abf16_wfp4_bf16_t32x128x128_atomic_bnt2_xcd4,0.5%,470.7721,0,0,0,95.79,10485.27,
+gfx950,256,512,3584,512,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,303.9467,flydsl_moe1_abf16_wfp4_bf16_t32x128x128_xcd4,0.0%,187.0567,flydsl_moe2_abf16_wfp4_bf16_t32x128x128_atomic_bnt2,0.5%,491.0034,0,0,0,183.69,10060.71,
+gfx950,256,1024,3584,512,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,320.947,flydsl_moe1_abf16_wfp4_bf16_t32x128x128_xcd4,0.0%,200.6498,flydsl_moe2_abf16_wfp4_bf16_t32x128x128_atomic_bnt2,0.5%,521.5968,0,0,0,345.84,9484.69,
+gfx950,256,2048,3584,512,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,552.6688,flydsl_moe1_abf16_wfp4_bf16_t32x256x128_bnt0_xcd1,0.0%,309.4969,flydsl_moe2_abf16_wfp4_bf16_t32x256x128_atomic,0.5%,862.1657,0,0,0,418.45,5755.11,
+gfx950,256,4096,3584,512,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,823.6266,flydsl_moe1_abf16_wfp4_bf16_t32x256x128_bnt0_xcd1,0.0%,499.2724,flydsl_moe2_abf16_wfp4_bf16_t32x256x128_atomic,0.5%,1322.8991,0,0,0,545.43,3772.94,
+gfx950,256,8192,3584,512,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,1353.7099,flydsl_moe1_abf16_wfp4_bf16_t32x256x128_bnt0_xcd1,0.0%,881.6882,flydsl_moe2_abf16_wfp4_bf16_t32x256x128_atomic,0.5%,2235.398,0,0,0,645.57,2259.08,
+gfx950,256,16384,3584,512,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,2429.886,flydsl_moe1_abf16_wfp4_bf16_t32x256x128_bnt0_xcd1,0.0%,1657.9878,flydsl_moe2_abf16_wfp4_bf16_t32x256x128_atomic,0.5%,4087.8738,0,0,0,706.04,1264.08,
+gfx950,256,1,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,16,0,10.55,flydsl_moe1_abf16_wfp4_bf16_t16x32x256_xcd4_kw2,0.0%,7.02,flydsl_moe2_abf16_wfp4_bf16_t16x128x128_atomic_bnt2_xcd4,0.2%,17.57,0,0,0,7.53,210551.54,
+gfx950,256,2,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,16,0,13.02,flydsl_moe1_abf16_wfp4_bf16_t16x32x128_kw4,0.0%,7.2,flydsl_moe2_abf16_wfp4_bf16_t16x128x128_atomic_bnt2,0.2%,20.22,0,0,0,13.07,182957.7,
+gfx950,256,3,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,16,0,19.24,flydsl_moe1_abf16_wfp4_bf16_t16x64x256_kw2,0.0%,8.93,flydsl_moe2_abf16_wfp4_bf16_t16x128x128_atomic_bnt2,0.3%,28.17,0,0,0,14.06,131324.78,
+gfx950,256,4,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,16,0,21.3,flydsl_moe1_abf16_wfp4_bf16_t16x64x256_xcd4_kw2,0.0%,13.26,flydsl_moe2_abf16_wfp4_bf16_t16x128x128_atomic_xcd4,0.2%,34.56,0,0,0,15.29,107043.8,
+gfx950,256,8,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,16,0,32.46,flydsl_moe1_abf16_wfp4_bf16_t16x128x256_xcd1,0.0%,18.53,flydsl_moe2_abf16_wfp4_bf16_t16x128x128_atomic_bnt2,0.2%,50.99,0,0,0,20.73,72553.27,
+gfx950,256,16,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,16,0,56.04,flydsl_moe1_abf16_wfp4_bf16_t16x128x128_xcd4,0.0%,30.62,flydsl_moe2_abf16_wfp4_bf16_t16x128x128_atomic_bnt2,0.3%,86.66,0,0,0,24.4,42691.04,
+gfx950,256,32,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,16,0,98.86,flydsl_moe1_abf16_wfp4_bf16_t16x128x128_xcd4,0.0%,52.39,flydsl_moe2_abf16_wfp4_bf16_t16x128x128_atomic_bnt2,0.3%,151.25,0,0,0,27.95,24461.72,
+gfx950,256,64,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,16,0,147.43,flydsl_moe1_abf16_wfp4_bf16_t16x128x256_xcd4,0.0%,83.72,flydsl_moe2_abf16_wfp4_bf16_t16x128x128_atomic_bnt2,0.3%,231.15,0,0,0,36.58,16008.19,
+gfx950,256,128,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,16,0,193.86,flydsl_moe1_abf16_wfp4_bf16_t16x128x256_xcd4,0.0%,113.87,flydsl_moe2_abf16_wfp4_bf16_t16x128x128_atomic_bnt2,0.3%,307.73,0,0,0,54.95,12027.46,
+gfx950,256,256,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,16,0,218.5,flydsl_moe1_abf16_wfp4_bf16_t16x128x128_xcd4,0.0%,127.2,flydsl_moe2_abf16_wfp4_bf16_t16x128x128_atomic_bnt2_xcd4,0.3%,345.7,0,0,0,97.84,10711.73,
+gfx950,256,512,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,16,0,234.53,flydsl_moe1_abf16_wfp4_bf16_t16x128x128_xcd4,0.0%,139.52,flydsl_moe2_abf16_wfp4_bf16_t16x128x128_atomic_bnt2,0.3%,374.05,0,0,0,180.85,9909.68,
+gfx950,256,1024,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,243.1013,flydsl_moe1_abf16_wfp4_bf16_t32x128x128_xcd4,0.0%,164.3789,flydsl_moe2_abf16_wfp4_bf16_t32x128x128_atomic_bnt2,0.3%,407.4802,0,0,0,332.02,9114.69,
+gfx950,256,2048,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,429.8417,flydsl_moe1_abf16_wfp4_bf16_t32x192x128_bnt0_xcd1,0.0%,255.091,flydsl_moe2_abf16_wfp4_bf16_t32x256x128_atomic,0.3%,684.9327,0,0,0,395.05,5443.95,
+gfx950,256,4096,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,638.614,flydsl_moe1_abf16_wfp4_bf16_t32x192x128_bnt0_xcd1,0.0%,407.5472,flydsl_moe2_abf16_wfp4_bf16_t32x256x128_atomic,0.3%,1046.1613,0,0,0,517.29,3592.27,
+gfx950,256,8192,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,1038.8538,flydsl_moe1_abf16_wfp4_bf16_t32x192x128_bnt0_xcd1,0.0%,789.3389,flydsl_moe2_abf16_wfp4_bf16_t32x256x128_atomic,0.3%,1828.1927,0,0,0,592.02,2087.75,
+gfx950,256,16384,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.bfloat16,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,1882.9619,flydsl_moe1_abf16_wfp4_bf16_t32x192x128_bnt0_xcd1,0.0%,1504.1563,flydsl_moe2_abf16_wfp4_bf16_t32x256x128_atomic,0.3%,3387.1182,0,0,0,639.09,1161.54,

aiter/fused_moe.py

@@ -1479,7 +1479,8 @@ def _flydsl_stage1_wrapper(
         sorted_weights=sorted_weights,
         use_async_copy=True,
         k_batch=parsed.get("k_batch", 1),
-        waves_per_eu=parsed.get("waves_per_eu", 3),
+        # None, matching stage 2: the int4 registry emits no waves_per_eu key.
+        waves_per_eu=parsed.get("waves_per_eu", None),
         b_nt=parsed.get("b_nt", 2),
         gate_mode=parsed.get("gate_mode", "separated"),
         inter_dim_pad=inter_dim_pad,

aiter/ops/flydsl/kernels/moe_2stage_a16wmix/gemm1.py

@@ -441,6 +441,11 @@ def compile_gemm1_a16w4_port(
     assert (
         TILE_N // (4 // k_wave)
     ) >= 16, f"TILE_N//(4//k_wave) must be >= 16 (num_acc_n>=1), got TILE_N={TILE_N}, k_wave={k_wave}"
+    # Whole 16-wide groups per N-wave, else num_acc_n truncates (TILE_N=96: 8 of 24).
+    assert TILE_N % (16 * (4 // k_wave)) == 0, (
+        f"TILE_N must be a multiple of {16 * (4 // k_wave)} (16*(4//k_wave), else "
+        f"num_acc_n truncates and drops columns), got TILE_N={TILE_N}, k_wave={k_wave}"
+    )
     assert BM % 16 == 0, f"BM must be a multiple of 16, got {BM}"
     NUM_N_BLOCKS = _INTER // TILE_N