From d993158ea735faecc528bc3066cc0f46f055291f Mon Sep 17 00:00:00 2001 From: BlivionIaG Date: Fri, 13 Nov 2026 00:19:08 +0100 Subject: [PATCH 01/18] feat(rdna2): opt-in LeapQuant INT8 GDN recurrent state (VLLM_RDNA_GDN_LEAPQUANT) Port of LeapQuant (arXiv 2609.38166, vllm#64725) to the RDNA2 GDN path: the FP32 [V, K] state per value head becomes a 28,768-byte blob (INT8 residual + rank-4 FP16 compensator + FP16 per-token records, window 16). MTP verify commits accepted records instead of per-token state copies; align-mode prefix-cache copies commit bias + 1 records in the HIP copy kernel. Default off; gfx10x only. decode_kl.py: decode-path top-k KL between served configurations. Signed-off-by: BlivionIaG --- CMakeLists.txt | 1 + csrc/rocm/gdn_leapquant_rdna2.cu | 887 ++++++++++++++++++ csrc/rocm/ops.h | 21 + csrc/rocm/torch_bindings.cpp | 24 + csrc/rocm/v1_mamba_rdna2.cu | 49 +- docs/rdna2/leapquant-gdn-2026-11-13.md | 71 ++ .../kernels/mamba/test_gdn_leapquant_rdna2.py | 323 +++++++ tools/rdna/port_v031/decode_kl.py | 216 +++++ tools/rdna/recipes/flashnext-mtp2.env | 3 + vllm/model_executor/layers/mamba/gdn/base.py | 9 + .../layers/mamba/gdn/qwen_gdn_linear_attn.py | 235 ++++- .../layers/mamba/mamba_utils.py | 37 +- .../layers/mamba/ops/leapquant_gdn.py | 375 ++++++++ vllm/v1/worker/mamba_utils.py | 44 +- 14 files changed, 2269 insertions(+), 26 deletions(-) create mode 100644 csrc/rocm/gdn_leapquant_rdna2.cu create mode 100644 docs/rdna2/leapquant-gdn-2026-11-13.md create mode 100644 tests/kernels/mamba/test_gdn_leapquant_rdna2.py create mode 100755 tools/rdna/port_v031/decode_kl.py create mode 100644 vllm/model_executor/layers/mamba/ops/leapquant_gdn.py diff --git a/CMakeLists.txt b/CMakeLists.txt index e453a73620de..21533c557ab4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1668,6 +1668,7 @@ if(VLLM_GPU_LANG STREQUAL "HIP") "csrc/rocm/moe_q_gemm_rdna2.cu" "csrc/rocm/moe_w4a8_rdna2.cu" "csrc/rocm/gdn_decode_rdna2.cu" + "csrc/rocm/gdn_leapquant_rdna2.cu" "csrc/rocm/causal_conv1d_rdna2.cu" "csrc/rocm/gdn_prefill_prep_rdna2.cu" "csrc/rocm/gdn_prefill_kkt_rdna2.cu" diff --git a/csrc/rocm/gdn_leapquant_rdna2.cu b/csrc/rocm/gdn_leapquant_rdna2.cu new file mode 100644 index 000000000000..6b996be53b2e --- /dev/null +++ b/csrc/rocm/gdn_leapquant_rdna2.cu @@ -0,0 +1,887 @@ +// LeapQuant INT8 recurrent-state storage for GatedDeltaNet on AMD RDNA2 +// (gfx1030). arXiv 2609.38166; layout and FP32 oracle: +// vllm/model_executor/layers/mamba/ops/leapquant_gdn.py. +// +// One uint8 blob per (cache block, value head) holds an INT8 residual with +// per-value-row scales and per-key-column smoothing, r FP16 compensator +// tokens, a ring of FP16 per-token records (k_j, u_j, g_j) and an int32 +// header [cursor, pending]. The FP32 state is +// S_c = replay(records[0:c], Z*B*C + U~^T K~), +// replay step j: S <- exp(g_j) * S + u_j k_j^T. +// +// Kernels (all launched on the current stream, cudagraph-safe): +// gdn_lq_normalize_rdna2 commit `num_accepted` pending records (one for a +// slot-0 reader: num_accepted = None), fold full +// windows into a new boundary (one WG per head). +// gdn_lq_decode_rdna2 decode / MTP verify on normalized blobs: replay, +// run T tokens, append T records, pending = T. +// gdn_lq_materialize_rdna2 normalized blob -> FP32 [N, HV, V, K]. +// gdn_lq_store_rdna2 FP32 [N, HV, V, K] -> blob (fold, cursor 0). + +#include +#include +#include + +#include +#include + +#include + +namespace { + +constexpr int LQ_K = 128; // head_k_dim +constexpr int LQ_BV = 32; // V rows per decode workgroup +constexpr int LQ_THREADS = 256; // decode: 32 v-rows x 8 k-slices +constexpr int LQ_R = 4; // compensator tokens (paper: r = 4 at 8 bits) +constexpr int LQ_ALIGN = 16; +constexpr float LQ_INT8_MAX = 127.0f; +constexpr float LQ_SMOOTH_FLOOR = 1e-6f; +constexpr float LQ_ORTH_EPS = 1e-12f; +constexpr float LQ_FP16_MAX = 65504.0f; +constexpr float LQ_L2NORM_EPS = 1e-6f; +constexpr float LQ_SOFTPLUS_THRESHOLD = 20.0f; +constexpr float LQ_LOG2E = 0x1.715476p+0f; + +__device__ __forceinline__ float lq_exp(float x) { return exp2f(x * LQ_LOG2E); } + +__device__ __forceinline__ float lq_ksum(float x) { + x += __shfl_xor_sync(0xffffffffffffffffULL, x, 1); + x += __shfl_xor_sync(0xffffffffffffffffULL, x, 2); + x += __shfl_xor_sync(0xffffffffffffffffULL, x, 4); + return x; +} + +__device__ __forceinline__ float lq_kmax(float x) { + x = fmaxf(x, __shfl_xor_sync(0xffffffffffffffffULL, x, 1)); + x = fmaxf(x, __shfl_xor_sync(0xffffffffffffffffULL, x, 2)); + x = fmaxf(x, __shfl_xor_sync(0xffffffffffffffffULL, x, 4)); + return x; +} + +__device__ __forceinline__ float lq_wsum(float x) { +#pragma unroll + for (int m = 1; m < 32; m <<= 1) + x += __shfl_xor_sync(0xffffffffffffffffULL, x, m); + return x; +} + +__device__ __forceinline__ float lq_f32(float x) { return x; } +__device__ __forceinline__ float lq_f32(__half x) { return __half2float(x); } + +__host__ __device__ __forceinline__ long lq_align(long n) { + return (n + LQ_ALIGN - 1) / LQ_ALIGN * LQ_ALIGN; +} + +// Byte offsets of the blob sections; must match LeapQuantLayout.offsets. +struct LqLayout { + long z, bscale, csmooth, comp_k, comp_u, rec_k, rec_u, rec_g, header, total; + int V, r, cap, window, power_iters; + + __host__ __device__ void init(int V_, int r_, int cap_, int window_, + int iters_) { + V = V_; + r = r_; + cap = cap_; + window = window_; + power_iters = iters_; + long off = 0; + z = off; + off = lq_align(off + (long)V * LQ_K); + bscale = off; + off = lq_align(off + 4L * V); + csmooth = off; + off = lq_align(off + 4L * LQ_K); + comp_k = off; + off = lq_align(off + 2L * r * LQ_K); + comp_u = off; + off = lq_align(off + 2L * r * V); + rec_k = off; + off = lq_align(off + 2L * cap * LQ_K); + rec_u = off; + off = lq_align(off + 2L * cap * V); + rec_g = off; + off = lq_align(off + 4L * cap); + header = off; + off = lq_align(off + 16); + total = off; + } +}; + +template +__device__ __forceinline__ T* lq_ptr(uint8_t* base, long off) { + return reinterpret_cast(base + off); +} + +// Headers of blocks a request has not written yet hold stale bytes; anything +// outside 0 <= cursor, 0 <= pending, cursor + pending <= cap reads as empty +// so no kernel indexes the record ring out of bounds. Such blocks are either +// overwritten by a prefill store or masked by has_initial_state. +__device__ __forceinline__ bool lq_header_ok(int cursor, int pending, + const LqLayout& L) { + return cursor >= 0 && pending >= 0 && cursor <= L.cap && + pending <= L.cap - cursor; +} + +// --------------------------------------------------------------------------- +// Fold: one workgroup (256 threads) per (request, head). Thread t owns rows +// v = vg + 32 * i (i < RPT = V / 32, vg = t >> 3) and columns +// k0 = (t & 7) * 16 .. k0 + 15. Every reduction is a fixed-order tree, so a +// fold is bitwise reproducible run to run. +// --------------------------------------------------------------------------- + +template +struct LqFoldSmem { + float red[8][LQ_K * LQ_R]; // per-wave partial sums over v + float kr[LQ_K * LQ_R]; // [k][r] (X0, then S^T Q) + float vr[RPT * 32 * LQ_R]; // [v][r] (Y, then Q) + float csm[LQ_K]; + int flag; +}; + +// Sum over all v of part[j][c] (j < 16 local k, c < NCOL) into +// out[k * NCOL + c]; part is per thread, already summed over its RPT rows. +template +__device__ void lq_reduce_over_v(LqFoldSmem& sm, + const float (&part)[16][NCOL], float* out) { + const int tid = threadIdx.x; + const int lane = tid & 31; + const int wave = tid >> 5; + const int k0 = (tid & 7) * 16; +#pragma unroll + for (int j = 0; j < 16; ++j) { +#pragma unroll + for (int c = 0; c < NCOL; ++c) { + float x = part[j][c]; + x += __shfl_xor_sync(0xffffffffffffffffULL, x, 8); + x += __shfl_xor_sync(0xffffffffffffffffULL, x, 16); + if (lane < 8) sm.red[wave][(k0 + j) * NCOL + c] = x; + } + } + __syncthreads(); + for (int e = tid; e < LQ_K * NCOL; e += LQ_THREADS) { + float s = 0.0f; +#pragma unroll + for (int w = 0; w < 8; ++w) s += sm.red[w][e]; + out[e] = s; + } + __syncthreads(); +} + +// Modified Gram-Schmidt over the LQ_R columns of vr ([V][r]) by wave 0. +template +__device__ void lq_orthonormalize(LqFoldSmem& sm) { + const int tid = threadIdx.x; + if (tid < 32) { +#pragma unroll + for (int c = 0; c < LQ_R; ++c) { +#pragma unroll + for (int p = 0; p < c; ++p) { + float d = 0.0f; +#pragma unroll + for (int e = 0; e < RPT; ++e) { + const int v = tid + 32 * e; + d += sm.vr[v * LQ_R + c] * sm.vr[v * LQ_R + p]; + } + d = lq_wsum(d); +#pragma unroll + for (int e = 0; e < RPT; ++e) { + const int v = tid + 32 * e; + sm.vr[v * LQ_R + c] -= d * sm.vr[v * LQ_R + p]; + } + } + float n2 = 0.0f; +#pragma unroll + for (int e = 0; e < RPT; ++e) { + const float y = sm.vr[(tid + 32 * e) * LQ_R + c]; + n2 += y * y; + } + const float n = sqrtf(lq_wsum(n2)); + const float inv = n > LQ_ORTH_EPS ? 1.0f / n : 0.0f; +#pragma unroll + for (int e = 0; e < RPT; ++e) sm.vr[(tid + 32 * e) * LQ_R + c] *= inv; + } + } + __syncthreads(); +} + +// vr[v][c] = sum_k S[v][k] * kr[k][c] +template +__device__ void lq_apply_s(LqFoldSmem& sm, const float (&S)[RPT][16]) { + const int tid = threadIdx.x; + const int vg = tid >> 3; + const int k0 = (tid & 7) * 16; +#pragma unroll + for (int i = 0; i < RPT; ++i) { +#pragma unroll + for (int c = 0; c < LQ_R; ++c) { + float acc = 0.0f; +#pragma unroll + for (int j = 0; j < 16; ++j) acc += S[i][j] * sm.kr[(k0 + j) * LQ_R + c]; + acc = lq_ksum(acc); + if ((tid & 7) == 0) sm.vr[(vg + 32 * i) * LQ_R + c] = acc; + } + } + __syncthreads(); +} + +// kr[k][c] = sum_v S[v][k] * vr[v][c] +template +__device__ void lq_apply_st(LqFoldSmem& sm, const float (&S)[RPT][16]) { + const int tid = threadIdx.x; + const int vg = tid >> 3; + float part[16][LQ_R]; +#pragma unroll + for (int j = 0; j < 16; ++j) { +#pragma unroll + for (int c = 0; c < LQ_R; ++c) { + float acc = 0.0f; +#pragma unroll + for (int i = 0; i < RPT; ++i) + acc += S[i][j] * sm.vr[(vg + 32 * i) * LQ_R + c]; + part[j][c] = acc; + } + } + lq_reduce_over_v(sm, part, sm.kr); +} + +// Fold the FP32 state held in registers into the blob (cursor = pending = 0). +// warm: start the subspace iteration from the blob's previous compensator +// keys (a live window boundary); otherwise (a freshly allocated block whose +// bytes are stale) from the fixed integer block. +template +__device__ void lq_fold(LqFoldSmem& sm, float (&S)[RPT][16], uint8_t* blob, + const LqLayout& L, bool warm) { + const int tid = threadIdx.x; + const int vg = tid >> 3; + const int ks = tid & 7; + const int k0 = ks * 16; + const int V = L.V; + + const __half* pck = lq_ptr<__half>(blob, L.comp_k); + int nz = 0; + if (warm) { + for (int e = tid; e < LQ_R * LQ_K; e += LQ_THREADS) { + const float x = __half2float(pck[e]); + sm.kr[(e % LQ_K) * LQ_R + e / LQ_K] = x; + nz |= (x != 0.0f); + } + } + nz = __syncthreads_or(nz); + if (!nz) { + for (int e = tid; e < LQ_R * LQ_K; e += LQ_THREADS) { + const int k = e % LQ_K, c = e / LQ_K; + sm.kr[k * LQ_R + c] = (float)(((k * (2 * c + 3) + 7 * c + 1) % 17) - 8); + } + __syncthreads(); + } + + lq_apply_s(sm, S); + lq_orthonormalize(sm); + for (int it = 0; it < L.power_iters; ++it) { + lq_apply_st(sm, S); + lq_apply_s(sm, S); + lq_orthonormalize(sm); + } + lq_apply_st(sm, S); // kr = K~ (S^T Q), vr = Q = U~ + + // Round compensators to fp16, store them, keep the rounded values. + __half* ck = lq_ptr<__half>(blob, L.comp_k); + __half* cu = lq_ptr<__half>(blob, L.comp_u); + for (int e = tid; e < LQ_R * LQ_K; e += LQ_THREADS) { + const int k = e % LQ_K, c = e / LQ_K; + const __half h = __float2half( + fminf(fmaxf(sm.kr[k * LQ_R + c], -LQ_FP16_MAX), LQ_FP16_MAX)); + ck[c * LQ_K + k] = h; + sm.kr[k * LQ_R + c] = __half2float(h); + } + for (int e = tid; e < LQ_R * V; e += LQ_THREADS) { + const int v = e % V, c = e / V; + const __half h = __float2half( + fminf(fmaxf(sm.vr[v * LQ_R + c], -LQ_FP16_MAX), LQ_FP16_MAX)); + cu[c * V + v] = h; + sm.vr[v * LQ_R + c] = __half2float(h); + } + __syncthreads(); + + // Residual R = S - U~ K~^T (in place) and per-key smoothing. + float part[16][1]; +#pragma unroll + for (int j = 0; j < 16; ++j) part[j][0] = 0.0f; +#pragma unroll + for (int i = 0; i < RPT; ++i) { + const int v = vg + 32 * i; +#pragma unroll + for (int j = 0; j < 16; ++j) { + float c_sum = 0.0f; +#pragma unroll + for (int c = 0; c < LQ_R; ++c) + c_sum += sm.vr[v * LQ_R + c] * sm.kr[(k0 + j) * LQ_R + c]; + S[i][j] -= c_sum; + part[j][0] += fabsf(S[i][j]); + } + } + lq_reduce_over_v(sm, part, sm.csm); + float* csm_out = lq_ptr(blob, L.csmooth); + for (int k = tid; k < LQ_K; k += LQ_THREADS) { + const float c = fmaxf(sqrtf(sm.csm[k] / (float)V), LQ_SMOOTH_FLOOR); + sm.csm[k] = c; + csm_out[k] = c; + } + __syncthreads(); + + // Per-value-row scale and INT8 payload. + float* bsc = lq_ptr(blob, L.bscale); + int8_t* z = lq_ptr(blob, L.z); +#pragma unroll + for (int i = 0; i < RPT; ++i) { + const int v = vg + 32 * i; + float m = 0.0f; +#pragma unroll + for (int j = 0; j < 16; ++j) { + S[i][j] /= sm.csm[k0 + j]; + m = fmaxf(m, fabsf(S[i][j])); + } + const float bscale = lq_kmax(m) / LQ_INT8_MAX; + if (ks == 0) bsc[v] = bscale; + const float inv = bscale > 0.0f ? 1.0f / bscale : 0.0f; + union { + int8_t c[16]; + uint4 u; + } pk; +#pragma unroll + for (int j = 0; j < 16; ++j) { + const float q = + fminf(fmaxf(rintf(S[i][j] * inv), -LQ_INT8_MAX), LQ_INT8_MAX); + pk.c[j] = (int8_t)q; + } + *reinterpret_cast(z + (long)v * LQ_K + k0) = pk.u; + } + if (tid == 0) { + int* hdr = lq_ptr(blob, L.header); + hdr[0] = 0; + hdr[1] = 0; + } +} + +// Boundary state + replay of c records into this thread's [RPT][16] tile. +template +__device__ void lq_load_full(float (&S)[RPT][16], uint8_t* blob, + const LqLayout& L, int c) { + const int tid = threadIdx.x; + const int vg = tid >> 3; + const int k0 = (tid & 7) * 16; + const int8_t* z = lq_ptr(blob, L.z); + const float* bsc = lq_ptr(blob, L.bscale); + const float* csm = lq_ptr(blob, L.csmooth); + const __half* ck = lq_ptr<__half>(blob, L.comp_k); + const __half* cu = lq_ptr<__half>(blob, L.comp_u); + float cs[16]; +#pragma unroll + for (int j = 0; j < 16; ++j) cs[j] = csm[k0 + j]; + for (int i = 0; i < RPT; ++i) { + const int v = vg + 32 * i; + const float bs = bsc[v]; + union { + int8_t c[16]; + uint4 u; + } pk; + pk.u = *reinterpret_cast(z + (long)v * LQ_K + k0); +#pragma unroll + for (int j = 0; j < 16; ++j) S[i][j] = (float)pk.c[j] * bs * cs[j]; + for (int cc = 0; cc < L.r; ++cc) { + const float u = __half2float(cu[cc * L.V + v]); +#pragma unroll + for (int j = 0; j < 16; ++j) + S[i][j] += u * __half2float(ck[cc * LQ_K + k0 + j]); + } + } + const __half* rk = lq_ptr<__half>(blob, L.rec_k); + const __half* ru = lq_ptr<__half>(blob, L.rec_u); + const float* rg = lq_ptr(blob, L.rec_g); + for (int t = 0; t < c; ++t) { + const float decay = lq_exp(rg[t]); + float kk[16]; +#pragma unroll + for (int j = 0; j < 16; ++j) kk[j] = __half2float(rk[t * LQ_K + k0 + j]); + for (int i = 0; i < RPT; ++i) { + const float u = __half2float(ru[t * L.V + vg + 32 * i]); +#pragma unroll + for (int j = 0; j < 16; ++j) S[i][j] = S[i][j] * decay + u * kk[j]; + } + } +} + +template +__global__ void __launch_bounds__(LQ_THREADS) + gdn_lq_normalize_kernel(uint8_t* __restrict__ blobs, + const int* __restrict__ state_indices, + long idx_stride, + const int* __restrict__ num_accepted, + long stride_block, long stride_head, + long num_blocks, int HV, LqLayout L) { + __shared__ LqFoldSmem sm; + const int i_n = blockIdx.x / HV; + const int i_hv = blockIdx.x % HV; + const long idx = state_indices[(long)i_n * idx_stride]; + if (idx <= 0 || idx >= num_blocks) return; + uint8_t* blob = blobs + idx * stride_block + (long)i_hv * stride_head; + int* hdr = lq_ptr(blob, L.header); + if (threadIdx.x == 0) { + int cursor = hdr[0]; + int pending = hdr[1]; + if (!lq_header_ok(cursor, pending, L)) { + cursor = 0; + pending = 0; + } + int acc = 0; + if (pending > 0) { + acc = min(num_accepted != nullptr ? num_accepted[i_n] : 1, pending); + acc = max(acc, 0); + } + sm.flag = cursor + acc; + } + __syncthreads(); + const int c = sm.flag; + if (c < L.window) { + if (threadIdx.x == 0) { + hdr[0] = c; + hdr[1] = 0; + } + return; + } + float S[RPT][16]; + lq_load_full(S, blob, L, c); + __syncthreads(); + lq_fold(sm, S, blob, L, /*warm=*/true); +} + +template +__global__ void __launch_bounds__(LQ_THREADS) + gdn_lq_store_kernel(const float* __restrict__ state, // [N, HV, V, K] + uint8_t* __restrict__ blobs, + const int* __restrict__ state_indices, long idx_stride, + long stride_state_n, long stride_state_h, + long stride_block, long stride_head, long num_blocks, + int HV, LqLayout L) { + __shared__ LqFoldSmem sm; + const int i_n = blockIdx.x / HV; + const int i_hv = blockIdx.x % HV; + const long idx = state_indices[(long)i_n * idx_stride]; + if (idx <= 0 || idx >= num_blocks) return; + uint8_t* blob = blobs + idx * stride_block + (long)i_hv * stride_head; + const int tid = threadIdx.x; + const int vg = tid >> 3; + const int k0 = (tid & 7) * 16; + const float* src = state + i_n * stride_state_n + i_hv * stride_state_h; + float S[RPT][16]; + for (int i = 0; i < RPT; ++i) { + const float4* p = + reinterpret_cast(src + (long)(vg + 32 * i) * LQ_K + k0); +#pragma unroll + for (int j = 0; j < 4; ++j) { + const float4 x = p[j]; + S[i][4 * j] = x.x; + S[i][4 * j + 1] = x.y; + S[i][4 * j + 2] = x.z; + S[i][4 * j + 3] = x.w; + } + } + lq_fold(sm, S, blob, L, /*warm=*/false); +} + +template +__global__ void __launch_bounds__(LQ_THREADS) + gdn_lq_materialize_kernel(const uint8_t* __restrict__ blobs, + float* __restrict__ state, + const int* __restrict__ state_indices, + long idx_stride, long stride_state_n, + long stride_state_h, long stride_block, + long stride_head, long num_blocks, int HV, + LqLayout L) { + const int i_n = blockIdx.x / HV; + const int i_hv = blockIdx.x % HV; + const long idx = state_indices[(long)i_n * idx_stride]; + const int tid = threadIdx.x; + const int vg = tid >> 3; + const int k0 = (tid & 7) * 16; + float* dst = state + i_n * stride_state_n + i_hv * stride_state_h; + float S[RPT][16]; + if (idx <= 0 || idx >= num_blocks) { + for (int i = 0; i < RPT; ++i) +#pragma unroll + for (int j = 0; j < 16; ++j) S[i][j] = 0.0f; + } else { + uint8_t* blob = const_cast(blobs) + idx * stride_block + + (long)i_hv * stride_head; + const int* hdr = lq_ptr(blob, L.header); + lq_load_full(S, blob, L, lq_header_ok(hdr[0], hdr[1], L) ? hdr[0] : 0); + } + for (int i = 0; i < RPT; ++i) { + float4* p = + reinterpret_cast(dst + (long)(vg + 32 * i) * LQ_K + k0); +#pragma unroll + for (int j = 0; j < 4; ++j) + p[j] = make_float4(S[i][4 * j], S[i][4 * j + 1], S[i][4 * j + 2], + S[i][4 * j + 3]); + } +} + +// --------------------------------------------------------------------------- +// Decode / verify: one workgroup per (V-tile of 32 rows, request x head), +// lane = v_local * 8 + k-slice, 16 fp32 state registers per thread, the same +// tiling and op order as gdn_spec_verify_rdna2. +// --------------------------------------------------------------------------- + +template +__global__ void __launch_bounds__(LQ_THREADS) + __attribute__((amdgpu_waves_per_eu(2, 4))) gdn_lq_decode_kernel( + const __half* __restrict__ q, const __half* __restrict__ k, + const __half* __restrict__ v, const __half* __restrict__ a, + const __half* __restrict__ b, const AT* __restrict__ A_log, + const DT* __restrict__ dt_bias, __half* __restrict__ out, + uint8_t* __restrict__ blobs, const int* __restrict__ state_indices, + long idx_stride, const int* __restrict__ cu_seqlens, int H, int HV, + long stride_q_tok, long stride_k_tok, long stride_v_tok, + long stride_a_tok, long stride_b_tok, long stride_o_tok, + long stride_block, long stride_head, long num_blocks, float scale, + LqLayout L) { + const int i_v = blockIdx.x; + const int i_nh = blockIdx.y; + const int i_n = i_nh / HV; + const int i_hv = i_nh % HV; + const int i_h = i_hv / (HV / H); + const int V = L.V; + + const int lane = threadIdx.x; + const int v_local = lane >> 3; + const int ks = lane & 7; + const int o_v = i_v * LQ_BV + v_local; + const int k0 = ks * 16; + + const int bos = cu_seqlens[i_n]; + const int n_tok = cu_seqlens[i_n + 1] - bos; + if (n_tok <= 0) return; + + const long idx = state_indices[(long)i_n * idx_stride]; + if (idx <= 0 || idx >= num_blocks) { + for (int t = 0; t < n_tok; ++t) + out[(long)(bos + t) * stride_o_tok + (long)i_hv * V + o_v] = + __float2half(0.0f); + return; + } + uint8_t* blob = blobs + idx * stride_block + (long)i_hv * stride_head; + int* hdr = lq_ptr(blob, L.header); + const int c = lq_header_ok(hdr[0], hdr[1], L) ? hdr[0] : 0; + + // Boundary state tile. + float h[16]; + { + const float bs = lq_ptr(blob, L.bscale)[o_v]; + const float* csm = lq_ptr(blob, L.csmooth); + union { + int8_t c8[16]; + uint4 u; + } pk; + pk.u = *reinterpret_cast(lq_ptr(blob, L.z) + + (long)o_v * LQ_K + k0); +#pragma unroll + for (int j = 0; j < 16; ++j) h[j] = (float)pk.c8[j] * bs * csm[k0 + j]; + const __half* ck = lq_ptr<__half>(blob, L.comp_k); + const __half* cu = lq_ptr<__half>(blob, L.comp_u); + for (int cc = 0; cc < L.r; ++cc) { + const float u = __half2float(cu[cc * V + o_v]); +#pragma unroll + for (int j = 0; j < 16; ++j) + h[j] += u * __half2float(ck[cc * LQ_K + k0 + j]); + } + } + __half* rk = lq_ptr<__half>(blob, L.rec_k); + __half* ru = lq_ptr<__half>(blob, L.rec_u); + float* rg = lq_ptr(blob, L.rec_g); + for (int t = 0; t < c; ++t) { + const float decay = lq_exp(rg[t]); + const float u = __half2float(ru[t * V + o_v]); +#pragma unroll + for (int j = 0; j < 16; ++j) + h[j] = h[j] * decay + u * __half2float(rk[t * LQ_K + k0 + j]); + } + + const int n_rec = min(n_tok, L.cap - c); + for (int t = 0; t < n_tok; ++t) { + const long g = bos + t; + const __half* p_q = q + g * stride_q_tok + (long)i_h * LQ_K + k0; + const __half* p_k = k + g * stride_k_tok + (long)i_h * LQ_K + k0; + float qf[16], kf[16]; +#pragma unroll + for (int j = 0; j < 16; ++j) { + qf[j] = __half2float(p_q[j]); + kf[j] = __half2float(p_k[j]); + } + float sq = 0.0f, sk = 0.0f; +#pragma unroll + for (int j = 0; j < 16; ++j) { + sq += qf[j] * qf[j]; + sk += kf[j] * kf[j]; + } + sq = lq_ksum(sq); + sk = lq_ksum(sk); + const float q_norm = __frsqrt_rn(sq + LQ_L2NORM_EPS); + const float k_norm = __frsqrt_rn(sk + LQ_L2NORM_EPS); +#pragma unroll + for (int j = 0; j < 16; ++j) { + qf[j] = qf[j] * q_norm * scale; + kf[j] = kf[j] * k_norm; + } + const float a_val = __half2float(a[g * stride_a_tok + i_hv]); + const float b_val = __half2float(b[g * stride_b_tok + i_hv]); + const float x = a_val + lq_f32(dt_bias[i_hv]); + const float softplus = + (x <= LQ_SOFTPLUS_THRESHOLD) ? logf(1.0f + lq_exp(x)) : x; + const float g_val = -lq_exp(lq_f32(A_log[i_hv])) * softplus; + const float beta = 1.0f / (1.0f + lq_exp(-b_val)); + const float decay = lq_exp(g_val); +#pragma unroll + for (int j = 0; j < 16; ++j) h[j] *= decay; + const float v_val = + __half2float(v[g * stride_v_tok + (long)i_hv * V + o_v]); + float acc = 0.0f; +#pragma unroll + for (int j = 0; j < 16; ++j) acc += h[j] * kf[j]; + acc = lq_ksum(acc); + const float u = (v_val - acc) * beta; +#pragma unroll + for (int j = 0; j < 16; ++j) h[j] += u * kf[j]; + float o_acc = 0.0f; +#pragma unroll + for (int j = 0; j < 16; ++j) o_acc += h[j] * qf[j]; + o_acc = lq_ksum(o_acc); + out[g * stride_o_tok + (long)i_hv * V + o_v] = __float2half(o_acc); + + if (t < n_rec) { + const int pos = c + t; + ru[pos * V + o_v] = __float2half(u); + if (i_v == 0 && v_local == 0) { +#pragma unroll + for (int j = 0; j < 16; ++j) + rk[pos * LQ_K + k0 + j] = __float2half(kf[j]); + if (ks == 0) rg[pos] = g_val; + } + } + } + if (i_v == 0 && lane == 0) hdr[1] = n_rec; +} + +LqLayout make_layout(int V, int64_t num_comp, int64_t capacity, int64_t window, + int64_t power_iters) { + TORCH_CHECK(num_comp == LQ_R, + "LeapQuant GDN is built for num_comp == ", LQ_R); + TORCH_CHECK(window >= 1 && capacity >= window, + "LeapQuant needs capacity >= window >= 1"); + TORCH_CHECK(power_iters >= 0, "power_iters must be >= 0"); + LqLayout L; + L.init(V, (int)num_comp, (int)capacity, (int)window, (int)power_iters); + return L; +} + +void check_blob(const torch::Tensor& blob, const LqLayout& L, int64_t HV) { + TORCH_CHECK( + blob.is_cuda() && blob.scalar_type() == at::kByte && blob.dim() == 3, + "LeapQuant blob must be CUDA uint8 [blocks, HV, nbytes]"); + TORCH_CHECK( + blob.size(1) == HV && blob.size(2) == L.total && blob.stride(2) == 1, + "LeapQuant blob shape mismatch: expected [blocks, ", HV, ", ", L.total, + "] with a dense last dim"); + TORCH_CHECK(blob.stride(0) % LQ_ALIGN == 0 && blob.stride(1) % LQ_ALIGN == 0, + "LeapQuant blob strides must be 16-byte multiples"); + TORCH_CHECK(reinterpret_cast(blob.data_ptr()) % LQ_ALIGN == 0, + "LeapQuant blob must be 16-byte aligned"); +} + +// Index vectors may be strided column views (spec_state_indices[:, 0]). +void check_indices(const torch::Tensor& idx, int64_t N) { + TORCH_CHECK(idx.is_cuda() && idx.scalar_type() == at::kInt && + idx.dim() == 1 && idx.size(0) >= N, + "state_indices must be CUDA int32 [N]"); +} + +#define LQ_DISPATCH_RPT(V, ...) \ + switch ((V) / 32) { \ + case 4: { \ + constexpr int RPT = 4; \ + __VA_ARGS__; \ + break; \ + } \ + default: \ + TORCH_CHECK(false, "LeapQuant GDN supports head_v_dim == 128"); \ + } + +} // namespace + +void gdn_lq_normalize_rdna2(torch::Tensor blob, torch::Tensor state_indices, + const std::optional& num_accepted, + int64_t head_v_dim, int64_t num_comp, + int64_t capacity, int64_t window, + int64_t power_iters) { + const LqLayout L = + make_layout((int)head_v_dim, num_comp, capacity, window, power_iters); + const int64_t HV = blob.size(1); + check_blob(blob, L, HV); + const int64_t N = state_indices.size(0); + check_indices(state_indices, N); + const bool has_acc = num_accepted.has_value(); + if (has_acc) { + TORCH_CHECK(num_accepted->is_cuda() && + num_accepted->scalar_type() == at::kInt && + num_accepted->dim() == 1 && num_accepted->stride(0) == 1 && + num_accepted->numel() >= N, + "num_accepted must be unit-stride CUDA int32 [N]"); + } + if (N == 0) return; + const at::cuda::OptionalCUDAGuard guard(blob.device()); + auto stream = at::cuda::getCurrentCUDAStream(); + LQ_DISPATCH_RPT( + head_v_dim, + gdn_lq_normalize_kernel<<>>( + blob.data_ptr(), state_indices.data_ptr(), + state_indices.stride(0), + has_acc ? num_accepted->data_ptr() : nullptr, blob.stride(0), + blob.stride(1), blob.size(0), (int)HV, L)); + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} + +void gdn_lq_store_rdna2(torch::Tensor state, torch::Tensor blob, + torch::Tensor state_indices, int64_t num_comp, + int64_t capacity, int64_t window, int64_t power_iters) { + TORCH_CHECK(state.is_cuda() && state.scalar_type() == at::kFloat && + state.dim() == 4 && state.size(3) == LQ_K && + state.stride(3) == 1 && state.stride(2) == LQ_K, + "state must be CUDA fp32 [N, HV, V, 128] with dense [V, K]"); + const int64_t V = state.size(2); + const LqLayout L = + make_layout((int)V, num_comp, capacity, window, power_iters); + const int64_t HV = blob.size(1); + check_blob(blob, L, HV); + TORCH_CHECK(state.size(1) == HV, "state / blob head count mismatch"); + const int64_t N = state.size(0); + check_indices(state_indices, N); + if (N == 0) return; + const at::cuda::OptionalCUDAGuard guard(blob.device()); + auto stream = at::cuda::getCurrentCUDAStream(); + LQ_DISPATCH_RPT( + V, gdn_lq_store_kernel<<>>( + state.data_ptr(), blob.data_ptr(), + state_indices.data_ptr(), state_indices.stride(0), + state.stride(0), state.stride(1), blob.stride(0), blob.stride(1), + blob.size(0), (int)HV, L)); + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} + +void gdn_lq_materialize_rdna2(torch::Tensor blob, torch::Tensor state, + torch::Tensor state_indices, int64_t num_comp, + int64_t capacity, int64_t window) { + TORCH_CHECK(state.is_cuda() && state.scalar_type() == at::kFloat && + state.dim() == 4 && state.size(3) == LQ_K && + state.stride(3) == 1 && state.stride(2) == LQ_K, + "state must be CUDA fp32 [N, HV, V, 128] with dense [V, K]"); + const int64_t V = state.size(2); + const LqLayout L = make_layout((int)V, num_comp, capacity, window, 0); + const int64_t HV = blob.size(1); + check_blob(blob, L, HV); + TORCH_CHECK(state.size(1) == HV, "state / blob head count mismatch"); + const int64_t N = state.size(0); + check_indices(state_indices, N); + if (N == 0) return; + const at::cuda::OptionalCUDAGuard guard(blob.device()); + auto stream = at::cuda::getCurrentCUDAStream(); + LQ_DISPATCH_RPT( + V, gdn_lq_materialize_kernel<<>>( + blob.data_ptr(), state.data_ptr(), + state_indices.data_ptr(), state_indices.stride(0), + state.stride(0), state.stride(1), blob.stride(0), blob.stride(1), + blob.size(0), (int)HV, L)); + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} + +void gdn_lq_decode_rdna2(torch::Tensor q, torch::Tensor k, torch::Tensor v, + torch::Tensor a, torch::Tensor b, torch::Tensor A_log, + torch::Tensor dt_bias, torch::Tensor out, + torch::Tensor blob, torch::Tensor cu_seqlens, + torch::Tensor state_indices, double scale, + int64_t num_comp, int64_t capacity, int64_t window) { + TORCH_CHECK( + q.scalar_type() == at::kHalf && k.scalar_type() == at::kHalf && + v.scalar_type() == at::kHalf && a.scalar_type() == at::kHalf && + b.scalar_type() == at::kHalf && out.scalar_type() == at::kHalf, + "gdn_lq_decode_rdna2 is fp16-only (gfx1030)"); + TORCH_CHECK(q.dim() == 4 && k.dim() == 4 && v.dim() == 4 && out.dim() == 4, + "gdn_lq_decode_rdna2 wants [1, T, H, K] / [1, T, HV, V]"); + TORCH_CHECK((a.dim() == 2 || a.dim() == 3) && a.dim() == b.dim(), + "gdn_lq_decode_rdna2 wants a/b as [T, HV] or [1, T, HV]"); + TORCH_CHECK(q.stride(3) == 1 && q.stride(2) == q.size(3) && + k.stride(3) == 1 && k.stride(2) == k.size(3) && + v.stride(3) == 1 && v.stride(2) == v.size(3) && + a.stride(-1) == 1 && b.stride(-1) == 1 && out.is_contiguous(), + "gdn_lq_decode_rdna2 requires dense per-token q/k/v heads, " + "unit-stride a/b and contiguous out"); + TORCH_CHECK(cu_seqlens.is_cuda() && cu_seqlens.dim() == 1 && + cu_seqlens.stride(0) == 1 && + cu_seqlens.scalar_type() == at::kInt, + "cu_seqlens must be unit-stride int32 [N + 1]"); + const int64_t T = q.size(1); + const int64_t H = q.size(2); + const int64_t HV = v.size(2); + const int64_t V = v.size(3); + const int64_t N = cu_seqlens.size(0) - 1; + TORCH_CHECK(q.size(3) == LQ_K, "gdn_lq_decode_rdna2 requires head_k_dim 128"); + TORCH_CHECK(HV % H == 0, "HV must be divisible by H"); + const int a_tok_dim = a.dim() - 2; + TORCH_CHECK(v.size(1) == T && out.size(1) == T && a.size(a_tok_dim) == T && + b.size(a_tok_dim) == T && a.size(-1) == HV && + b.size(-1) == HV, + "gdn_lq_decode_rdna2: token / head dim mismatch"); + const auto a_ty = A_log.scalar_type(); + const auto d_ty = dt_bias.scalar_type(); + TORCH_CHECK(A_log.dim() == 1 && A_log.is_contiguous() && + (a_ty == at::kFloat || a_ty == at::kHalf) && + dt_bias.dim() == 1 && dt_bias.is_contiguous() && + (d_ty == at::kFloat || d_ty == at::kHalf), + "A_log / dt_bias must be contiguous fp32/fp16 [HV]"); + const LqLayout L = make_layout((int)V, num_comp, capacity, window, 0); + check_blob(blob, L, HV); + check_indices(state_indices, N); + TORCH_CHECK(V == 128, "LeapQuant GDN supports head_v_dim == 128"); + if (N == 0 || T == 0) return; + + const at::cuda::OptionalCUDAGuard guard(q.device()); + auto stream = at::cuda::getCurrentCUDAStream(); + dim3 grid(V / LQ_BV, N * HV); +#define LQ_DECODE_LAUNCH(AT, DT, APTR, DPTR) \ + gdn_lq_decode_kernel<<>>( \ + reinterpret_cast(q.data_ptr()), \ + reinterpret_cast(k.data_ptr()), \ + reinterpret_cast(v.data_ptr()), \ + reinterpret_cast(a.data_ptr()), \ + reinterpret_cast(b.data_ptr()), APTR, DPTR, \ + reinterpret_cast<__half*>(out.data_ptr()), blob.data_ptr(), \ + state_indices.data_ptr(), state_indices.stride(0), \ + cu_seqlens.data_ptr(), (int)H, (int)HV, q.stride(1), k.stride(1), \ + v.stride(1), a.stride(a_tok_dim), b.stride(a_tok_dim), out.stride(1), \ + blob.stride(0), blob.stride(1), blob.size(0), (float)scale, L) + if (a_ty == at::kFloat && d_ty == at::kFloat) { + LQ_DECODE_LAUNCH(float, float, A_log.data_ptr(), + dt_bias.data_ptr()); + } else if (a_ty == at::kFloat) { + LQ_DECODE_LAUNCH(float, __half, A_log.data_ptr(), + reinterpret_cast(dt_bias.data_ptr())); + } else if (d_ty == at::kFloat) { + LQ_DECODE_LAUNCH(__half, float, + reinterpret_cast(A_log.data_ptr()), + dt_bias.data_ptr()); + } else { + LQ_DECODE_LAUNCH(__half, __half, + reinterpret_cast(A_log.data_ptr()), + reinterpret_cast(dt_bias.data_ptr())); + } +#undef LQ_DECODE_LAUNCH + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} diff --git a/csrc/rocm/ops.h b/csrc/rocm/ops.h index 832d0b1c306f..41c9bae562d2 100644 --- a/csrc/rocm/ops.h +++ b/csrc/rocm/ops.h @@ -564,6 +564,27 @@ void gdn_spec_verify_rdna2( torch::Tensor num_accepted_tokens, // [N] int32 or undefined double scale); +// LeapQuant INT8 GDN recurrent state (arXiv 2609.38166) for AMD RDNA2. +// Blob layout and FP32 oracle: vllm/model_executor/layers/mamba/ops/ +// leapquant_gdn.py. blob is uint8 [blocks, HV, nbytes]. +void gdn_lq_normalize_rdna2(torch::Tensor blob, torch::Tensor state_indices, + const std::optional& num_accepted, + int64_t head_v_dim, int64_t num_comp, + int64_t capacity, int64_t window, + int64_t power_iters); +void gdn_lq_decode_rdna2(torch::Tensor q, torch::Tensor k, torch::Tensor v, + torch::Tensor a, torch::Tensor b, torch::Tensor A_log, + torch::Tensor dt_bias, torch::Tensor out, + torch::Tensor blob, torch::Tensor cu_seqlens, + torch::Tensor state_indices, double scale, + int64_t num_comp, int64_t capacity, int64_t window); +void gdn_lq_materialize_rdna2(torch::Tensor blob, torch::Tensor state, + torch::Tensor state_indices, int64_t num_comp, + int64_t capacity, int64_t window); +void gdn_lq_store_rdna2(torch::Tensor state, torch::Tensor blob, + torch::Tensor state_indices, int64_t num_comp, + int64_t capacity, int64_t window, int64_t power_iters); + // GDN prefill kernels for AMD RDNA2 (gfx1030). Hand ports of the Triton/FLA // chain `chunk_gated_delta_rule_fwd` (chunk.py:23-86) decomposed into 5 HIP // kernels; varlen uses cu_seqlens/chunk_indices/chunk_offsets (pass empty diff --git a/csrc/rocm/torch_bindings.cpp b/csrc/rocm/torch_bindings.cpp index b38f4d2b3d35..62e934947c36 100644 --- a/csrc/rocm/torch_bindings.cpp +++ b/csrc/rocm/torch_bindings.cpp @@ -362,6 +362,30 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, rocm_ops) { "Tensor num_accepted_tokens, float scale) -> ()"); rocm_ops.impl("gdn_spec_verify_rdna2", torch::kCUDA, &gdn_spec_verify_rdna2); + // LeapQuant INT8 GDN recurrent state for AMD RDNA2 (gfx1030); opt-in via + // VLLM_RDNA_GDN_LEAPQUANT. blob = uint8 [blocks, HV, nbytes]. + rocm_ops.def( + "gdn_lq_normalize_rdna2(Tensor! blob, Tensor state_indices, " + "Tensor? num_accepted, int head_v_dim, int num_comp, int capacity, " + "int window, int power_iters) -> ()"); + rocm_ops.impl("gdn_lq_normalize_rdna2", torch::kCUDA, + &gdn_lq_normalize_rdna2); + rocm_ops.def( + "gdn_lq_decode_rdna2(Tensor q, Tensor k, Tensor v, Tensor a, Tensor b, " + "Tensor A_log, Tensor dt_bias, Tensor! out, Tensor! blob, " + "Tensor cu_seqlens, Tensor state_indices, float scale, int num_comp, " + "int capacity, int window) -> ()"); + rocm_ops.impl("gdn_lq_decode_rdna2", torch::kCUDA, &gdn_lq_decode_rdna2); + rocm_ops.def( + "gdn_lq_materialize_rdna2(Tensor blob, Tensor! state, " + "Tensor state_indices, int num_comp, int capacity, int window) -> ()"); + rocm_ops.impl("gdn_lq_materialize_rdna2", torch::kCUDA, + &gdn_lq_materialize_rdna2); + rocm_ops.def( + "gdn_lq_store_rdna2(Tensor state, Tensor! blob, Tensor state_indices, " + "int num_comp, int capacity, int window, int power_iters) -> ()"); + rocm_ops.impl("gdn_lq_store_rdna2", torch::kCUDA, &gdn_lq_store_rdna2); + // causal_conv1d_update single-token decode for AMD RDNA2 (gfx1030). // Replaces the Triton kernel that captured scratch buffer pointers and // page-faulted on cudagraph replay. diff --git a/csrc/rocm/v1_mamba_rdna2.cu b/csrc/rocm/v1_mamba_rdna2.cu index e66e71fb1e2a..bae73b51f5c3 100644 --- a/csrc/rocm/v1_mamba_rdna2.cu +++ b/csrc/rocm/v1_mamba_rdna2.cu @@ -110,10 +110,13 @@ struct MambaCopyMeta { const int64_t* block_strides; // int64: bytes per block const int32_t* elem_sizes; // int32: element size in bytes const int64_t* inner_sizes; // int64: elements in inner dimensions - const int32_t* conv_widths; // int32: conv width (0 = temporal state) + const int32_t* conv_widths; // int32: conv width (0 = temporal state, + // -1 = LeapQuant GDN blob) const int32_t* group_indices; // int32: state_idx -> block-table group const int32_t* dim_row_count; // int32: per-block dim row count + // (LeapQuant: heads per block) const int64_t* dim_row_stride; // int64: bytes between dim rows + // (LeapQuant: header byte offset) const int64_t* block_table_ptrs; // int64: per-group block-table base addrs int64_t block_table_stride_req; }; @@ -123,6 +126,12 @@ struct MambaCopyMeta { // conv state (conv_width > 0): shift the window by `token_bias` tokens, // state[bt[src_col], token_bias:] -> state[bt[dst_col], :conv_width - bias] // temporal state: state[bt[src_col + token_bias]] -> state[bt[dst_col]] +// LeapQuant GDN blob (conv_width < 0): the running state lives in the home +// block bt[src_col] as cursor + pending records, and "slot src_col + t" of +// the per-token layout is "commit t + 1 pending records". Copy the home +// blob and commit min(token_bias + 1, pending) records in the copy's +// header (cursor += that, pending = 0); a blob without pending records is +// copied as is. // --------------------------------------------------------------------------- template __device__ __forceinline__ void copy_mamba_state_block( @@ -210,6 +219,44 @@ __device__ __forceinline__ void copy_mamba_state_block( return; } + if (conv_width < 0) { + // One CTA: the header patch must follow the whole-blob copy. + if (tile_idx > 0) { + return; + } + const int64_t src_block_id = (int64_t)block_table[src_col]; + const uint64_t src_addr = base + (uint64_t)(src_block_id * block_stride); + const int32_t num_heads = m.dim_row_count[state_idx]; + const int64_t header_off = m.dim_row_stride[state_idx]; + const int64_t blob_bytes = inner_size * (int64_t)elem_size; + const int64_t head_bytes = blob_bytes / num_heads; + int32_t cursor = 0; + int32_t pending = 0; + if ((int32_t)threadIdx.x < num_heads) { + const int32_t* hdr = + (const int32_t*)(uintptr_t)(src_addr + + (uint64_t)((int64_t)threadIdx.x * + head_bytes + + header_off)); + cursor = hdr[0]; + pending = hdr[1]; + } + __syncthreads(); + memcpy_u64_tiled(src_addr, dst_addr, blob_bytes, 0, 1); + __syncthreads(); + if ((int32_t)threadIdx.x < num_heads) { + const int32_t bias = token_bias > 0 ? token_bias : 0; + const int32_t commit = pending > 0 ? min(bias + 1, pending) : 0; + int32_t* hdr = + (int32_t*)(uintptr_t)(dst_addr + + (uint64_t)((int64_t)threadIdx.x * head_bytes + + header_off)); + hdr[0] = cursor + commit; + hdr[1] = 0; + } + return; + } + // Temporal state: the u64 body is partitioned across num_temporal_tiles CTAs // to keep the SMs filled at small batch. const int64_t actual_src_block_id = diff --git a/docs/rdna2/leapquant-gdn-2026-11-13.md b/docs/rdna2/leapquant-gdn-2026-11-13.md new file mode 100644 index 000000000000..fbf5369a6e45 --- /dev/null +++ b/docs/rdna2/leapquant-gdn-2026-11-13.md @@ -0,0 +1,71 @@ +# LeapQuant INT8 GDN recurrent state (opt-in, 2026-11-13) + +Port of LeapQuant ([arXiv 2609.38166](https://arxiv.org/abs/2609.38166), +requested in [vllm#64725](https://github.com/vllm-project/vllm/issues/64725)) +to the RDNA2 Gated DeltaNet path. Opt-in with `VLLM_RDNA_GDN_LEAPQUANT=1` +(gfx10x only; the flag raises elsewhere). Default off: the change is lossy. + +## What it stores + +The FP32 state of one value head (`[V, K] = [128, 128]`, 64 KiB) becomes one +28,768-byte blob (2.28x smaller): + +| section | dtype | shape | bytes | offset | +|---|---|---|---:|---:| +| `z` INT8 residual | int8 | `[V, K]` | 16384 | 0 | +| `bscale` per-row scale | fp32 | `[V]` | 512 | 16384 | +| `csmooth` per-column smoothing | fp32 | `[K]` | 512 | 16896 | +| `comp_k` compensator keys | fp16 | `[r, K]` | 1024 | 17408 | +| `comp_u` compensator values | fp16 | `[r, V]` | 1024 | 18432 | +| `rec_k` token records | fp16 | `[cap, K]` | 4608 | 19456 | +| `rec_u` token records | fp16 | `[cap, V]` | 4608 | 24064 | +| `rec_g` decays | fp32 | `[cap]` | 72 | 28672 | +| header `[cursor, pending, 0, 0]` | int32 | `[4]` | 16 | 28752 | + +`r = 4` compensator tokens, window 16, `cap = window + num_spec = 18` for MTP2. +The state is `replay(cursor records, Z*B*C + comp_u^T comp_k)`: the boundary +state (INT8 + rank-4 FP16 compensator of its top subspace, found by 2 power +iterations) plus the exact FP16 records `S_j = exp(g_j) S_{j-1} + u_j k_j^T` +since the boundary. Every 16 committed tokens the records are folded into a +new boundary state, so quantization error enters once per window instead of +once per token. The FP32 oracle is +`vllm/model_executor/layers/mamba/ops/leapquant_gdn.py`; the kernels are +`csrc/rocm/gdn_leapquant_rdna2.cu`. + +## How it fits the serving path (MTP2, cudagraphs, prefix caching) + +* **Slots are record counts.** The FP32 layout keeps one state per speculative + token (`spec_state_indices[n, t]`). The blob lives at the home block + `spec_state_indices[n, 0]`; "slot `t`" is "commit `t + 1` pending records". + A decode/verify step appends its records and sets `pending`; + `gdn_lq_normalize_rdna2` (first op of the next step) commits + `num_accepted` of them (rejected drafts are dropped by never committing) and + folds once `cursor >= window`. MTP verify needs no per-token state copies. +* **Readers of slot 0** (chunked prefill, non-spec decodes) commit + `min(1, pending)`, the same state the FP32 path reads. +* **Prefill** materializes the blob to an FP32 `[N, HV, V, K]` initial state, + runs the unchanged HIP/Triton chunk kernels, and stores the final state + back (`gdn_lq_store_rdna2`). Inside a chunk nothing changes; LeapQuant error + enters at chunk boundaries (MAXBAT 2048). +* **Prefix caching (`mamba-align`, V2 runner):** the align precopy/postprocess + HIP kernels (`v1_mamba_rdna2.cu`, `copy_mamba_state_block`) copy the whole + blob from the home block and write `cursor + min(bias + 1, pending)` into + the copy's header, which is exactly the FP32 "copy slot `bias`" rule. The V1 + runner's memcpy specs and the Triton precopy fallback raise instead of + silently copying wrong bytes. +* **Cudagraphs:** every op is a fixed-shape launch on the current stream that + reads its metadata from device tensors, so FULL_AND_PIECEWISE captures it + like the FP32 kernels. +* **Refused combinations** (raise at init): mono decode op, fused CUDA spec + decode, AITER/FlashInfer GDN, align mode on the V1 runner, head dims other + than 128. + +## Use + +```bash +bash tools/rdna/serve_rdna.sh RECIPE=flashnext-mtp2 VLLM_RDNA_GDN_LEAPQUANT=1 +``` + +or uncomment `VLLM_RDNA_GDN_LEAPQUANT=1` in +`tools/rdna/recipes/flashnext-mtp2.env`. The worker log prints +`GDN recurrent state: LeapQuant INT8 (...)`. diff --git a/tests/kernels/mamba/test_gdn_leapquant_rdna2.py b/tests/kernels/mamba/test_gdn_leapquant_rdna2.py new file mode 100644 index 000000000000..db1ab484c672 --- /dev/null +++ b/tests/kernels/mamba/test_gdn_leapquant_rdna2.py @@ -0,0 +1,323 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""LeapQuant GDN state kernels (gdn_lq_*_rdna2) against the FP32 oracle in +vllm/model_executor/layers/mamba/ops/leapquant_gdn.py. + +The failure modes guarded here are the ones that corrupt serving silently: +records committed for rejected MTP drafts, folds that drift from the FP32 +trajectory, NULL-block slots that get written, align-mode block copies that +hand out an uncommitted state, and kernels that do not replay inside a CUDA +graph. +""" + +import pytest +import torch + +from vllm.model_executor.layers.mamba.ops import leapquant_gdn as lq + +pytestmark = pytest.mark.skipif( + not torch.cuda.is_available() + or "gfx103" not in getattr(torch.cuda.get_device_properties(0), "gcnArchName", ""), + reason="RDNA2 (gfx103x) GPU required", +) + +K = V = 128 +H, HV = 2, 6 +WINDOW, R, ITERS = 16, 4, 2 + + +def _layout(max_query_len: int) -> lq.LeapQuantLayout: + return lq.LeapQuantLayout( + K, + V, + window=WINDOW, + num_comp=R, + max_query_len=max_query_len, + power_iters=ITERS, + ) + + +def _ops(): + return torch.ops._rocm_C + + +def _normalize(L, blob, idx, num_acc): + _ops().gdn_lq_normalize_rdna2( + blob, + idx, + num_acc, + V, + R, + L.capacity, + WINDOW, + ITERS, + ) + + +def _store(L, state, blob, idx): + _ops().gdn_lq_store_rdna2(state, blob, idx, R, L.capacity, WINDOW, ITERS) + + +def _materialize(L, blob, idx, n): + out = torch.empty(n, HV, V, K, dtype=torch.float32, device=blob.device) + _ops().gdn_lq_materialize_rdna2(blob, out, idx, R, L.capacity, WINDOW) + return out + + +def _params(seed=0): + g = torch.Generator(device="cpu").manual_seed(seed) + A_log = torch.log(torch.rand(HV, generator=g) * 0.99 + 0.01).cuda() + dt_bias = (torch.randn(HV, generator=g) * 0.1).cuda() + return A_log, dt_bias + + +def _tokens(n, t, seed, outlier=None): + g = torch.Generator(device="cpu").manual_seed(seed) + q = torch.randn(n, t, H, K, generator=g) + k = torch.randn(n, t, H, K, generator=g) + v = torch.randn(n, t, HV, V, generator=g) + if outlier is not None: + v = v + outlier + a = torch.randn(n, t, HV, generator=g) - 3 + b = torch.randn(n, t, HV, generator=g) + return [x.half().cuda() for x in (q, k, v, a, b)] + + +def _decode(L, blob, idx, q, k, v, a, b, A_log, dt_bias): + n, t = q.shape[:2] + out = torch.empty(1, n * t, HV, V, dtype=torch.float16, device="cuda") + cu = torch.arange(0, n * t + 1, t, dtype=torch.int32, device="cuda") + _ops().gdn_lq_decode_rdna2( + q.reshape(1, n * t, H, K), + k.reshape(1, n * t, H, K), + v.reshape(1, n * t, HV, V), + a.reshape(n * t, HV), + b.reshape(n * t, HV), + A_log, + dt_bias, + out, + blob, + cu, + idx, + K**-0.5, + R, + L.capacity, + WINDOW, + ) + return out.view(n, t, HV, V) + + +def _rel(a, b): + return ((a.float() - b.float()).norm() / b.float().norm().clamp_min(1e-30)).item() + + +def test_store_materialize_matches_reference(): + L = _layout(1) + n = 3 + torch.manual_seed(0) + # Low-rank outliers plus noise, the structure the compensators target. + S = torch.randn(n, HV, V, K, device="cuda") + S += ( + 40 + * torch.randn(n, HV, V, 1, device="cuda") + * torch.randn(n, HV, 1, K, device="cuda") + ) + blob = torch.zeros(n + 1, HV, L.nbytes, dtype=torch.uint8, device="cuda") + blob.random_(0, 255) # stale bytes must not leak into a fresh store + idx = torch.arange(1, n + 1, dtype=torch.int32, device="cuda") + _store(L, S, blob, idx) + got = _materialize(L, blob, idx, n) + + ref_blob = blob[1:].clone() + lq.leapquant_store_state(L, ref_blob, S) + ref = lq.leapquant_materialize(L, ref_blob) + + assert _rel(got, S) < 5e-3, "HIP fold loses more than INT8 resolution" + assert _rel(got, ref) < 5e-3 + assert torch.isfinite(got).all() + hdr = L.views(blob[1:])["header"] + assert (hdr[..., :2] == 0).all() + + +def test_null_block_untouched_and_zero_output(): + L = _layout(3) + A_log, dt_bias = _params() + blob = torch.zeros(3, HV, L.nbytes, dtype=torch.uint8, device="cuda") + blob[0].fill_(7) + snapshot = blob[0].clone() + idx = torch.tensor([0, 2], dtype=torch.int32, device="cuda") + q, k, v, a, b = _tokens(2, 3, seed=1) + _normalize(L, blob, idx, torch.tensor([1, 1], dtype=torch.int32, device="cuda")) + o = _decode(L, blob, idx, q, k, v, a, b, A_log, dt_bias) + assert torch.equal(blob[0], snapshot) + assert (o[0] == 0).all() + assert o[1].abs().sum() > 0 + + +def test_decode_matches_reference_on_same_blob(): + L = _layout(3) + A_log, dt_bias = _params() + n = 4 + blob = torch.zeros(n + 1, HV, L.nbytes, dtype=torch.uint8, device="cuda") + # Home column of a [n, 1 + num_spec] state-index table: a strided view. + table = torch.zeros(n, 3, dtype=torch.int32, device="cuda") + table[:, 0] = torch.arange(1, n + 1, dtype=torch.int32) + idx = table[:, 0] + S = torch.randn(n, HV, V, K, device="cuda") * 0.5 + _store(L, S, blob, idx.contiguous()) + # Fill part of the window so the replay path is exercised. + for step in range(5): + q, k, v, a, b = _tokens(n, 3, seed=10 + step) + _normalize(L, blob, idx, torch.full((n,), 2, dtype=torch.int32, device="cuda")) + _decode(L, blob, idx, q, k, v, a, b, A_log, dt_bias) + _normalize( + L, blob, idx, torch.tensor([1, 2, 3, 2], dtype=torch.int32, device="cuda") + ) + + ref_blob = blob[1:].clone() + q, k, v, a, b = _tokens(n, 3, seed=99) + got = _decode(L, blob, idx, q, k, v, a, b, A_log, dt_bias) + ref = lq.leapquant_decode(L, ref_blob, q, k, v, a, b, A_log, dt_bias, K**-0.5) + assert _rel(got, ref) < 2e-3 + gv, rv = L.views(blob[1:]), L.views(ref_blob) + assert torch.equal(gv["header"], rv["header"]) + c = int(gv["header"][0, 0, 0]) + assert _rel(gv["rec_u"][..., c : c + 3, :], rv["rec_u"][..., c : c + 3, :]) < 2e-3 + assert _rel(gv["rec_k"][..., c : c + 3, :], rv["rec_k"][..., c : c + 3, :]) < 2e-3 + + +@pytest.mark.parametrize("spec", [False, True]) +def test_trajectory_tracks_fp32_with_rejections(spec): + """Long decode with folds every window; with MTP-style rejections the + committed state must follow only the accepted prefix.""" + T = 3 if spec else 1 + L = _layout(T) + A_log, dt_bias = _params(seed=3) + n = 2 + blob = torch.zeros(n + 1, HV, L.nbytes, dtype=torch.uint8, device="cuda") + idx = torch.arange(1, n + 1, dtype=torch.int32, device="cuda") + S_ref = torch.zeros(n, HV, V, K, device="cuda") + outlier = torch.randn(V) * 4 + gen = torch.Generator(device="cpu").manual_seed(5) + num_acc = None + errs = [] + for step in range(120): + q, k, v, a, b = _tokens(n, T, seed=1000 + step, outlier=outlier) + _normalize(L, blob, idx, num_acc) + o = _decode(L, blob, idx, q, k, v, a, b, A_log, dt_bias) + acc = ( + torch.randint(1, T + 1, (n,), generator=gen) + if spec + else torch.ones(n, dtype=torch.long) + ) + qn = lq.l2norm(q).repeat_interleave(HV // H, 2) * K**-0.5 + kn = lq.l2norm(k).repeat_interleave(HV // H, 2) + g, beta = lq.gdn_gates(a, b, A_log, dt_bias) + for i in range(n): + Sr = S_ref[i] + for t in range(int(acc[i])): + Sr, o_r, _ = lq.gdn_step_fp32( + Sr, qn[i, t], kn[i, t], v[i, t], g[i, t], beta[i, t] + ) + errs.append(_rel(o[i, t], o_r)) + S_ref[i] = Sr + # Non-spec steps read slot 0: num_accepted None commits one record. + num_acc = acc.to(torch.int32).cuda() if spec else None + late = errs[len(errs) // 2 :] + assert max(late) < 3e-2, f"worst late-token rel err {max(late):.3e}" + assert sum(late) / len(late) < 6e-3, ( + f"mean late rel err {sum(late) / len(late):.3e}" + ) + + +def test_normalize_and_decode_replay_in_cuda_graph(): + L = _layout(3) + A_log, dt_bias = _params() + n = 2 + blob = torch.zeros(n + 1, HV, L.nbytes, dtype=torch.uint8, device="cuda") + idx = torch.arange(1, n + 1, dtype=torch.int32, device="cuda") + num_acc = torch.full((n,), 3, dtype=torch.int32, device="cuda") + q, k, v, a, b = _tokens(n, 3, seed=7) + eager_blob = blob.clone() + + def step(bl): + _normalize(L, bl, idx, num_acc) + return _decode(L, bl, idx, q, k, v, a, b, A_log, dt_bias) + + step(blob) # warm up / build kernels outside capture + blob.zero_() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = step(blob) + blob.zero_() + eager = [] + for _ in range(8): # crosses a window boundary: folds inside the graph + graph.replay() + eager.append(step(eager_blob)) + torch.accelerator.synchronize() + assert torch.equal(out, eager[-1]) + assert torch.equal(blob, eager_blob) + + +def _precopy_lq(blob, block_table, dst_col, src_col, token_bias, tiles=4): + """Run the align pre-copy kernel with one LeapQuant state tensor.""" + dev = blob.device + i64 = dict(dtype=torch.int64, device=dev) + i32 = dict(dtype=torch.int32, device=dev) + _ops().precopy_mamba_align_fused_rdna2( + dst_col, + src_col, + token_bias, + torch.tensor([block_table.data_ptr()], **i64), + block_table.stride(0), + torch.tensor([blob.data_ptr()], **i64), + torch.tensor([blob.stride(0)], **i64), + torch.tensor([1], **i32), + torch.tensor([blob[0].numel()], **i64), + torch.tensor([-1], **i32), + torch.tensor([0], **i32), + torch.tensor([blob.shape[1]], **i32), + torch.tensor([blob.shape[2] - lq.LEAPQUANT_HEADER_BYTES], **i64), + None, + dst_col.shape[0], + False, + tiles, + ) + + +def test_align_precopy_commits_accepted_records(): + """Migrating a running home blob to the next block column must produce + exactly the state a verify with num_accepted = token_bias + 1 reads.""" + L = _layout(3) + A_log, dt_bias = _params(seed=11) + n = 2 + blob = torch.zeros(2 * n + 1, HV, L.nbytes, dtype=torch.uint8, device="cuda") + blob[n + 1 :].random_(0, 255) # destination blocks hold stale bytes + block_table = torch.zeros(n, 4, dtype=torch.int32, device="cuda") + block_table[:, 0] = torch.arange(1, n + 1, dtype=torch.int32) + block_table[:, 1] = torch.arange(n + 1, 2 * n + 1, dtype=torch.int32) + home = block_table[:, 0] + _store(L, torch.randn(n, HV, V, K, device="cuda") * 0.5, blob, home.contiguous()) + for step in range(4): + q, k, v, a, b = _tokens(n, 3, seed=40 + step) + _normalize(L, blob, home, torch.full((n,), 3, dtype=torch.int32, device="cuda")) + _decode(L, blob, home, q, k, v, a, b, A_log, dt_bias) + src_before = blob[1 : n + 1].clone() + + bias = torch.tensor([1, 0], dtype=torch.int32, device="cuda") + _precopy_lq( + blob, + block_table, + dst_col=torch.ones(n, dtype=torch.int32, device="cuda"), + src_col=torch.zeros(n, dtype=torch.int32, device="cuda"), + token_bias=bias, + ) + assert torch.equal(blob[1 : n + 1], src_before), "source home block modified" + for i in range(n): + ref = lq.leapquant_copy_slot(L, src_before[i], int(bias[i])) + assert torch.equal(blob[n + 1 + i], ref) + # The migrated blob is the state the verify would have read in place. + expect = src_before.clone() + lq.leapquant_normalize(L, expect, bias.long() + 1) + got = _materialize(L, blob, block_table[:, 1].contiguous(), n) + assert _rel(got, lq.leapquant_materialize(L, expect)) < 1e-5 diff --git a/tools/rdna/port_v031/decode_kl.py b/tools/rdna/port_v031/decode_kl.py new file mode 100755 index 000000000000..f045fe5abc0f --- /dev/null +++ b/tools/rdna/port_v031/decode_kl.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Decode-path top-k KL between served configurations. + +``topk_kl.py`` and ``quality_ab.py`` score prompts, i.e. the prefill path. A +change that only acts across decode steps (a lossy recurrent-state format such +as the LeapQuant GDN state, which a single-chunk prefill never stores) is +invisible there. This tool greedy-generates from fixed passages with the +top-k logprobs of every generated token. Up to the first token where two runs +disagree, both runs decoded the same context, so the per-position KL there is a +decode-path KL; the divergence point itself is reported, and a second run of +the reference configuration (``--floor``) gives the noise floor of both. + + python decode_kl.py collect --url http://127.0.0.1:18096 --model flash-next \\ + --out A.json [--passages 32] [--passage-chars 6000] [--gen-tokens 512] \\ + [--workers 8] + python decode_kl.py compare A.json B.json [--floor A2.json] [--json out.json] + +``--workers`` requests run concurrently, so decode batches hold several +requests as in serving. KL(ref || cand) is taken on the partition of the +vocabulary into the tokens of both top-k lists plus one cell for the rest +(a lower bound on the full KL, as in ``topk_kl.py``). +""" + +import argparse +import glob +import json +import math +import os +import statistics +import sys +import urllib.request +from concurrent.futures import ThreadPoolExecutor + +TREE = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) + + +def post(url: str, body: dict, timeout: float) -> dict: + req = urllib.request.Request( + url, + data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read()) + + +def load_passages(files: list[str], count: int, chars: int) -> list[str]: + text = "" + for path in sorted(files): + with open(path, encoding="utf-8", errors="replace") as handle: + text += handle.read() + "\n\n" + if len(text) < count * chars: + raise SystemExit( + f"need {count * chars} characters of text, the files hold {len(text)}" + ) + stride = len(text) // count + return [text[i * stride : i * stride + chars] for i in range(count)] + + +def token_id(tok: str) -> int: + # return_tokens_as_token_ids renders tokens as "token_id:". + return int(tok.rsplit(":", 1)[1]) + + +def generate(args: argparse.Namespace, prompt: str) -> dict: + body = { + "model": args.model, + "prompt": prompt, + "max_tokens": args.gen_tokens, + "temperature": 0.0, + "seed": 12345, + "ignore_eos": True, + "logprobs": args.top, + "return_tokens_as_token_ids": True, + } + out = post(args.url.rstrip("/") + "/v1/completions", body, args.timeout) + lp = out["choices"][0]["logprobs"] + ids = [token_id(t) for t in lp["tokens"]] + top = [ + {str(token_id(t)): v for t, v in (d or {}).items()} for d in lp["top_logprobs"] + ] + return { + "ids": ids, + "lp": lp["token_logprobs"], + "top": top, + "prompt_tokens": out["usage"]["prompt_tokens"], + } + + +def cmd_collect(args: argparse.Namespace) -> int: + files = args.files or glob.glob( + os.path.join(TREE, "docs", "**", "*.md"), recursive=True + ) + passages = load_passages(files, args.passages, args.passage_chars) + with ThreadPoolExecutor(args.workers) as pool: + runs = list(pool.map(lambda p: generate(args, p), passages)) + meta = { + "url": args.url, + "passages": args.passages, + "passage_chars": args.passage_chars, + "gen_tokens": args.gen_tokens, + "top": args.top, + "workers": args.workers, + } + with open(args.out, "w") as handle: + json.dump({"meta": meta, "runs": runs}, handle) + print( + f"collect: {len(runs)} passages, prompt tokens " + f"{min(r['prompt_tokens'] for r in runs)}-" + f"{max(r['prompt_tokens'] for r in runs)}, {args.gen_tokens} generated each" + ) + return 0 + + +def kl_partition(ref: dict, cand: dict) -> tuple[float, float]: + """KL(ref || cand) on (shared top-k tokens, rest); returns (kl, coverage).""" + shared = [t for t in ref if t in cand] + p = [math.exp(ref[t]) for t in shared] + q = [math.exp(cand[t]) for t in shared] + p_rest = max(1.0 - sum(p), 0.0) + q_rest = max(1.0 - sum(q), 1e-12) + kl = sum(pi * (math.log(pi) - math.log(qi)) for pi, qi in zip(p, q) if pi > 0) + if p_rest > 0: + kl += p_rest * (math.log(p_rest) - math.log(q_rest)) + return max(kl, 0.0), sum(p) + + +def pct(values: list[float], q: float) -> float: + if not values: + return float("nan") + s = sorted(values) + return s[min(len(s) - 1, int(q * (len(s) - 1) + 0.5))] + + +def pair(ref: dict, cand: dict) -> dict: + kls, covs, divergence, flips = [], [], [], 0 + for r, c in zip(ref["runs"], cand["runs"]): + n = min(len(r["ids"]), len(c["ids"])) + d = next((i for i in range(n) if r["ids"][i] != c["ids"][i]), n) + divergence.append(d) + for i in range(d): + kl, cov = kl_partition(r["top"][i], c["top"][i]) + kls.append(kl) + covs.append(cov) + if d < n: + # A divergence at a confident reference position is a real flip, + # not a near-tie: margin of the reference top-1 over its runner-up. + lps = sorted(r["top"][d].values(), reverse=True) + if len(lps) > 1 and lps[0] - lps[1] > 1.0: + flips += 1 + gen = ref["meta"]["gen_tokens"] + return { + "positions": len(kls), + "kl_mean": statistics.fmean(kls) if kls else float("nan"), + "kl_p50": pct(kls, 0.5), + "kl_p99": pct(kls, 0.99), + "kl_max": max(kls) if kls else float("nan"), + "coverage_mean": statistics.fmean(covs) if covs else float("nan"), + "divergence_median": statistics.median(divergence), + "identical": sum(d >= gen for d in divergence), + "passages": len(divergence), + "confident_flips": flips, + } + + +def cmd_compare(args: argparse.Namespace) -> int: + def load(path: str) -> dict: + with open(path) as handle: + return json.load(handle) + + ref, cand = load(args.ref), load(args.cand) + report = {"ref-cand": pair(ref, cand)} + if args.floor: + report["ref-floor"] = pair(ref, load(args.floor)) + for name, s in report.items(): + print( + f"{name}: KL mean {s['kl_mean']:.3e} p50 {s['kl_p50']:.3e} " + f"p99 {s['kl_p99']:.3e} max {s['kl_max']:.3e} over {s['positions']} " + f"decode positions (coverage {s['coverage_mean']:.3f}); divergence " + f"median {s['divergence_median']} tokens, {s['identical']}/" + f"{s['passages']} identical, {s['confident_flips']} confident flips" + ) + if args.json: + with open(args.json, "w") as handle: + json.dump(report, handle, indent=1) + return 0 + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + sub = ap.add_subparsers(dest="cmd", required=True) + c = sub.add_parser("collect") + c.add_argument("--url", required=True) + c.add_argument("--model", required=True) + c.add_argument("--out", required=True) + c.add_argument("--files", nargs="*") + c.add_argument("--passages", type=int, default=32) + c.add_argument("--passage-chars", type=int, default=6000) + c.add_argument("--gen-tokens", type=int, default=512) + c.add_argument("--top", type=int, default=20) + c.add_argument("--workers", type=int, default=8) + c.add_argument("--timeout", type=float, default=1800) + p = sub.add_parser("compare") + p.add_argument("ref") + p.add_argument("cand") + p.add_argument("--floor") + p.add_argument("--json") + args = ap.parse_args() + return cmd_collect(args) if args.cmd == "collect" else cmd_compare(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/rdna/recipes/flashnext-mtp2.env b/tools/rdna/recipes/flashnext-mtp2.env index 023e7b8ad26b..63facabf6116 100644 --- a/tools/rdna/recipes/flashnext-mtp2.env +++ b/tools/rdna/recipes/flashnext-mtp2.env @@ -87,3 +87,6 @@ TUNED_CONFIG=1 # decode gap). See docs/rdna2/mixed-batch-stall-probe.md "Decode-stall cap". # VLLM_RDNA_DECODE_STALL_MS=250 # VLLM_DECODE_STALL_MAX_OVERHEAD=1.0 +# LeapQuant INT8 GDN recurrent state (arXiv 2609.38166): opt-in, lossy. See +# docs/rdna2/leapquant-gdn-2026-11-13.md for the on/off receipt. +# VLLM_RDNA_GDN_LEAPQUANT=1 diff --git a/vllm/model_executor/layers/mamba/gdn/base.py b/vllm/model_executor/layers/mamba/gdn/base.py index 249d56685ee7..3bcd60eb210a 100644 --- a/vllm/model_executor/layers/mamba/gdn/base.py +++ b/vllm/model_executor/layers/mamba/gdn/base.py @@ -15,6 +15,7 @@ from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateDtypeCalculator, ) +from vllm.model_executor.layers.mamba.ops.leapquant_gdn import leapquant_gdn_enabled from vllm.model_executor.models.utils import extract_layer_index from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum @@ -22,6 +23,9 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase): """Base class for GatedDeltaNet attention layer.""" + # Layers whose forward reads LeapQuant blobs (VLLM_RDNA_GDN_LEAPQUANT=1). + supports_leapquant_state: bool = False + def __init__( self, config: PreTrainedConfig, @@ -45,6 +49,11 @@ def __init__( if self.speculative_config else 0 ) + if leapquant_gdn_enabled() and not self.supports_leapquant_state: + raise ValueError( + f"VLLM_RDNA_GDN_LEAPQUANT=1 is not implemented for " + f"{type(self).__name__}" + ) @property def mamba_type(self) -> MambaAttentionBackendEnum: diff --git a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py index 689b69a5caca..6d316ec6fdc8 100644 --- a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @@ -51,6 +51,11 @@ causal_conv1d_fn, causal_conv1d_update, ) +from vllm.model_executor.layers.mamba.ops.leapquant_gdn import ( + LeapQuantLayout, + leapquant_gdn_enabled, + leapquant_gdn_layout, +) from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig @@ -769,6 +774,8 @@ def forward_aiter_flydsl( @PluggableLayer.register("qwen_gated_delta_net_attention") class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): + supports_leapquant_state = True + def get_state_shape( self, ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[int, ...]]: @@ -985,12 +992,48 @@ def __init__( self._mono_gdn_op: Any self._mono_gdn: Any self._mono_gdn_op, self._mono_gdn = mono if mono else (None, None) + self._lq_layout = self._init_leapquant(vllm_config) compilation_config = get_current_vllm_config().compilation_config if prefix in compilation_config.static_forward_context: raise ValueError(f"Duplicate layer name: {prefix}") compilation_config.static_forward_context[prefix] = self + def _init_leapquant(self, vllm_config: VllmConfig) -> LeapQuantLayout | None: + """LeapQuant GDN state (VLLM_RDNA_GDN_LEAPQUANT=1): the recurrent + state is a uint8 blob per (block, head) read by the gdn_lq_* kernels + on every path that reads or writes it here (spec verify, non-spec + decode, prefill).""" + if not leapquant_gdn_enabled(): + return None + if self._mono_gdn_op is not None or self.enable_fused_gdn_spec_decode: + raise ValueError( + "VLLM_RDNA_GDN_LEAPQUANT=1 does not cover the GDN monokernel or " + "the fused CUDA spec-decode path" + ) + if GDN_AITER_TRITON_AVAILABLE or self._checkpoint_alignment is not None: + raise ValueError( + "VLLM_RDNA_GDN_LEAPQUANT=1 does not cover the AITER GDN path " + "or FlashInfer prefill checkpoints" + ) + if ( + vllm_config.cache_config.mamba_cache_mode == "align" + and not vllm_config.use_v2_model_runner + ): + raise ValueError( + "VLLM_RDNA_GDN_LEAPQUANT=1 with mamba align prefix caching " + "needs the V2 model runner (its fused align copies commit " + "LeapQuant records)" + ) + layout = leapquant_gdn_layout(self.head_k_dim, self.head_v_dim, self.num_spec) + logger.info_once( + "GDN recurrent state: LeapQuant INT8 (window %d, r %d, %d B/head)", + layout.window, + layout.num_comp, + layout.nbytes, + ) + return layout + def _init_gdn_state_arenas(self) -> None: """Permanent arenas on the layer device, before first BeginCapture.""" conv_shape, ssm_shape = self.get_state_shape()[:2] @@ -2140,6 +2183,63 @@ def _forward_core_rocm( core_attn_out=core_attn_out, ) + def _lq_recurrent( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + a: torch.Tensor, + b: torch.Tensor, + blob: torch.Tensor, + cu_seqlens: torch.Tensor, + state_indices: torch.Tensor, + num_accepted: torch.Tensor | None, + out: torch.Tensor | None = None, + ) -> torch.Tensor: + """Decode / MTP verify on LeapQuant blobs. + + Commits the previous step's accepted records (``num_accepted``, or one + for slot-0 readers), then runs each request's tokens from its home + block ``state_indices`` and appends their records.""" + layout = self._lq_layout + assert layout is not None + num_reqs = cu_seqlens.shape[0] - 1 + state_indices = state_indices[:num_reqs] + torch.ops._rocm_C.gdn_lq_normalize_rdna2( + blob, + state_indices, + num_accepted, + layout.head_v_dim, + layout.num_comp, + layout.capacity, + layout.window, + layout.power_iters, + ) + if out is None: + out = torch.zeros( + (1, q.shape[1], v.shape[2], v.shape[3]), + dtype=q.dtype, + device=q.device, + ) + torch.ops._rocm_C.gdn_lq_decode_rdna2( + q, + k, + v, + a, + b, + self.A_log, + self.dt_bias, + out, + blob, + cu_seqlens, + state_indices, + self.head_k_dim**-0.5, + layout.num_comp, + layout.capacity, + layout.window, + ) + return out + def _forward_core( self, mixed_qkv: torch.Tensor, @@ -2171,6 +2271,7 @@ def _forward_core( if ( self.enable_packed_recurrent_decode + and self._lq_layout is None and attn_metadata.spec_sequence_masks is None and attn_metadata.num_prefills == 0 and attn_metadata.num_decodes > 0 @@ -2421,7 +2522,29 @@ def _forward_core( and core_attn_out.dtype == query_spec.dtype and core_attn_out.is_contiguous() ) - if spec_sequence_masks is not None: + lq = self._lq_layout + if spec_sequence_masks is not None and lq is not None: + assert spec_state_indices_tensor is not None + core_attn_out_spec = self._lq_recurrent( + query_spec, + key_spec, + value_spec, + a_spec, + b_spec, + ssm_state, + cu_seqlens=spec_query_start_loc[ # type: ignore[index] + : attn_metadata.num_spec_decodes + 1 + ], + state_indices=spec_state_indices_tensor[:, 0], + num_accepted=num_accepted_tokens, + out=( + core_attn_out[:num_actual_tokens].unsqueeze(0) + if spec_direct + else None + ), + ) + last_recurrent_state = None + elif spec_sequence_masks is not None: core_attn_out_spec, last_recurrent_state = ( fused_sigmoid_gating_delta_rule_update( A_log=self.A_log, @@ -2455,22 +2578,36 @@ def _forward_core( query_decode, key_decode, value_decode = split_qkv( mixed_qkv_non_spec[:num_decode_tokens] # type: ignore[index] ) - core_attn_out_decode, _ = fused_sigmoid_gating_delta_rule_update( - A_log=self.A_log, - a=a[:num_decode_tokens], - b=b[:num_decode_tokens], - dt_bias=self.dt_bias, - q=query_decode, - k=key_decode, - v=value_decode, - initial_state=ssm_state, - inplace_final_state=True, - cu_seqlens=non_spec_query_start_loc[ # type: ignore[index] - : attn_metadata.num_decodes + 1 - ], - ssm_state_indices=non_spec_state_indices_tensor, - use_qk_l2norm_in_kernel=True, - ) + decode_cu_seqlens = non_spec_query_start_loc[ # type: ignore[index] + : attn_metadata.num_decodes + 1 + ] + if lq is not None: + core_attn_out_decode = self._lq_recurrent( + query_decode, + key_decode, + value_decode, + a[:num_decode_tokens], + b[:num_decode_tokens], + ssm_state, + cu_seqlens=decode_cu_seqlens, + state_indices=non_spec_state_indices_tensor, # type: ignore[arg-type] + num_accepted=None, + ) + else: + core_attn_out_decode, _ = fused_sigmoid_gating_delta_rule_update( + A_log=self.A_log, + a=a[:num_decode_tokens], + b=b[:num_decode_tokens], + dt_bias=self.dt_bias, + q=query_decode, + k=key_decode, + v=value_decode, + initial_state=ssm_state, + inplace_final_state=True, + cu_seqlens=decode_cu_seqlens, + ssm_state_indices=non_spec_state_indices_tensor, + use_qk_l2norm_in_kernel=True, + ) else: core_attn_out_decode = None @@ -2484,7 +2621,39 @@ def _forward_core( prefill_has_initial_state = attn_metadata.prefill_has_initial_state assert prefill_state_indices is not None assert prefill_has_initial_state is not None - initial_state = ssm_state[prefill_state_indices] + if lq is not None: + # Commit the slot-0 record a previous decode left pending, then + # expand the blobs into the FP32 initial state of the chunk. + torch.ops._rocm_C.gdn_lq_normalize_rdna2( + ssm_state, + prefill_state_indices, + None, + lq.head_v_dim, + lq.num_comp, + lq.capacity, + lq.window, + lq.power_iters, + ) + initial_state = torch.empty( + ( + prefill_state_indices.shape[0], + ssm_state.shape[1], + lq.head_v_dim, + lq.head_k_dim, + ), + dtype=torch.float32, + device=ssm_state.device, + ) + torch.ops._rocm_C.gdn_lq_materialize_rdna2( + ssm_state, + initial_state, + prefill_state_indices, + lq.num_comp, + lq.capacity, + lq.window, + ) + else: + initial_state = ssm_state[prefill_state_indices] initial_state[~prefill_has_initial_state, ...] = 0 with _GdnPrefillTimer("chain"): plan = attn_metadata.flashinfer_checkpoint_plan @@ -2557,7 +2726,20 @@ def _forward_core( recurrent_rows=plan.rows, ) # Init cache - ssm_state[prefill_state_indices] = last_recurrent_state.to(ssm_state.dtype) + if lq is not None: + torch.ops._rocm_C.gdn_lq_store_rdna2( + last_recurrent_state.float().contiguous(), + ssm_state, + prefill_state_indices, + lq.num_comp, + lq.capacity, + lq.window, + lq.power_iters, + ) + else: + ssm_state[prefill_state_indices] = last_recurrent_state.to( + ssm_state.dtype + ) if split_non_spec: # Stitch the peeled decode outputs in front of the prefill @@ -2565,6 +2747,21 @@ def _forward_core( core_attn_out_non_spec = torch.cat( [core_attn_out_decode, core_attn_out_non_spec], dim=1 ) + elif attn_metadata.num_decodes > 0 and lq is not None: + core_attn_out_non_spec = self._lq_recurrent( + query_non_spec, + key_non_spec, + value_non_spec, + a, + b, + ssm_state, + cu_seqlens=non_spec_query_start_loc[ # type: ignore[index] + : attn_metadata.num_decodes + 1 + ], + state_indices=non_spec_state_indices_tensor, # type: ignore[arg-type] + num_accepted=None, + ) + last_recurrent_state = None elif attn_metadata.num_decodes > 0: core_attn_out_non_spec, last_recurrent_state = ( fused_sigmoid_gating_delta_rule_update( diff --git a/vllm/model_executor/layers/mamba/mamba_utils.py b/vllm/model_executor/layers/mamba/mamba_utils.py index 0ede48005af3..8d48011eb36a 100644 --- a/vllm/model_executor/layers/mamba/mamba_utils.py +++ b/vllm/model_executor/layers/mamba/mamba_utils.py @@ -14,6 +14,10 @@ from vllm.config.model import ModelDType from vllm.distributed import divide from vllm.logger import init_logger +from vllm.model_executor.layers.mamba.ops.leapquant_gdn import ( + leapquant_gdn_enabled, + leapquant_gdn_layout, +) from vllm.utils.torch_utils import ( STR_DTYPE_TO_TORCH_DTYPE, get_kv_cache_torch_dtype, @@ -125,9 +129,13 @@ def gated_delta_net_state_dtype( mamba_cache_dtype: MambaDType, mamba_ssm_cache_dtype: MambaDType = "auto", ) -> tuple[torch.dtype, torch.dtype]: - return cls._mamba_state_dtype( + conv_dtype, ssm_dtype = cls._mamba_state_dtype( model_dtype, mamba_cache_dtype, mamba_ssm_cache_dtype ) + if leapquant_gdn_enabled(): + # LeapQuant byte blobs replace the FP32 recurrent state. + ssm_dtype = torch.uint8 + return conv_dtype, ssm_dtype @classmethod def kda_state_dtype( @@ -285,6 +293,13 @@ def gated_delta_net_state_shape( conv_kernel_size - 1 + num_spec, ) + if leapquant_gdn_enabled(): + layout = leapquant_gdn_layout(head_k_dim, head_v_dim, num_spec) + return conv_state_shape, ( + divide(num_v_heads, tp_world_size), + layout.nbytes, + ) + temporal_state_shape = ( divide(num_v_heads, tp_world_size), head_v_dim, @@ -394,6 +409,24 @@ def get_temporal_copy_spec( ) +def get_leapquant_copy_spec( + state: torch.Tensor, + block_ids: list[int], + cur_block_idx: int, + num_accepted_tokens: int, +) -> MambaCopySpec: + """Marker copy func of LeapQuant GDN blobs. + + Copying slot ``num_accepted_tokens - 1`` must commit that many pending + records in the copy's header, which only the fused RDNA2 align kernels do + (``copy_mamba_state_block`` in ``csrc/rocm/v1_mamba_rdna2.cu``); a plain + memcpy would hand out the uncommitted state.""" + raise NotImplementedError( + "VLLM_RDNA_GDN_LEAPQUANT=1 state copies run in the fused RDNA2 align " + "kernels of the V2 model runner" + ) + + class MambaStateCopyFuncCalculator: @classmethod def linear_attention_state_copy_func(cls): @@ -413,6 +446,8 @@ def short_conv_state_copy_func(cls): @classmethod def gated_delta_net_state_copy_func(cls): + if leapquant_gdn_enabled(): + return (get_conv_copy_spec, get_leapquant_copy_spec) return (get_conv_copy_spec, get_temporal_copy_spec) @classmethod diff --git a/vllm/model_executor/layers/mamba/ops/leapquant_gdn.py b/vllm/model_executor/layers/mamba/ops/leapquant_gdn.py new file mode 100644 index 000000000000..e327e475b5b7 --- /dev/null +++ b/vllm/model_executor/layers/mamba/ops/leapquant_gdn.py @@ -0,0 +1,375 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""LeapQuant recurrent-state storage for Gated DeltaNet (arXiv 2609.38166). + +The FP32 GDN state ``S`` of one value head (``[V, K]``, the row-major layout of +the ROCm decode kernels) is stored as one byte blob per (cache block, head): + +* an INT8 residual ``Z`` with per-value-row scales ``B`` and per-key-column + smoothing ``C`` (dequant: ``Z[v, k] * B[v] * C[k]``), +* ``r`` FP16 compensator tokens ``S ~ U~ K~^T`` (the top-``r`` subspace of the + state at the last window boundary), +* a ring of FP16 per-token records ``(k_j, u_j, g_j)`` for the tokens since the + boundary, where ``S_j = exp(g_j) S_{j-1} + u_j k_j^T``, +* an int32 header ``[cursor, pending, 0, 0]``. + +The state is ``replay(cursor records, Z + U~ K~^T)``. A decode step appends its +records at ``cursor`` and sets ``pending`` to the number of tokens it ran. +Pending records play the role of the per-token state slots of the FP32 layout: +slot ``t`` of the home block is "commit ``t + 1`` records". +``leapquant_normalize`` (run first in the next step) commits the accepted ones +(``cursor += num_accepted``, or one record for a reader of slot 0) and, once +``cursor >= window``, folds the records into a new quantized boundary state. +Rejected speculative tokens are dropped by never committing their records, so +MTP verify needs no per-token state copies. The align-mode prefix-cache copies +(``v1_mamba_rdna2.cu``) apply the same rule: copying slot ``bias`` copies the +home blob and commits ``bias + 1`` records in the copy's header. An all-zero +blob is the zero state. + +This module holds the layout, the opt-in switch and an FP32 PyTorch reference +used as the oracle for the ROCm kernels; the reference is not on the serving +path. +""" + +from __future__ import annotations + +import functools +import os +from dataclasses import dataclass + +import torch + +LEAPQUANT_ALIGN = 16 +# The int32 header [cursor, pending, 0, 0] is the last section of every blob. +LEAPQUANT_HEADER_BYTES = 16 +LEAPQUANT_WINDOW = 16 +LEAPQUANT_NUM_COMP = 4 +LEAPQUANT_POWER_ITERS = 2 +INT8_MAX = 127.0 +SMOOTH_FLOOR = 1e-6 +ORTH_EPS = 1e-12 +FP16_MAX = 65504.0 + + +@functools.cache +def leapquant_gdn_enabled() -> bool: + """Opt-in LeapQuant GDN state storage (``VLLM_RDNA_GDN_LEAPQUANT=1``). + + The kernels are RDNA2 HIP only, so the flag fails loudly elsewhere instead + of silently serving FP32 state.""" + if os.environ.get("VLLM_RDNA_GDN_LEAPQUANT", "0") != "1": + return False + from vllm.platforms.rdna import on_rdna2_hip + + if not on_rdna2_hip(): + raise ValueError( + "VLLM_RDNA_GDN_LEAPQUANT=1 needs gfx10x: the LeapQuant GDN " + "kernels are RDNA2 HIP only" + ) + return True + + +def leapquant_gdn_layout( + head_k_dim: int, head_v_dim: int, num_spec: int +) -> LeapQuantLayout: + """Serving layout: window 16, r = 4 (paper, 8-bit), ring sized for one + verify step of ``num_spec + 1`` tokens.""" + if head_k_dim != 128 or head_v_dim != 128: + raise ValueError( + "VLLM_RDNA_GDN_LEAPQUANT=1 supports head_k_dim == head_v_dim == 128, " + f"got {head_k_dim} / {head_v_dim}" + ) + return LeapQuantLayout( + head_k_dim=head_k_dim, + head_v_dim=head_v_dim, + window=LEAPQUANT_WINDOW, + num_comp=LEAPQUANT_NUM_COMP, + max_query_len=num_spec + 1, + power_iters=LEAPQUANT_POWER_ITERS, + ) + + +def _align(n: int) -> int: + return (n + LEAPQUANT_ALIGN - 1) // LEAPQUANT_ALIGN * LEAPQUANT_ALIGN + + +@dataclass(frozen=True) +class LeapQuantLayout: + """Byte layout of one (block, value head) LeapQuant blob.""" + + head_k_dim: int + head_v_dim: int + window: int = 16 + num_comp: int = 4 + max_query_len: int = 1 # 1 + num_speculative_tokens + power_iters: int = 2 + + @property + def capacity(self) -> int: + # normalize leaves cursor <= window - 1; one step appends at most + # max_query_len records. + return self.window + self.max_query_len - 1 + + def _sections(self) -> list[tuple[str, torch.dtype, tuple[int, ...]]]: + K, V, r, cap = self.head_k_dim, self.head_v_dim, self.num_comp, self.capacity + return [ + ("z", torch.int8, (V, K)), + ("bscale", torch.float32, (V,)), + ("csmooth", torch.float32, (K,)), + ("comp_k", torch.float16, (r, K)), + ("comp_u", torch.float16, (r, V)), + ("rec_k", torch.float16, (cap, K)), + ("rec_u", torch.float16, (cap, V)), + ("rec_g", torch.float32, (cap,)), + ("header", torch.int32, (4,)), + ] + + @property + def offsets(self) -> dict[str, int]: + out, off = {}, 0 + for name, dtype, shape in self._sections(): + out[name] = off + n = dtype.itemsize + for s in shape: + n *= s + off = _align(off + n) + out["_total"] = off + return out + + @property + def nbytes(self) -> int: + return self.offsets["_total"] + + def views(self, blob: torch.Tensor) -> dict[str, torch.Tensor]: + """Typed views of ``blob`` (uint8, ``[..., nbytes]``, last dim dense).""" + assert blob.dtype == torch.uint8 and blob.shape[-1] == self.nbytes + assert blob.stride(-1) == 1 + offs = self.offsets + lead = blob.shape[:-1] + out = {} + for name, dtype, shape in self._sections(): + n = dtype.itemsize + for s in shape: + n *= s + sl = blob[..., offs[name] : offs[name] + n] + out[name] = sl.view(dtype).view(*lead, *shape) + return out + + +def gdn_gates( + a: torch.Tensor, b: torch.Tensor, A_log: torch.Tensor, dt_bias: torch.Tensor +) -> tuple[torch.Tensor, torch.Tensor]: + """Per-token log decay ``g`` and write strength ``beta`` (fp32), matching + ``gdn_spec_verify_rdna2`` / ``gdn_lq_decode_rdna2``.""" + x = a.float() + dt_bias.float() + softplus = torch.where(x <= 20.0, torch.log1p(torch.exp(x)), x) + g = -torch.exp(A_log.float()) * softplus + beta = torch.sigmoid(b.float()) + return g, beta + + +def l2norm(x: torch.Tensor, eps: float = 1e-6) -> torch.Tensor: + x = x.float() + return x / torch.sqrt((x * x).sum(-1, keepdim=True) + eps) + + +def gdn_step_fp32( + S: torch.Tensor, # [..., V, K] fp32 + q: torch.Tensor, # [..., K] l2-normalized and scaled + k: torch.Tensor, # [..., K] l2-normalized + v: torch.Tensor, # [..., V] + g: torch.Tensor, # [...] + beta: torch.Tensor, # [...] +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """One FP32 GDN token: returns (new S, output, record u).""" + S = S * torch.exp(g)[..., None, None] + acc = (S * k[..., None, :]).sum(-1) + u = (v.float() - acc) * beta[..., None] + S = S + u[..., :, None] * k[..., None, :] + o = (S * q[..., None, :]).sum(-1) + return S, o, u + + +def _boundary_state(views: dict[str, torch.Tensor]) -> torch.Tensor: + z = views["z"].float() + S = z * views["bscale"][..., :, None] * views["csmooth"][..., None, :] + cu = views["comp_u"].float() # [..., r, V] + ck = views["comp_k"].float() # [..., r, K] + return S + torch.einsum("...rv,...rk->...vk", cu, ck) + + +def leapquant_materialize(layout: LeapQuantLayout, blob: torch.Tensor) -> torch.Tensor: + """FP32 state ``[..., V, K]`` of normalized blobs (pending must be 0).""" + vw = layout.views(blob) + assert bool((vw["header"][..., 1] == 0).all()), "materialize needs pending == 0" + S = _boundary_state(vw) + cursor = vw["header"][..., 0] + rk, ru, rg = vw["rec_k"].float(), vw["rec_u"].float(), vw["rec_g"] + for j in range(int(cursor.max().item()) if cursor.numel() else 0): + live = (j < cursor)[..., None, None] + upd = ( + S * torch.exp(rg[..., j])[..., None, None] + + ru[..., j, :, None] * rk[..., j, None, :] + ) + S = torch.where(live, upd, S) + return S + + +def _orthonormalize(Y: torch.Tensor) -> torch.Tensor: + """Modified Gram-Schmidt over the last dim's columns: Y [..., N, r].""" + cols: list[torch.Tensor] = [] + for i in range(Y.shape[-1]): + y = Y[..., i] + for c in cols: + y = y - (c * y).sum(-1, keepdim=True) * c + n = torch.sqrt((y * y).sum(-1, keepdim=True)) + cols.append(torch.where(n > ORTH_EPS, y / n.clamp_min(ORTH_EPS), 0.0)) + return torch.stack(cols, dim=-1) + + +def _power_init(layout: LeapQuantLayout, device) -> torch.Tensor: + """Deterministic start block X0 [K, r] (integers in [-8, 8]).""" + K, r = layout.head_k_dim, layout.num_comp + k = torch.arange(K, device=device)[:, None] + i = torch.arange(r, device=device)[None, :] + return (((k * (2 * i + 3) + 7 * i + 1) % 17) - 8).float() + + +def leapquant_fold_parts( + layout: LeapQuantLayout, S: torch.Tensor, prev_comp_k: torch.Tensor | None +) -> dict[str, torch.Tensor]: + """Quantize FP32 states ``S [..., V, K]`` into boundary parts. + + Compensators come from subspace iteration warm-started at the previous + boundary's ``comp_k`` (or a fixed block when it is all zero); the residual + is taken against the fp16-rounded compensators, smoothed per key column, + and quantized per value row to symmetric INT8. + """ + S = S.float() + X0 = _power_init(layout, S.device).expand(*S.shape[:-2], -1, -1) + if prev_comp_k is not None: + pk = prev_comp_k.float().transpose(-1, -2) # [..., K, r] + use_prev = (pk.abs().sum((-1, -2)) > 0)[..., None, None] + X0 = torch.where(use_prev, pk, X0) + Q = _orthonormalize(S @ X0) # [..., V, r] + for _ in range(layout.power_iters): + Q = _orthonormalize(S @ (S.transpose(-1, -2) @ Q)) + Kt = S.transpose(-1, -2) @ Q # [..., K, r] + comp_u = Q.transpose(-1, -2).clamp(-FP16_MAX, FP16_MAX).half() # [..., r, V] + comp_k = Kt.transpose(-1, -2).clamp(-FP16_MAX, FP16_MAX).half() # [..., r, K] + R = S - torch.einsum("...rv,...rk->...vk", comp_u.float(), comp_k.float()) + csmooth = torch.sqrt(R.abs().mean(-2)).clamp_min(SMOOTH_FLOOR) # [..., K] + Rs = R / csmooth[..., None, :] + bscale = Rs.abs().amax(-1) / INT8_MAX # [..., V] + safe = torch.where(bscale > 0, bscale, torch.ones_like(bscale)) + z = torch.round(Rs / safe[..., None]).clamp(-INT8_MAX, INT8_MAX) + z = torch.where((bscale > 0)[..., None], z, torch.zeros_like(z)) + return { + "z": z.to(torch.int8), + "bscale": bscale, + "csmooth": csmooth, + "comp_k": comp_k, + "comp_u": comp_u, + } + + +def leapquant_store_state( + layout: LeapQuantLayout, blob: torch.Tensor, S: torch.Tensor, warm: bool = False +) -> None: + """Fold FP32 states into ``blob`` in place (cursor = pending = 0). + + ``warm`` starts the subspace iteration from the blob's previous + compensators (a live window boundary); a freshly allocated block holds + stale bytes, so prefill stores start from the fixed block.""" + vw = layout.views(blob) + parts = leapquant_fold_parts(layout, S, vw["comp_k"] if warm else None) + for name, val in parts.items(): + vw[name].copy_(val) + vw["header"].zero_() + + +def leapquant_normalize( + layout: LeapQuantLayout, + blob: torch.Tensor, # [N, HV, nbytes] + num_accepted: torch.Tensor | None, # [N] int, None = slot-0 reader +) -> None: + """Commit accepted records and fold full windows, in place. + + ``num_accepted`` is the per-request count a speculative verify reads the + state at (slot ``num_accepted - 1``); ``None`` is a reader of slot 0 + (prefill, non-spec decode), which commits one pending record.""" + vw = layout.views(blob) + hdr = vw["header"] + pending = hdr[..., 1] + if num_accepted is None: + acc = torch.clamp(pending, max=1) + else: + acc = torch.minimum( + num_accepted.to(torch.int32)[:, None].expand_as(pending), pending + ) + hdr[..., 0] += acc + hdr[..., 1] = 0 + full = hdr[..., 0] >= layout.window # [N, HV] + if bool(full.any()): + idx = full.nonzero(as_tuple=True) + sub = blob[idx] + S = leapquant_materialize(layout, sub) + leapquant_store_state(layout, sub, S, warm=True) + blob[idx] = sub + + +def leapquant_copy_slot( + layout: LeapQuantLayout, src: torch.Tensor, token_bias: int +) -> torch.Tensor: + """Align-mode copy of per-token slot ``token_bias`` from a home blob + ``[HV, nbytes]``: the copy commits ``token_bias + 1`` pending records + (mirrors the LeapQuant branch of ``copy_mamba_state_block``).""" + dst = src.clone() + hdr = layout.views(dst)["header"] + pending = hdr[..., 1] + commit = torch.where( + pending > 0, torch.clamp(pending, max=max(token_bias, 0) + 1), 0 + ) + hdr[..., 0] += commit + hdr[..., 1] = 0 + return dst + + +def leapquant_decode( + layout: LeapQuantLayout, + blob: torch.Tensor, # [N, HV, nbytes], normalized + q: torch.Tensor, # [N, T, H, K] raw + k: torch.Tensor, # [N, T, H, K] raw + v: torch.Tensor, # [N, T, HV, V] + a: torch.Tensor, # [N, T, HV] + b: torch.Tensor, # [N, T, HV] + A_log: torch.Tensor, + dt_bias: torch.Tensor, + scale: float, +) -> torch.Tensor: + """Run T tokens per request on normalized blobs; returns o [N, T, HV, V]. + + Appends T records at ``cursor`` and sets ``pending = T``; the next + ``leapquant_normalize`` commits ``num_accepted`` of them. + """ + N, T, H, K = q.shape + HV = v.shape[2] + rep = HV // H + vw = layout.views(blob) + S = leapquant_materialize(layout, blob) # [N, HV, V, K] + qn = l2norm(q).repeat_interleave(rep, dim=2) * scale + kn = l2norm(k).repeat_interleave(rep, dim=2) + g, beta = gdn_gates(a, b, A_log, dt_bias) + cursor = vw["header"][..., 0].long() # [N, HV] + outs = [] + n_idx = torch.arange(N, device=blob.device)[:, None] + h_idx = torch.arange(HV, device=blob.device)[None, :] + for t in range(T): + S, o, u = gdn_step_fp32(S, qn[:, t], kn[:, t], v[:, t], g[:, t], beta[:, t]) + outs.append(o) + pos = cursor + t + vw["rec_k"][n_idx, h_idx, pos] = kn[:, t].half() + vw["rec_u"][n_idx, h_idx, pos] = u.half() + vw["rec_g"][n_idx, h_idx, pos] = g[:, t].float() + vw["header"][..., 1] = T + return torch.stack(outs, dim=1) diff --git a/vllm/v1/worker/mamba_utils.py b/vllm/v1/worker/mamba_utils.py index 379d0e40c8c0..e87f7dcd0c3f 100644 --- a/vllm/v1/worker/mamba_utils.py +++ b/vllm/v1/worker/mamba_utils.py @@ -12,9 +12,11 @@ from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateCopyFuncsByType, get_conv_copy_spec, + get_leapquant_copy_spec, get_temporal_copy_spec, is_conv_state_dim_first, ) +from vllm.model_executor.layers.mamba.ops.leapquant_gdn import LEAPQUANT_HEADER_BYTES from vllm.platforms.rdna import on_rdna2_hip from vllm.triton_utils import tl, triton from vllm.utils.gpu_sync_debug import gpu_sync_allowed @@ -866,6 +868,8 @@ class MambaSpecDecodeGPUContext: # Flag to track if metadata has been populated is_initialized: bool = False + # LeapQuant GDN blobs need the HIP copy (it commits pending records). + has_leapquant_states: bool = False @classmethod def create( @@ -1060,11 +1064,34 @@ def _populate_metadata( # Element size self.state_elem_sizes[idx] = state.element_size() - assert ( - copy_func is get_conv_copy_spec - or copy_func is get_temporal_copy_spec + assert copy_func in ( + get_conv_copy_spec, + get_temporal_copy_spec, + get_leapquant_copy_spec, ), f"unexpected copy func: {copy_func}" - if copy_func is get_conv_copy_spec: + if copy_func is get_leapquant_copy_spec: + # LeapQuant GDN blob [blocks, heads, bytes]: the HIP + # copy commits pending records in the copy's header + # (the last LEAPQUANT_HEADER_BYTES of each head). + if not _ON_RDNA2: + raise ValueError( + "LeapQuant GDN state copies need the RDNA2 " + "HIP mamba copy kernels" + ) + if state.dim() != 3 or state.dtype != torch.uint8: + raise ValueError( + "Expected a uint8 [blocks, heads, bytes] " + f"LeapQuant state, got {tuple(state.shape)} " + f"{state.dtype}" + ) + self.state_conv_widths[idx] = -1 + self.state_inner_sizes[idx] = state[0].numel() + self.has_leapquant_states = True + self.state_dim_row_count[idx] = state.size(1) + self.state_dim_row_stride[idx] = ( + state.size(2) - LEAPQUANT_HEADER_BYTES + ) + elif copy_func is get_conv_copy_spec: if state.dim() != 3: raise ValueError( "Expected 3D conv state cache, got " @@ -1186,7 +1213,9 @@ def bind_batch_block_tables(self, block_tables: list[torch.Tensor]) -> None: if key == self._batch_block_table_key: return self._batch_block_table_key, self.batch_block_table_stride_req = ( - self._write_block_table_ptrs(block_tables, self.aligned_index_block_table_ptrs) + self._write_block_table_ptrs( + block_tables, self.aligned_index_block_table_ptrs + ) ) def compute_aligned_state_indices( @@ -1371,6 +1400,11 @@ def run_fused_precopy( ) return + if self.has_leapquant_states: + raise RuntimeError( + "VLLM_RDNA_GDN_LEAPQUANT=1 does not support dynamic speculation " + "(zero-draft collapse takes the Triton align copy)" + ) grid = (num_reqs, total_states, _TEMPORAL_TILES) precopy_mamba_align_fused_kernel[grid]( state_idx_gpu, From f6b0bac4186f47f5101351099f809c678ff1d444 Mon Sep 17 00:00:00 2001 From: BlivionIaG Date: Fri, 13 Nov 2026 00:48:40 +0100 Subject: [PATCH 02/18] feat(rdna2): LeapQuant GDN state under the MonoKernel mono_gdn_lq_rdna2 is the fused mono GDN launch (conv + delta rule + gated norm) with the LeapQuant blob load/append in place of the FP32 state load/store; gdn_lq_normalize_rdna2 commits the accepted records first. The flag no longer refuses --monokernel, and the FP32 Triton decode warmup is skipped under the flag (it never runs there). Shared blob layout moves to gdn_leapquant_layout.cuh. decode_kl.py accepts a /v1/completions URL. Signed-off-by: BlivionIaG --- csrc/rocm/gdn_leapquant_layout.cuh | 67 ++++++ csrc/rocm/gdn_leapquant_rdna2.cu | 60 +---- csrc/rocm/mono_gdn_rdna2.cu | 211 +++++++++++++++--- csrc/rocm/ops.h | 10 + csrc/rocm/torch_bindings.cpp | 7 + docs/rdna2/leapquant-gdn-2026-11-13.md | 21 +- .../kernels/mamba/test_gdn_leapquant_rdna2.py | 96 ++++++++ tools/rdna/port_v031/decode_kl.py | 7 +- .../layers/mamba/gdn/qwen_gdn_linear_attn.py | 45 ++-- vllm/models/qwen4_exp/amd/mono/rdna2/gdn.py | 18 +- 10 files changed, 428 insertions(+), 114 deletions(-) create mode 100644 csrc/rocm/gdn_leapquant_layout.cuh diff --git a/csrc/rocm/gdn_leapquant_layout.cuh b/csrc/rocm/gdn_leapquant_layout.cuh new file mode 100644 index 000000000000..8b8520f0eed1 --- /dev/null +++ b/csrc/rocm/gdn_leapquant_layout.cuh @@ -0,0 +1,67 @@ +// LeapQuant GDN blob layout shared by gdn_leapquant_rdna2.cu and the +// MonoKernel GDN (mono_gdn_rdna2.cu). Must match LeapQuantLayout.offsets in +// vllm/model_executor/layers/mamba/ops/leapquant_gdn.py. +#pragma once + +#include + +namespace rdna2_lq { + +constexpr int LQ_K = 128; // head_k_dim +constexpr int LQ_ALIGN = 16; + +__host__ __device__ __forceinline__ long lq_align(long n) { + return (n + LQ_ALIGN - 1) / LQ_ALIGN * LQ_ALIGN; +} + +// Byte offsets of the blob sections. +struct LqLayout { + long z, bscale, csmooth, comp_k, comp_u, rec_k, rec_u, rec_g, header, total; + int V, r, cap, window, power_iters; + + __host__ __device__ void init(int V_, int r_, int cap_, int window_, + int iters_) { + V = V_; + r = r_; + cap = cap_; + window = window_; + power_iters = iters_; + long off = 0; + z = off; + off = lq_align(off + (long)V * LQ_K); + bscale = off; + off = lq_align(off + 4L * V); + csmooth = off; + off = lq_align(off + 4L * LQ_K); + comp_k = off; + off = lq_align(off + 2L * r * LQ_K); + comp_u = off; + off = lq_align(off + 2L * r * V); + rec_k = off; + off = lq_align(off + 2L * cap * LQ_K); + rec_u = off; + off = lq_align(off + 2L * cap * V); + rec_g = off; + off = lq_align(off + 4L * cap); + header = off; + off = lq_align(off + 16); + total = off; + } +}; + +template +__device__ __forceinline__ T* lq_ptr(uint8_t* base, long off) { + return reinterpret_cast(base + off); +} + +// Headers of blocks a request has not written yet hold stale bytes; anything +// outside 0 <= cursor, 0 <= pending, cursor + pending <= cap reads as empty +// so no kernel indexes the record ring out of bounds. Such blocks are either +// overwritten by a prefill store or masked by has_initial_state. +__device__ __forceinline__ bool lq_header_ok(int cursor, int pending, + const LqLayout& L) { + return cursor >= 0 && pending >= 0 && cursor <= L.cap && + pending <= L.cap - cursor; +} + +} // namespace rdna2_lq diff --git a/csrc/rocm/gdn_leapquant_rdna2.cu b/csrc/rocm/gdn_leapquant_rdna2.cu index 6b996be53b2e..df41a145e2db 100644 --- a/csrc/rocm/gdn_leapquant_rdna2.cu +++ b/csrc/rocm/gdn_leapquant_rdna2.cu @@ -27,13 +27,15 @@ #include +#include "gdn_leapquant_layout.cuh" + namespace { -constexpr int LQ_K = 128; // head_k_dim +using namespace rdna2_lq; + constexpr int LQ_BV = 32; // V rows per decode workgroup constexpr int LQ_THREADS = 256; // decode: 32 v-rows x 8 k-slices constexpr int LQ_R = 4; // compensator tokens (paper: r = 4 at 8 bits) -constexpr int LQ_ALIGN = 16; constexpr float LQ_INT8_MAX = 127.0f; constexpr float LQ_SMOOTH_FLOOR = 1e-6f; constexpr float LQ_ORTH_EPS = 1e-12f; @@ -68,60 +70,6 @@ __device__ __forceinline__ float lq_wsum(float x) { __device__ __forceinline__ float lq_f32(float x) { return x; } __device__ __forceinline__ float lq_f32(__half x) { return __half2float(x); } -__host__ __device__ __forceinline__ long lq_align(long n) { - return (n + LQ_ALIGN - 1) / LQ_ALIGN * LQ_ALIGN; -} - -// Byte offsets of the blob sections; must match LeapQuantLayout.offsets. -struct LqLayout { - long z, bscale, csmooth, comp_k, comp_u, rec_k, rec_u, rec_g, header, total; - int V, r, cap, window, power_iters; - - __host__ __device__ void init(int V_, int r_, int cap_, int window_, - int iters_) { - V = V_; - r = r_; - cap = cap_; - window = window_; - power_iters = iters_; - long off = 0; - z = off; - off = lq_align(off + (long)V * LQ_K); - bscale = off; - off = lq_align(off + 4L * V); - csmooth = off; - off = lq_align(off + 4L * LQ_K); - comp_k = off; - off = lq_align(off + 2L * r * LQ_K); - comp_u = off; - off = lq_align(off + 2L * r * V); - rec_k = off; - off = lq_align(off + 2L * cap * LQ_K); - rec_u = off; - off = lq_align(off + 2L * cap * V); - rec_g = off; - off = lq_align(off + 4L * cap); - header = off; - off = lq_align(off + 16); - total = off; - } -}; - -template -__device__ __forceinline__ T* lq_ptr(uint8_t* base, long off) { - return reinterpret_cast(base + off); -} - -// Headers of blocks a request has not written yet hold stale bytes; anything -// outside 0 <= cursor, 0 <= pending, cursor + pending <= cap reads as empty -// so no kernel indexes the record ring out of bounds. Such blocks are either -// overwritten by a prefill store or masked by has_initial_state. -__device__ __forceinline__ bool lq_header_ok(int cursor, int pending, - const LqLayout& L) { - return cursor >= 0 && pending >= 0 && cursor <= L.cap && - pending <= L.cap - cursor; -} - // --------------------------------------------------------------------------- // Fold: one workgroup (256 threads) per (request, head). Thread t owns rows // v = vg + 32 * i (i < RPT = V / 32, vg = t >> 3) and columns diff --git a/csrc/rocm/mono_gdn_rdna2.cu b/csrc/rocm/mono_gdn_rdna2.cu index a773fa0cd99c..0b611f5328d3 100644 --- a/csrc/rocm/mono_gdn_rdna2.cu +++ b/csrc/rocm/mono_gdn_rdna2.cu @@ -28,6 +28,14 @@ // // Output y [M, HV * V] fp16 is today's out_proj input. Null / padded slots // (index <= 0) write y = 0 and leave both states untouched. +// +// LeapQuant (mono_gdn_lq_rdna2, VLLM_RDNA_GDN_LEAPQUANT=1): the recurrent +// state is a LeapQuant blob per (block, head) (gdn_leapquant_layout.cuh), +// normalized beforehand by gdn_lq_normalize_rdna2. Each tile rebuilds its 32 +// rows from the home blob srow[0] (INT8 residual + compensator + replay of +// `cursor` records), runs the same math, and appends one record per token +// (u rows per tile, k and g by tile 0) instead of storing per-token states; +// tile 0 sets pending = T. #include #include @@ -38,6 +46,8 @@ #include #include +#include "gdn_leapquant_layout.cuh" + namespace gdn_fused { constexpr int kK = 128, kV = 128, kBV = 32, kThreads = 256, kConvW = 4; @@ -65,6 +75,9 @@ struct Args { const float* dtb; // [HV] float* ssm; // [slots, HV, V, K] fp32 long ss_slot; + uint8_t* lq; // LeapQuant blobs [slots, HV, nbytes] (LQ) or null + long lq_slot, lq_head; + rdna2_lq::LqLayout lql; const int* sidx; // non-spec [N]; spec [N, T] long sidx_seq; const int* nacc; // spec or null @@ -187,7 +200,7 @@ __device__ __forceinline__ float ksum8(float x) { return x; } -template +template __global__ void __launch_bounds__(kThreads) gdn_fused_k(const Args a) { const int seq = blockIdx.x / a.HV, hv = blockIdx.x % a.HV; const int tile = blockIdx.y, h = hv / (a.HV / a.H); @@ -221,7 +234,9 @@ __global__ void __launch_bounds__(kThreads) gdn_fused_k(const Args a) { return; } const int acc_i = accepted - 1; - const long slot0 = srow[acc_i]; + // LeapQuant: the running state is the home blob srow[0], whose accepted + // records gdn_lq_normalize_rdna2 has already committed. + const long slot0 = LQ ? cslot : (long)srow[acc_i]; if (slot0 <= 0 || slot0 >= a.slots) { if (tile == 0) // null / padded: y = 0 for (int t = 0; t < T; t++) @@ -266,7 +281,45 @@ __global__ void __launch_bounds__(kThreads) gdn_fused_k(const Args a) { // ---- 2. gated delta rule float hs[16]; - { + uint8_t* blob = nullptr; + int lq_c = 0, lq_nrec = 0; + if constexpr (LQ) { + using namespace rdna2_lq; + const LqLayout& L = a.lql; + blob = a.lq + slot0 * a.lq_slot + (long)hv * a.lq_head; + const int* hdr = lq_ptr(blob, L.header); + lq_c = lq_header_ok(hdr[0], hdr[1], L) ? hdr[0] : 0; + lq_nrec = min(T, L.cap - lq_c); + const float bs = lq_ptr(blob, L.bscale)[o_v]; + const float* csm = lq_ptr(blob, L.csmooth); + const int8_t* zr = lq_ptr(blob, L.z) + (long)o_v * kK; +#pragma unroll + for (int j = 0; j < 16; j++) hs[j] = (float)zr[KIDX(j)] * bs * csm[KIDX(j)]; + const half* ck = lq_ptr(blob, L.comp_k); + const half* cu = lq_ptr(blob, L.comp_u); + for (int cc = 0; cc < L.r; cc++) { + const float u = __half2float(cu[cc * kV + o_v]); +#pragma unroll + for (int j = 0; j < 16; j++) + hs[j] += u * __half2float(ck[cc * kK + KIDX(j)]); + } + const half* rk = lq_ptr(blob, L.rec_k); + const half* ru = lq_ptr(blob, L.rec_u); + const float* rg = lq_ptr(blob, L.rec_g); + for (int r = 0; r < lq_c; r++) { + // exp2 form: the same replay as gdn_lq_decode_rdna2 + const float d = exp2f(rg[r] * 0x1.715476p+0f); + const float u = __half2float(ru[r * kV + o_v]); +#pragma unroll + for (int j = 0; j < 16; j++) + hs[j] = hs[j] * d + u * __half2float(rk[r * kK + KIDX(j)]); + } + if constexpr (DIAG) { +#pragma unroll + for (int j = 0; j < 16; j++) + if (!finite32(hs[j])) diagnose(a, 4, hs[j], bos, slot0); + } + } else { const float* p = a.ssm + slot0 * a.ss_slot + ((long)hv * kV + o_v) * kK; #pragma unroll for (int j = 0; j < 16; j++) { @@ -332,11 +385,26 @@ __global__ void __launch_bounds__(kThreads) gdn_fused_k(const Args a) { else if (!finite32(__half2float(o16))) diagnose(a, 8, oa, tok, slot0); } - const long slot_t = spec ? srow[t] : slot0; - if (slot_t > 0 && slot_t < a.slots) { - float* p = a.ssm + slot_t * a.ss_slot + ((long)hv * kV + o_v) * kK; + if constexpr (LQ) { + if (t < lq_nrec) { + using namespace rdna2_lq; + const int pos = lq_c + t; + if (ks == 0) + lq_ptr(blob, a.lql.rec_u)[pos * kV + o_v] = __float2half(v2); + if (tile == 0 && v_local == 0) { + half* rk = lq_ptr(blob, a.lql.rec_k) + pos * kK; #pragma unroll - for (int j = 0; j < 16; j++) st_state(p + KIDX(j), hs[j]); + for (int j = 0; j < 16; j++) rk[KIDX(j)] = __float2half(kk[j]); + if (ks == 0) lq_ptr(blob, a.lql.rec_g)[pos] = -A * softplus; + } + } + } else { + const long slot_t = spec ? srow[t] : slot0; + if (slot_t > 0 && slot_t < a.slots) { + float* p = a.ssm + slot_t * a.ss_slot + ((long)hv * kV + o_v) * kK; +#pragma unroll + for (int j = 0; j < 16; j++) st_state(p + KIDX(j), hs[j]); + } } // this tile's 32 outputs + their sum of squares (one lane per v row) float sq = 0.f; @@ -359,6 +427,11 @@ __global__ void __launch_bounds__(kThreads) gdn_fused_k(const Args a) { } __syncthreads(); } + if constexpr (LQ) { + // Other tiles read only the cursor word, which no tile changes here. + if (tile == 0 && tid == 0) + rdna2_lq::lq_ptr(blob, a.lql.header)[1] = lq_nrec; + } // ---- 3. the last tile of this (sequence, head) normalizes the head if (tid == 0) { @@ -401,19 +474,23 @@ __global__ void __launch_bounds__(kThreads) gdn_fused_k(const Args a) { } // namespace gdn_fused +namespace { + // y = gated-RMSNorm(delta-rule(conv(qkvz))) for a decode / verify batch. // Non-spec: sidx [N] int32 (N == M), nacc / qsl empty. Spec: sidx [N, T] // int32, nacc [N], qsl [N + 1]. scratch: uint8 >= mono_gdn_rdna2_scratch_bytes, -// zero-initialized once (the kernel leaves its counters zero). -void mono_gdn_rdna2(const torch::Tensor& qkvz, const torch::Tensor& ba, - const torch::Tensor& conv_w, const torch::Tensor& conv_b, - torch::Tensor& conv_state, torch::Tensor& ssm_state, - const torch::Tensor& A_log, const torch::Tensor& dt_bias, - const torch::Tensor& norm_w, const torch::Tensor& sidx, - const torch::Tensor& nacc, const torch::Tensor& qsl, - torch::Tensor& y, torch::Tensor& scratch, - int64_t num_k_heads, double scale, double eps, - int64_t gate_sigmoid) { +// zero-initialized once (the kernel leaves its counters zero). lql: null for +// the fp32 state [slots, HV, 128, 128], else the LeapQuant blob layout of a +// uint8 state [slots, HV, nbytes]. +void mono_gdn_impl(const torch::Tensor& qkvz, const torch::Tensor& ba, + const torch::Tensor& conv_w, const torch::Tensor& conv_b, + torch::Tensor& conv_state, torch::Tensor& ssm_state, + const torch::Tensor& A_log, const torch::Tensor& dt_bias, + const torch::Tensor& norm_w, const torch::Tensor& sidx, + const torch::Tensor& nacc, const torch::Tensor& qsl, + torch::Tensor& y, torch::Tensor& scratch, + int64_t num_k_heads, double scale, double eps, + int64_t gate_sigmoid, const rdna2_lq::LqLayout* lql) { using namespace gdn_fused; const int M = qkvz.size(0); const int HV = ssm_state.size(1); @@ -436,12 +513,25 @@ void mono_gdn_rdna2(const torch::Tensor& qkvz, const torch::Tensor& ba, TORCH_CHECK(conv_state.dim() == 3 && conv_state.size(1) == CD && conv_state.scalar_type() == at::kHalf, "mono_gdn_rdna2: conv state (slots, CD, len)"); - TORCH_CHECK(ssm_state.dim() == 4 && ssm_state.size(2) == kV && - ssm_state.size(3) == kK && - ssm_state.scalar_type() == at::kFloat && - ssm_state.stride(3) == 1 && ssm_state.stride(2) == kK && - ssm_state.stride(1) == (int64_t)kV * kK, - "mono_gdn_rdna2: ssm state [slots, HV, 128, 128] fp32"); + if (lql == nullptr) { + TORCH_CHECK(ssm_state.dim() == 4 && ssm_state.size(2) == kV && + ssm_state.size(3) == kK && + ssm_state.scalar_type() == at::kFloat && + ssm_state.stride(3) == 1 && ssm_state.stride(2) == kK && + ssm_state.stride(1) == (int64_t)kV * kK, + "mono_gdn_rdna2: ssm state [slots, HV, 128, 128] fp32"); + } else { + TORCH_CHECK(ssm_state.dim() == 3 && ssm_state.scalar_type() == at::kByte && + ssm_state.size(2) == lql->total && + ssm_state.stride(2) == 1 && + ssm_state.stride(0) % rdna2_lq::LQ_ALIGN == 0 && + ssm_state.stride(1) % rdna2_lq::LQ_ALIGN == 0 && + reinterpret_cast(ssm_state.data_ptr()) % + rdna2_lq::LQ_ALIGN == + 0, + "mono_gdn_lq_rdna2: LeapQuant state uint8 [slots, HV, ", + lql->total, "], 16-byte aligned"); + } TORCH_CHECK(H > 0 && HV % H == 0 && A_log.numel() == HV && dt_bias.numel() == HV && A_log.scalar_type() == at::kFloat && dt_bias.scalar_type() == at::kFloat, @@ -495,8 +585,15 @@ void mono_gdn_rdna2(const torch::Tensor& qkvz, const torch::Tensor& ba, a.cs_tok = conv_state.stride(2); a.alog = A_log.data_ptr(); a.dtb = dt_bias.data_ptr(); - a.ssm = ssm_state.data_ptr(); - a.ss_slot = ssm_state.stride(0); + if (lql == nullptr) { + a.ssm = ssm_state.data_ptr(); + a.ss_slot = ssm_state.stride(0); + } else { + a.lq = ssm_state.data_ptr(); + a.lq_slot = ssm_state.stride(0); + a.lq_head = ssm_state.stride(1); + a.lql = *lql; + } a.sidx = sidx.data_ptr(); a.sidx_seq = sidx.dim() > 1 ? sidx.stride(0) : 1; a.nacc = spec ? nacc.data_ptr() : nullptr; @@ -524,24 +621,70 @@ void mono_gdn_rdna2(const torch::Tensor& qkvz, const torch::Tensor& ba, } const size_t lds = (size_t)tmax * (2 * kK + kBV) * sizeof(float); const dim3 grid(N * HV, kV / kBV); - if (diagnostics) { + if (lql != nullptr) { + // KMAP 0: 16 contiguous k columns per lane, one 16-byte INT8 row read. + if (diagnostics) + hipLaunchKernelGGL((gdn_fused_k<0, true, true>), grid, dim3(kThreads), + lds, stream, a); + else + hipLaunchKernelGGL((gdn_fused_k<0, false, true>), grid, dim3(kThreads), + lds, stream, a); + } else if (diagnostics) { if (!spec && M > 16) - hipLaunchKernelGGL((gdn_fused_k<0, true>), grid, dim3(kThreads), lds, - stream, a); + hipLaunchKernelGGL((gdn_fused_k<0, true, false>), grid, dim3(kThreads), + lds, stream, a); else - hipLaunchKernelGGL((gdn_fused_k<1, true>), grid, dim3(kThreads), lds, - stream, a); + hipLaunchKernelGGL((gdn_fused_k<1, true, false>), grid, dim3(kThreads), + lds, stream, a); } else { if (!spec && M > 16) - hipLaunchKernelGGL((gdn_fused_k<0, false>), grid, dim3(kThreads), lds, - stream, a); + hipLaunchKernelGGL((gdn_fused_k<0, false, false>), grid, dim3(kThreads), + lds, stream, a); else - hipLaunchKernelGGL((gdn_fused_k<1, false>), grid, dim3(kThreads), lds, - stream, a); + hipLaunchKernelGGL((gdn_fused_k<1, false, false>), grid, dim3(kThreads), + lds, stream, a); } C10_CUDA_KERNEL_LAUNCH_CHECK(); } +} // namespace + +void mono_gdn_rdna2(const torch::Tensor& qkvz, const torch::Tensor& ba, + const torch::Tensor& conv_w, const torch::Tensor& conv_b, + torch::Tensor& conv_state, torch::Tensor& ssm_state, + const torch::Tensor& A_log, const torch::Tensor& dt_bias, + const torch::Tensor& norm_w, const torch::Tensor& sidx, + const torch::Tensor& nacc, const torch::Tensor& qsl, + torch::Tensor& y, torch::Tensor& scratch, + int64_t num_k_heads, double scale, double eps, + int64_t gate_sigmoid) { + mono_gdn_impl(qkvz, ba, conv_w, conv_b, conv_state, ssm_state, A_log, dt_bias, + norm_w, sidx, nacc, qsl, y, scratch, num_k_heads, scale, eps, + gate_sigmoid, nullptr); +} + +// The same fused decode core on LeapQuant blobs (VLLM_RDNA_GDN_LEAPQUANT=1). +// blob: uint8 [slots, HV, nbytes], normalized by gdn_lq_normalize_rdna2 with +// the batch's num_accepted (spec) or None (non-spec) right before this call. +void mono_gdn_lq_rdna2(const torch::Tensor& qkvz, const torch::Tensor& ba, + const torch::Tensor& conv_w, const torch::Tensor& conv_b, + torch::Tensor& conv_state, torch::Tensor& blob, + const torch::Tensor& A_log, const torch::Tensor& dt_bias, + const torch::Tensor& norm_w, const torch::Tensor& sidx, + const torch::Tensor& nacc, const torch::Tensor& qsl, + torch::Tensor& y, torch::Tensor& scratch, + int64_t num_k_heads, double scale, double eps, + int64_t gate_sigmoid, int64_t num_comp, int64_t capacity, + int64_t window) { + TORCH_CHECK(num_comp >= 0 && window >= 1 && capacity >= window, + "mono_gdn_lq_rdna2: bad LeapQuant layout"); + rdna2_lq::LqLayout L; + L.init(gdn_fused::kV, (int)num_comp, (int)capacity, (int)window, 0); + mono_gdn_impl(qkvz, ba, conv_w, conv_b, conv_state, blob, A_log, dt_bias, + norm_w, sidx, nacc, qsl, y, scratch, num_k_heads, scale, eps, + gate_sigmoid, &L); +} + int64_t mono_gdn_rdna2_scratch_bytes(int64_t M, int64_t HV, int64_t H) { return (int64_t)gdn_fused::kCapRows * (HV * 20 + H * 4) + M * HV * gdn_fused::kV * 2 + sizeof(int); diff --git a/csrc/rocm/ops.h b/csrc/rocm/ops.h index 41c9bae562d2..ce0ca2e2f7b5 100644 --- a/csrc/rocm/ops.h +++ b/csrc/rocm/ops.h @@ -99,6 +99,16 @@ void mono_gdn_rdna2(const torch::Tensor& qkvz, const torch::Tensor& ba, torch::Tensor& y, torch::Tensor& scratch, int64_t num_k_heads, double scale, double eps, int64_t gate_sigmoid); +void mono_gdn_lq_rdna2(const torch::Tensor& qkvz, const torch::Tensor& ba, + const torch::Tensor& conv_w, const torch::Tensor& conv_b, + torch::Tensor& conv_state, torch::Tensor& blob, + const torch::Tensor& A_log, const torch::Tensor& dt_bias, + const torch::Tensor& norm_w, const torch::Tensor& sidx, + const torch::Tensor& nacc, const torch::Tensor& qsl, + torch::Tensor& y, torch::Tensor& scratch, + int64_t num_k_heads, double scale, double eps, + int64_t gate_sigmoid, int64_t num_comp, int64_t capacity, + int64_t window); int64_t mono_gdn_rdna2_scratch_bytes(int64_t M, int64_t HV, int64_t H); void mono_qsa_decode_rdna2(const torch::Tensor& q, const torch::Tensor& k_cache, const torch::Tensor& v_cache, diff --git a/csrc/rocm/torch_bindings.cpp b/csrc/rocm/torch_bindings.cpp index 62e934947c36..8343030ebb4d 100644 --- a/csrc/rocm/torch_bindings.cpp +++ b/csrc/rocm/torch_bindings.cpp @@ -192,6 +192,13 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, rocm_ops) { "Tensor! scratch, int num_k_heads, float scale, float eps, " "int gate_sigmoid) -> ()"); rocm_ops.impl("mono_gdn_rdna2", torch::kCUDA, &mono_gdn_rdna2); + rocm_ops.def( + "mono_gdn_lq_rdna2(Tensor qkvz, Tensor ba, Tensor conv_w, Tensor conv_b, " + "Tensor! conv_state, Tensor! blob, Tensor A_log, Tensor dt_bias, " + "Tensor norm_w, Tensor sidx, Tensor nacc, Tensor qsl, Tensor! y, " + "Tensor! scratch, int num_k_heads, float scale, float eps, " + "int gate_sigmoid, int num_comp, int capacity, int window) -> ()"); + rocm_ops.impl("mono_gdn_lq_rdna2", torch::kCUDA, &mono_gdn_lq_rdna2); rocm_ops.def("mono_gdn_rdna2_scratch_bytes(int M, int HV, int H) -> int"); rocm_ops.impl("mono_gdn_rdna2_scratch_bytes", &mono_gdn_rdna2_scratch_bytes); rocm_ops.def( diff --git a/docs/rdna2/leapquant-gdn-2026-11-13.md b/docs/rdna2/leapquant-gdn-2026-11-13.md index fbf5369a6e45..6d1aca48ddc0 100644 --- a/docs/rdna2/leapquant-gdn-2026-11-13.md +++ b/docs/rdna2/leapquant-gdn-2026-11-13.md @@ -44,21 +44,30 @@ once per token. The FP32 oracle is * **Readers of slot 0** (chunked prefill, non-spec decodes) commit `min(1, pending)`, the same state the FP32 path reads. * **Prefill** materializes the blob to an FP32 `[N, HV, V, K]` initial state, - runs the unchanged HIP/Triton chunk kernels, and stores the final state - back (`gdn_lq_store_rdna2`). Inside a chunk nothing changes; LeapQuant error - enters at chunk boundaries (MAXBAT 2048). + runs the unchanged HIP prefill chain (`VLLM_GDN_HIP_PREFILL=1`) and stores + the final state back (`gdn_lq_store_rdna2`). Inside a chunk nothing + changes; LeapQuant error enters at chunk boundaries (MAXBAT 2048). * **Prefix caching (`mamba-align`, V2 runner):** the align precopy/postprocess HIP kernels (`v1_mamba_rdna2.cu`, `copy_mamba_state_block`) copy the whole blob from the home block and write `cursor + min(bias + 1, pending)` into the copy's header, which is exactly the FP32 "copy slot `bias`" rule. The V1 runner's memcpy specs and the Triton precopy fallback raise instead of silently copying wrong bytes. +* **MonoKernel (`EXTRA_ARGS=--monokernel`):** pure decode and pure verify + batches run `gdn_lq_normalize_rdna2` and then `mono_gdn_lq_rdna2`, the + `mono_gdn_rdna2` fused conv + delta rule + gated norm launch with the + LeapQuant load/append in place of the FP32 state load/store (each 32-row + tile rebuilds its rows from the blob; records are written per tile). Batches + with prefill take the layer's path as with FP32 state. +* **HIP only.** Every LeapQuant read and write is one of the kernels above; + the FP32-state Triton decode warmup is skipped under the flag (it never + runs). Paths the flag does not change (conv update, gated norm outside the + MonoKernel) are the recipe's, identical with the flag off. * **Cudagraphs:** every op is a fixed-shape launch on the current stream that reads its metadata from device tensors, so FULL_AND_PIECEWISE captures it like the FP32 kernels. -* **Refused combinations** (raise at init): mono decode op, fused CUDA spec - decode, AITER/FlashInfer GDN, align mode on the V1 runner, head dims other - than 128. +* **Refused combinations** (raise at init): fused CUDA spec decode, + AITER/FlashInfer GDN, align mode on the V1 runner, head dims other than 128. ## Use diff --git a/tests/kernels/mamba/test_gdn_leapquant_rdna2.py b/tests/kernels/mamba/test_gdn_leapquant_rdna2.py index db1ab484c672..1a89332bb36f 100644 --- a/tests/kernels/mamba/test_gdn_leapquant_rdna2.py +++ b/tests/kernels/mamba/test_gdn_leapquant_rdna2.py @@ -321,3 +321,99 @@ def test_align_precopy_commits_accepted_records(): lq.leapquant_normalize(L, expect, bias.long() + 1) got = _materialize(L, blob, block_table[:, 1].contiguous(), n) assert _rel(got, lq.leapquant_materialize(L, expect)) < 1e-5 + + +def _mono_inputs(m, conv_len, seed): + g = torch.Generator(device="cpu").manual_seed(seed) + cd = 2 * H * K + HV * V + + def rn(*s, std): + return (torch.randn(*s, generator=g) * std).half().cuda() + + qkvz = rn(m, cd + HV * V, std=0.5) + ba = rn(m, 2 * HV, std=1.0) + conv = rn(SLOTS_MONO, cd, conv_len, std=0.5) + return qkvz, ba, conv + + +SLOTS_MONO = 24 + + +@pytest.mark.parametrize("spec", [False, True]) +def test_mono_gdn_leapquant_matches_fp32_mono(spec): + """The MonoKernel GDN on LeapQuant blobs (mono_gdn_lq_rdna2) is the FP32 + MonoKernel (mono_gdn_rdna2) run on the materialized state: same outputs + and conv state, and every per-token state the FP32 kernel stores is the + blob with that many records committed. Two steps, so the second replays + the first one's records.""" + t = 3 if spec else 1 + L = _layout(t) + n = 4 + cd = 2 * H * K + HV * V + A_log, dt_bias = _params(5) + g = torch.Generator(device="cpu").manual_seed(6) + conv_w = (torch.randn(cd, 4, generator=g) * 0.3).half().cuda() + conv_b = (torch.randn(cd, generator=g) * 0.1).half().cuda() + norm_w = (1.0 + torch.randn(V, generator=g) * 0.1).half().cuda() + perm = torch.randperm(SLOTS_MONO - 1, generator=g)[: n * t] + 1 + sidx = perm.int().view(n, t).cuda() if spec else perm.int().cuda() + sidx[1] = 0 # null / padded sequence + home = (sidx[:, 0] if spec else sidx).contiguous() + blob = torch.zeros(SLOTS_MONO, HV, L.nbytes, dtype=torch.uint8, device="cuda") + live = home[home > 0] + _store(L, torch.randn(live.numel(), HV, V, K, device="cuda") * 0.3, blob, live) + empty = torch.empty(0, dtype=torch.int32, device="cuda") + qsl = torch.arange(0, n * t + 1, t, dtype=torch.int32, device="cuda") + scratch = torch.zeros( + int(_ops().mono_gdn_rdna2_scratch_bytes(64, HV, H)), + dtype=torch.uint8, + device="cuda", + ) + nacc = None + for step in range(2): + qkvz, ba, conv = _mono_inputs(n * t, 3 + t - 1, seed=60 + step) + if spec: + nacc = torch.tensor([3, 1, 2, 1], dtype=torch.int32, device="cuda") + if step == 0: + nacc.fill_(1) # fresh blobs hold no pending records + _normalize(L, blob, home, nacc) + # FP32 reference: the committed home state at the slot it reads. + ssm = torch.zeros(SLOTS_MONO, HV, V, K, device="cuda") + if nacc is not None: + read = sidx[torch.arange(n, device="cuda"), nacc.long() - 1] + else: + read = sidx + ssm[read.long()] = _materialize(L, blob, home, n) + ssm[0] = 0 + conv_ref, y_ref = conv.clone(), torch.empty(n * t, HV * V).half().cuda() + y = torch.empty_like(y_ref) + common = (A_log.float(), dt_bias.float(), norm_w, sidx) + tail = (empty if not spec else nacc, empty if not spec else qsl) + _ops().mono_gdn_rdna2( + qkvz, ba, conv_w, conv_b, conv_ref, ssm, *common, *tail, y_ref, + scratch, H, K**-0.5, 1e-6, 0, + ) # fmt: skip + _ops().mono_gdn_lq_rdna2( + qkvz, ba, conv_w, conv_b, conv, blob, *common, *tail, y, + scratch, H, K**-0.5, 1e-6, 0, R, L.capacity, WINDOW, + ) # fmt: skip + torch.accelerator.synchronize() + assert torch.equal(conv, conv_ref) + rows = (home > 0).repeat_interleave(t) + assert bool((y[~rows] == 0).all()) + assert _rel(y[rows], y_ref[rows]) < 2e-3 + # Each per-token FP32 state is the blob with t + 1 records committed. + for acc in range(1, t + 1): + trial = blob.clone() + _normalize( + L, + trial, + home, + torch.full((n,), acc, dtype=torch.int32, device="cuda") + if spec + else None, + ) + got = _materialize(L, trial, home, n) + want = ssm[(sidx[:, acc - 1] if spec else sidx).long()] + live_rows = home > 0 + assert _rel(got[live_rows], want[live_rows]) < 2e-3, (step, acc) diff --git a/tools/rdna/port_v031/decode_kl.py b/tools/rdna/port_v031/decode_kl.py index f045fe5abc0f..b0a2d4fdf827 100755 --- a/tools/rdna/port_v031/decode_kl.py +++ b/tools/rdna/port_v031/decode_kl.py @@ -76,7 +76,10 @@ def generate(args: argparse.Namespace, prompt: str) -> dict: "logprobs": args.top, "return_tokens_as_token_ids": True, } - out = post(args.url.rstrip("/") + "/v1/completions", body, args.timeout) + url = args.url.rstrip("/") + if not url.endswith("/v1/completions"): + url += "/v1/completions" + out = post(url, body, args.timeout) lp = out["choices"][0]["logprobs"] ids = [token_id(t) for t in lp["tokens"]] top = [ @@ -193,7 +196,7 @@ def main() -> int: ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) sub = ap.add_subparsers(dest="cmd", required=True) c = sub.add_parser("collect") - c.add_argument("--url", required=True) + c.add_argument("--url", required=True, help="server base or /v1/completions URL") c.add_argument("--model", required=True) c.add_argument("--out", required=True) c.add_argument("--files", nargs="*") diff --git a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py index 6d316ec6fdc8..0db1f6532224 100644 --- a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @@ -1006,10 +1006,10 @@ def _init_leapquant(self, vllm_config: VllmConfig) -> LeapQuantLayout | None: decode, prefill).""" if not leapquant_gdn_enabled(): return None - if self._mono_gdn_op is not None or self.enable_fused_gdn_spec_decode: + if self.enable_fused_gdn_spec_decode: raise ValueError( - "VLLM_RDNA_GDN_LEAPQUANT=1 does not cover the GDN monokernel or " - "the fused CUDA spec-decode path" + "VLLM_RDNA_GDN_LEAPQUANT=1 does not cover the fused CUDA " + "spec-decode path" ) if GDN_AITER_TRITON_AVAILABLE or self._checkpoint_alignment is not None: raise ValueError( @@ -1931,7 +1931,10 @@ def _warmup_decode_kernels(self, qkv_or_qkvz: torch.Tensor) -> None: if self._decode_kernels_warmed_up: return self._decode_kernels_warmed_up = True - if not on_rdna_family(): + if not on_rdna_family() or self._lq_layout is not None: + # LeapQuant runs the gdn_lq_* HIP kernels, not these (their FP32 + # state shapes do not fit the blob), and needs the V2 runner, + # whose JIT warmup replays the real decode steps. return device = qkv_or_qkvz.device @@ -2183,6 +2186,29 @@ def _forward_core_rocm( core_attn_out=core_attn_out, ) + def _lq_normalize( + self, + blob: torch.Tensor, + state_indices: torch.Tensor, + num_accepted: torch.Tensor | None, + ) -> None: + """Commit the previous step's accepted records of the home blobs + ``state_indices`` (``num_accepted``; one record for slot-0 readers, + ``None``) and fold full windows. Runs before every decode / verify + launch, the MonoKernel one included.""" + layout = self._lq_layout + assert layout is not None + torch.ops._rocm_C.gdn_lq_normalize_rdna2( + blob, + state_indices, + num_accepted, + layout.head_v_dim, + layout.num_comp, + layout.capacity, + layout.window, + layout.power_iters, + ) + def _lq_recurrent( self, q: torch.Tensor, @@ -2205,16 +2231,7 @@ def _lq_recurrent( assert layout is not None num_reqs = cu_seqlens.shape[0] - 1 state_indices = state_indices[:num_reqs] - torch.ops._rocm_C.gdn_lq_normalize_rdna2( - blob, - state_indices, - num_accepted, - layout.head_v_dim, - layout.num_comp, - layout.capacity, - layout.window, - layout.power_iters, - ) + self._lq_normalize(blob, state_indices, num_accepted) if out is None: out = torch.zeros( (1, q.shape[1], v.shape[2], v.shape[3]), diff --git a/vllm/models/qwen4_exp/amd/mono/rdna2/gdn.py b/vllm/models/qwen4_exp/amd/mono/rdna2/gdn.py index aea632224ef1..9c6335d41f04 100644 --- a/vllm/models/qwen4_exp/amd/mono/rdna2/gdn.py +++ b/vllm/models/qwen4_exp/amd/mono/rdna2/gdn.py @@ -109,7 +109,8 @@ def eligible(self, md: GDNAttentionMetadata, qkvz: torch.Tensor) -> bool: if ( qkvz.dtype != torch.float16 or layer.kv_cache[0].dtype != torch.float16 - or layer.kv_cache[1].dtype != torch.float32 + or layer.kv_cache[1].dtype + != (torch.float32 if layer._lq_layout is None else torch.uint8) or layer.norm.weight.dtype != torch.float16 or layer.conv1d.weight.dtype != torch.float16 ): @@ -216,7 +217,7 @@ def forward_core( sidx, BreakableCUDAGraphCapture.current(), ) - torch.ops._rocm_C.mono_gdn_rdna2( + args = ( mixed_qkvz[:M], ba[:M], conv_w, @@ -236,6 +237,19 @@ def forward_core( layer.norm.eps, sig, ) + lq = layer._lq_layout + if lq is None: + torch.ops._rocm_C.mono_gdn_rdna2(*args) + else: + # LeapQuant: commit the accepted records of the home blobs + # (spec: column 0), then the fused core appends this step's. + spec = md.spec_sequence_masks is not None + layer._lq_normalize( + ssm_state, sidx[:, 0] if spec else sidx, nacc if spec else None + ) + torch.ops._rocm_C.mono_gdn_lq_rdna2( + *args, lq.num_comp, lq.capacity, lq.window + ) if trace is not None: offload_trace.finish_gdn(trace, y[:M], conv_state, ssm_state) layer._unbind_decode_state_arenas(*paged, md, md.num_decodes) From 8ba674ad8bb0437909f3ad2a9fc7d6f3ca45bf62 Mon Sep 17 00:00:00 2001 From: BlivionIaG Date: Fri, 13 Nov 2026 10:51:32 +0100 Subject: [PATCH 03/18] perf(rdna2): one LeapQuant normalize launch per step for all GDN layers All LeapQuant GDN layers normalize with the same state indices and accepted counts each step, so the first layer of the model now normalizes every layer's paged blobs in one launch (gdn_lq_normalize_rdna2 gains an optional layer_offsets [layers] table, grid.y = layer; the group is rebuilt on bind_kv_cache, never under capture) and the other layers skip. Decode arenas stay per layer. Replaces 36 launches (~3 us each) and lets the folds of all layers run concurrently instead of one layer after the other. The record replay loops unroll by 2 (same arithmetic order). Per step, 36 layers, one V620, HIP graph (FP32 MonoKernel -> LeapQuant before -> after): decode n=1 343 -> 574 -> 361 us, verify n=1 ~500 -> 867 -> 582 us, decode n=8 1556 -> 1345 -> 1134 us, verify n=8 3323 -> 2217 -> 1851 us. test_grouped_normalize_matches_per_layer: same bytes as per-layer normalize, folds and NULL slots included. Signed-off-by: BlivionIaG --- csrc/rocm/gdn_leapquant_rdna2.cu | 29 ++++++- csrc/rocm/mono_gdn_rdna2.cu | 3 + csrc/rocm/ops.h | 3 +- csrc/rocm/torch_bindings.cpp | 4 +- .../kernels/mamba/test_gdn_leapquant_rdna2.py | 39 +++++++++ .../layers/mamba/gdn/qwen_gdn_linear_attn.py | 79 ++++++++++++++++--- 6 files changed, 140 insertions(+), 17 deletions(-) diff --git a/csrc/rocm/gdn_leapquant_rdna2.cu b/csrc/rocm/gdn_leapquant_rdna2.cu index df41a145e2db..7f788347667e 100644 --- a/csrc/rocm/gdn_leapquant_rdna2.cu +++ b/csrc/rocm/gdn_leapquant_rdna2.cu @@ -13,6 +13,8 @@ // gdn_lq_normalize_rdna2 commit `num_accepted` pending records (one for a // slot-0 reader: num_accepted = None), fold full // windows into a new boundary (one WG per head). +// With layer_offsets it covers every layer of a +// group in one launch (grid.y = layer). // gdn_lq_decode_rdna2 decode / MTP verify on normalized blobs: replay, // run T tokens, append T records, pending = T. // gdn_lq_materialize_rdna2 normalized blob -> FP32 [N, HV, V, K]. @@ -346,6 +348,7 @@ __device__ void lq_load_full(float (&S)[RPT][16], uint8_t* blob, const __half* rk = lq_ptr<__half>(blob, L.rec_k); const __half* ru = lq_ptr<__half>(blob, L.rec_u); const float* rg = lq_ptr(blob, L.rec_g); +#pragma unroll 2 for (int t = 0; t < c; ++t) { const float decay = lq_exp(rg[t]); float kk[16]; @@ -362,6 +365,7 @@ __device__ void lq_load_full(float (&S)[RPT][16], uint8_t* blob, template __global__ void __launch_bounds__(LQ_THREADS) gdn_lq_normalize_kernel(uint8_t* __restrict__ blobs, + const int64_t* __restrict__ layer_offsets, const int* __restrict__ state_indices, long idx_stride, const int* __restrict__ num_accepted, @@ -372,6 +376,7 @@ __global__ void __launch_bounds__(LQ_THREADS) const int i_hv = blockIdx.x % HV; const long idx = state_indices[(long)i_n * idx_stride]; if (idx <= 0 || idx >= num_blocks) return; + if (layer_offsets != nullptr) blobs += layer_offsets[blockIdx.y]; uint8_t* blob = blobs + idx * stride_block + (long)i_hv * stride_head; int* hdr = lq_ptr(blob, L.header); if (threadIdx.x == 0) { @@ -546,6 +551,7 @@ __global__ void __launch_bounds__(LQ_THREADS) __half* rk = lq_ptr<__half>(blob, L.rec_k); __half* ru = lq_ptr<__half>(blob, L.rec_u); float* rg = lq_ptr(blob, L.rec_g); +#pragma unroll 2 for (int t = 0; t < c; ++t) { const float decay = lq_exp(rg[t]); const float u = __half2float(ru[t * V + o_v]); @@ -669,7 +675,8 @@ void gdn_lq_normalize_rdna2(torch::Tensor blob, torch::Tensor state_indices, const std::optional& num_accepted, int64_t head_v_dim, int64_t num_comp, int64_t capacity, int64_t window, - int64_t power_iters) { + int64_t power_iters, + const std::optional& layer_offsets) { const LqLayout L = make_layout((int)head_v_dim, num_comp, capacity, window, power_iters); const int64_t HV = blob.size(1); @@ -684,14 +691,28 @@ void gdn_lq_normalize_rdna2(torch::Tensor blob, torch::Tensor state_indices, num_accepted->numel() >= N, "num_accepted must be unit-stride CUDA int32 [N]"); } + // layer_offsets[l]: byte offset of layer l's blob tensor from `blob`; every + // layer's blob has blob's shape and strides (the caller guarantees it). + int64_t num_layers = 1; + if (layer_offsets.has_value()) { + TORCH_CHECK( + layer_offsets->is_cuda() && layer_offsets->scalar_type() == at::kLong && + layer_offsets->dim() == 1 && layer_offsets->stride(0) == 1 && + layer_offsets->numel() >= 1, + "layer_offsets must be unit-stride CUDA int64 [layers]"); + num_layers = layer_offsets->numel(); + } if (N == 0) return; const at::cuda::OptionalCUDAGuard guard(blob.device()); auto stream = at::cuda::getCurrentCUDAStream(); + const dim3 grid((unsigned)(N * HV), (unsigned)num_layers); LQ_DISPATCH_RPT( head_v_dim, - gdn_lq_normalize_kernel<<>>( - blob.data_ptr(), state_indices.data_ptr(), - state_indices.stride(0), + gdn_lq_normalize_kernel<<>>( + blob.data_ptr(), + layer_offsets.has_value() ? layer_offsets->data_ptr() + : nullptr, + state_indices.data_ptr(), state_indices.stride(0), has_acc ? num_accepted->data_ptr() : nullptr, blob.stride(0), blob.stride(1), blob.size(0), (int)HV, L)); C10_CUDA_KERNEL_LAUNCH_CHECK(); diff --git a/csrc/rocm/mono_gdn_rdna2.cu b/csrc/rocm/mono_gdn_rdna2.cu index 0b611f5328d3..f16ec73355a6 100644 --- a/csrc/rocm/mono_gdn_rdna2.cu +++ b/csrc/rocm/mono_gdn_rdna2.cu @@ -306,6 +306,9 @@ __global__ void __launch_bounds__(kThreads) gdn_fused_k(const Args a) { const half* rk = lq_ptr(blob, L.rec_k); const half* ru = lq_ptr(blob, L.rec_u); const float* rg = lq_ptr(blob, L.rec_g); + // Unrolled so the record loads of 2 steps issue together: the replay is + // a chain through hs. Unroll 4 measured slower at 8 requests. +#pragma unroll 2 for (int r = 0; r < lq_c; r++) { // exp2 form: the same replay as gdn_lq_decode_rdna2 const float d = exp2f(rg[r] * 0x1.715476p+0f); diff --git a/csrc/rocm/ops.h b/csrc/rocm/ops.h index ce0ca2e2f7b5..224d2404d15c 100644 --- a/csrc/rocm/ops.h +++ b/csrc/rocm/ops.h @@ -581,7 +581,8 @@ void gdn_lq_normalize_rdna2(torch::Tensor blob, torch::Tensor state_indices, const std::optional& num_accepted, int64_t head_v_dim, int64_t num_comp, int64_t capacity, int64_t window, - int64_t power_iters); + int64_t power_iters, + const std::optional& layer_offsets); void gdn_lq_decode_rdna2(torch::Tensor q, torch::Tensor k, torch::Tensor v, torch::Tensor a, torch::Tensor b, torch::Tensor A_log, torch::Tensor dt_bias, torch::Tensor out, diff --git a/csrc/rocm/torch_bindings.cpp b/csrc/rocm/torch_bindings.cpp index 8343030ebb4d..83b534e56829 100644 --- a/csrc/rocm/torch_bindings.cpp +++ b/csrc/rocm/torch_bindings.cpp @@ -371,10 +371,12 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, rocm_ops) { // LeapQuant INT8 GDN recurrent state for AMD RDNA2 (gfx1030); opt-in via // VLLM_RDNA_GDN_LEAPQUANT. blob = uint8 [blocks, HV, nbytes]. + // layer_offsets: optional int64 [layers] byte offsets of same-shape blobs + // from `blob`; one launch then normalizes every layer of the group. rocm_ops.def( "gdn_lq_normalize_rdna2(Tensor! blob, Tensor state_indices, " "Tensor? num_accepted, int head_v_dim, int num_comp, int capacity, " - "int window, int power_iters) -> ()"); + "int window, int power_iters, Tensor? layer_offsets=None) -> ()"); rocm_ops.impl("gdn_lq_normalize_rdna2", torch::kCUDA, &gdn_lq_normalize_rdna2); rocm_ops.def( diff --git a/tests/kernels/mamba/test_gdn_leapquant_rdna2.py b/tests/kernels/mamba/test_gdn_leapquant_rdna2.py index 1a89332bb36f..a737e81d798c 100644 --- a/tests/kernels/mamba/test_gdn_leapquant_rdna2.py +++ b/tests/kernels/mamba/test_gdn_leapquant_rdna2.py @@ -230,6 +230,45 @@ def test_trajectory_tracks_fp32_with_rejections(spec): ) +def test_grouped_normalize_matches_per_layer(): + """One launch over a group of layers (layer_offsets) must leave every + layer's blobs exactly as normalizing each layer on its own does, folds + and NULL slots included.""" + L = _layout(3) + A_log, dt_bias = _params() + n, layers = 3, 3 + idx = torch.tensor([1, 0, 2], dtype=torch.int32, device="cuda") + blobs = [] + for layer in range(layers): + blob = torch.zeros(n, HV, L.nbytes, dtype=torch.uint8, device="cuda") + blob[0].fill_(7) + _store(L, torch.randn(n, HV, V, K, device="cuda") * 0.5, blob, idx) + # 8 steps commit 7 * 2 = 14 records; committing 3 (request 0) or 2 + # (request 2) more below reaches the window and folds. + for step in range(8): + q, k, v, a, b = _tokens(n, 3, seed=100 * layer + step) + _normalize(L, blob, idx, torch.full((n,), 2, dtype=torch.int32).cuda()) + _decode(L, blob, idx, q, k, v, a, b, A_log, dt_bias) + blobs.append(blob) + num_acc = torch.tensor([3, 1, 2], dtype=torch.int32, device="cuda") + ref = [blob.clone() for blob in blobs] + for blob in ref: + _normalize(L, blob, idx, num_acc) + base = blobs[0].data_ptr() + offsets = torch.tensor( + [blob.data_ptr() - base for blob in blobs], dtype=torch.int64, device="cuda" + ) + _ops().gdn_lq_normalize_rdna2( + blobs[0], idx, num_acc, V, R, L.capacity, WINDOW, ITERS, offsets + ) + folded = 0 + for got, want in zip(blobs, ref): + assert torch.equal(got, want) + cursor = L.views(got[1:])["header"][..., 0] + folded += int((cursor < 8).sum()) + assert folded > 0, "no fold exercised" + + def test_normalize_and_decode_replay_in_cuda_graph(): L = _layout(3) A_log, dt_bias = _params() diff --git a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py index 0db1f6532224..202661fc737e 100644 --- a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @@ -127,6 +127,49 @@ FUSED_GDN_STATE_DTYPES = (torch.float32, torch.bfloat16) +class _LqNormalizeGroup: + """The LeapQuant GDN layers of one model on this rank. + + Every layer normalizes its blobs with the same state indices and accepted + counts each step (they share one attention metadata), so the first layer + of the group normalizes all of them in one launch and the others skip: + one launch per step instead of one per layer (~3 us each in a graph). + ``offsets`` holds each layer's paged blob address relative to the + leader's; it is rebuilt on every ``bind_kv_cache``, which never runs under + capture, so captured graphs only ever see a finished table.""" + + def __init__(self) -> None: + self.members: dict[int, QwenGatedDeltaNetAttention] = {} + self.leader: QwenGatedDeltaNetAttention | None = None + self.leader_ptr = 0 + self.offsets: torch.Tensor | None = None + + def rebuild(self) -> None: + layers = [self.members[i] for i in sorted(self.members)] + blobs = [layer.kv_cache[1] for layer in layers] + ref = blobs[0] + uniform = all( + b.device == ref.device + and b.dtype == ref.dtype + and b.shape == ref.shape + and b.stride() == ref.stride() + for b in blobs + ) + if len(layers) < 2 or not uniform: + self.leader, self.leader_ptr, self.offsets = None, 0, None + return + self.leader = layers[0] + self.leader_ptr = ref.data_ptr() + self.offsets = torch.tensor( + [b.data_ptr() - self.leader_ptr for b in blobs], + dtype=torch.int64, + device=ref.device, + ) + + +_LQ_NORMALIZE_GROUPS: dict[str, _LqNormalizeGroup] = {} + + # Scratch buffers reused across GDN prefill calls. The key MUST carry a role # name: w is [B, T, H, K] and u/v_new are [B, T, H, V], which are the same # shape and dtype, and so are the caller's q and k -- a shape-only key makes @@ -993,6 +1036,7 @@ def __init__( self._mono_gdn: Any self._mono_gdn_op, self._mono_gdn = mono if mono else (None, None) self._lq_layout = self._init_leapquant(vllm_config) + self._lq_group: _LqNormalizeGroup | None = None compilation_config = get_current_vllm_config().compilation_config if prefix in compilation_config.static_forward_context: @@ -1034,6 +1078,16 @@ def _init_leapquant(self, vllm_config: VllmConfig) -> LeapQuantLayout | None: ) return layout + def bind_kv_cache(self, kv_cache: torch.Tensor) -> None: + super().bind_kv_cache(kv_cache) + if self._lq_layout is None: + return + root = self.prefix.rsplit(".layers.", 1)[0] + group = _LQ_NORMALIZE_GROUPS.setdefault(root, _LqNormalizeGroup()) + group.members[self.layer_idx] = self + group.rebuild() + self._lq_group = group + def _init_gdn_state_arenas(self) -> None: """Permanent arenas on the layer device, before first BeginCapture.""" conv_shape, ssm_shape = self.get_state_shape()[:2] @@ -2195,9 +2249,20 @@ def _lq_normalize( """Commit the previous step's accepted records of the home blobs ``state_indices`` (``num_accepted``; one record for slot-0 readers, ``None``) and fold full windows. Runs before every decode / verify - launch, the MonoKernel one included.""" + launch, the MonoKernel one included, and before prefill reads. + + On the paged blobs the group leader normalizes every layer of the + group in one launch and the other layers skip; decode arenas are + per-layer copies, so they are normalized per layer.""" layout = self._lq_layout assert layout is not None + group = self._lq_group + offsets = None + if group is not None and group.leader is not None and blob is self.kv_cache[1]: + if group.leader is not self: + return + assert blob.data_ptr() == group.leader_ptr, "stale LeapQuant group" + offsets = group.offsets torch.ops._rocm_C.gdn_lq_normalize_rdna2( blob, state_indices, @@ -2207,6 +2272,7 @@ def _lq_normalize( layout.capacity, layout.window, layout.power_iters, + offsets, ) def _lq_recurrent( @@ -2641,16 +2707,7 @@ def _forward_core( if lq is not None: # Commit the slot-0 record a previous decode left pending, then # expand the blobs into the FP32 initial state of the chunk. - torch.ops._rocm_C.gdn_lq_normalize_rdna2( - ssm_state, - prefill_state_indices, - None, - lq.head_v_dim, - lq.num_comp, - lq.capacity, - lq.window, - lq.power_iters, - ) + self._lq_normalize(ssm_state, prefill_state_indices, None) initial_state = torch.empty( ( prefill_state_indices.shape[0], From 1e9e44418eec16fba74835044e107b6278cd651a Mon Sep 17 00:00:00 2001 From: BlivionIaG Date: Fri, 13 Nov 2026 10:52:05 +0100 Subject: [PATCH 04/18] feat(rdna2): log once when a LeapQuant normalize launch covers the layer group Signed-off-by: BlivionIaG --- vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py index 202661fc737e..16e0b788d1a2 100644 --- a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @@ -2263,6 +2263,11 @@ def _lq_normalize( return assert blob.data_ptr() == group.leader_ptr, "stale LeapQuant group" offsets = group.offsets + assert offsets is not None + logger.info_once( + "LeapQuant GDN: one normalize launch covers %d layers", + offsets.numel(), + ) torch.ops._rocm_C.gdn_lq_normalize_rdna2( blob, state_indices, From d879da8c93445470a94fa9eb7ecbd8168c175c19 Mon Sep 17 00:00:00 2001 From: BlivionIaG Date: Fri, 13 Nov 2026 15:12:10 +0100 Subject: [PATCH 05/18] fix(rdna2): share a LeapQuant normalize launch only within one KV-cache group Hybrid models spread their GDN layers over several KV-cache groups, each with its own block table, so a request sits at different state slots per group. The grouped normalize (8ba674ad8b) let the first GDN layer of the model normalize every GDN layer at its own group's slots: the other groups' layers committed records at foreign slots and never at their own. On flashnext-mtp2 (6 KV-cache groups) decode fell into repetition loops (10-21 of 32 outputs at c=8 vs 2-7 for the base) while prefill KL and PPL stayed inside the floor. Groups are now built per KV-cache group in bind_kv_cache_to_layers, next to share_replayssm_ring_trackers (same namespace rule); without the group map every layer keeps its own launch. Signed-off-by: BlivionIaG --- .../kernels/mamba/test_gdn_leapquant_rdna2.py | 58 ++++++++++ .../layers/mamba/gdn/qwen_gdn_linear_attn.py | 105 ++++++++++-------- vllm/v1/worker/utils.py | 9 +- 3 files changed, 123 insertions(+), 49 deletions(-) diff --git a/tests/kernels/mamba/test_gdn_leapquant_rdna2.py b/tests/kernels/mamba/test_gdn_leapquant_rdna2.py index a737e81d798c..0384da2bae09 100644 --- a/tests/kernels/mamba/test_gdn_leapquant_rdna2.py +++ b/tests/kernels/mamba/test_gdn_leapquant_rdna2.py @@ -269,6 +269,64 @@ def test_grouped_normalize_matches_per_layer(): assert folded > 0, "no fold exercised" +def test_normalize_groups_follow_kv_cache_groups(): + """Hybrid models spread their GDN layers over several KV-cache groups with + separate block tables, so one request sits at different slots per group. + The shared normalize launch must cover only its own group's layers: the + layers normalized group by group in model order must end exactly as + normalizing every layer on its own at its group's slots.""" + from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( + QwenGatedDeltaNetAttention, + share_leapquant_normalize_groups, + ) + from vllm.v1.kv_cache_interface import KVCacheGroupSpec + + L = _layout(3) + A_log, dt_bias = _params() + n, slots = 2, 5 + names = [f"model.layers.{i}.linear_attn" for i in range(4)] + # group 0 = layers 0, 2; group 1 = layers 1, 3; request r at different slots + group_of = {name: i % 2 for i, name in enumerate(names)} + group_idx = [ + torch.tensor([1, 3], dtype=torch.int32, device="cuda"), + torch.tensor([4, 2], dtype=torch.int32, device="cuda"), + ] + ctx = {} + for i, name in enumerate(names): + idx = group_idx[group_of[name]] + blob = torch.zeros(slots, HV, L.nbytes, dtype=torch.uint8, device="cuda") + _store(L, torch.randn(n, HV, V, K, device="cuda") * 0.5, blob, idx) + # 8 steps leave 14 committed + 3 pending records: the commit folds + for step in range(8): + q, k, v, a, b = _tokens(n, 3, seed=100 * i + step) + _normalize(L, blob, idx, torch.full((n,), 2, dtype=torch.int32).cuda()) + _decode(L, blob, idx, q, k, v, a, b, A_log, dt_bias) + layer = QwenGatedDeltaNetAttention.__new__(QwenGatedDeltaNetAttention) + torch.nn.Module.__init__(layer) + layer._lq_layout = L + layer.kv_cache = (None, blob) + ctx[name] = layer + groups = [ + KVCacheGroupSpec([n_ for n_ in names if group_of[n_] == g], None) + for g in range(2) + ] + share_leapquant_normalize_groups(names, ctx, groups) + assert ctx[names[0]]._lq_group is ctx[names[2]]._lq_group + assert ctx[names[0]]._lq_group is not ctx[names[1]]._lq_group + + num_acc = torch.tensor([3, 2], dtype=torch.int32, device="cuda") + ref = [] + for name in names: + blob = ctx[name].kv_cache[1].clone() + _normalize(L, blob, group_idx[group_of[name]], num_acc) + ref.append(blob) + for name in names: + layer = ctx[name] + layer._lq_normalize(layer.kv_cache[1], group_idx[group_of[name]], num_acc) + for name, want in zip(names, ref): + assert torch.equal(ctx[name].kv_cache[1], want), name + + def test_normalize_and_decode_replay_in_cuda_graph(): L = _layout(3) A_log, dt_bias = _params() diff --git a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py index 16e0b788d1a2..92320f8aab03 100644 --- a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @@ -5,7 +5,7 @@ import functools import itertools import os -from collections.abc import Callable +from collections.abc import Callable, Sequence from dataclasses import replace from typing import Any, Literal @@ -96,7 +96,7 @@ scatter_gdn_state_arenas, static_gdn_cache_slots, ) -from vllm.v1.kv_cache_interface import KVCacheSpec, MambaSpec +from vllm.v1.kv_cache_interface import KVCacheGroupSpec, KVCacheSpec, MambaSpec from vllm.v1.worker.workspace import current_workspace_manager # Optional ROCm AITER Triton kernels for the GDN decode path. @@ -128,46 +128,66 @@ class _LqNormalizeGroup: - """The LeapQuant GDN layers of one model on this rank. - - Every layer normalizes its blobs with the same state indices and accepted - counts each step (they share one attention metadata), so the first layer - of the group normalizes all of them in one launch and the others skip: - one launch per step instead of one per layer (~3 us each in a graph). - ``offsets`` holds each layer's paged blob address relative to the - leader's; it is rebuilt on every ``bind_kv_cache``, which never runs under + """The LeapQuant GDN layers of one KV-cache group on this rank. + + A KV-cache group shares one block table, so its layers normalize the same + state indices with the same accepted counts each step: the first layer + normalizes all of them in one launch and the others skip (one launch per + group and step instead of one per layer, ~3 us each in a graph). Layers + of different groups hold different slots for the same request and never + share a launch. ``offsets`` holds each layer's paged blob address relative + to the leader's; it is built at KV-cache bind, which never runs under capture, so captured graphs only ever see a finished table.""" - def __init__(self) -> None: - self.members: dict[int, QwenGatedDeltaNetAttention] = {} - self.leader: QwenGatedDeltaNetAttention | None = None - self.leader_ptr = 0 - self.offsets: torch.Tensor | None = None - - def rebuild(self) -> None: - layers = [self.members[i] for i in sorted(self.members)] - blobs = [layer.kv_cache[1] for layer in layers] - ref = blobs[0] - uniform = all( - b.device == ref.device - and b.dtype == ref.dtype - and b.shape == ref.shape - and b.stride() == ref.stride() - for b in blobs - ) - if len(layers) < 2 or not uniform: - self.leader, self.leader_ptr, self.offsets = None, 0, None - return + def __init__(self, layers: list["QwenGatedDeltaNetAttention"]) -> None: + ref = layers[0].kv_cache[1] self.leader = layers[0] self.leader_ptr = ref.data_ptr() self.offsets = torch.tensor( - [b.data_ptr() - self.leader_ptr for b in blobs], + [layer.kv_cache[1].data_ptr() - self.leader_ptr for layer in layers], dtype=torch.int64, device=ref.device, ) -_LQ_NORMALIZE_GROUPS: dict[str, _LqNormalizeGroup] = {} +def share_leapquant_normalize_groups( + ordered_layer_names: list[str], + forward_context: dict[str, Any], + kv_cache_groups: Sequence[KVCacheGroupSpec] | None = None, +) -> None: + """Give the LeapQuant GDN layers of each KV-cache group one normalize + launch per step (see ``_LqNormalizeGroup``). Hybrid models spread their + GDN layers over several KV-cache groups, each with its own block table; + without the group map every layer keeps its own launch.""" + layer_to_group = { + name: group_idx + for group_idx, group in enumerate(kv_cache_groups or ()) + for name in group.layer_names + } + by_group: dict[int, list[QwenGatedDeltaNetAttention]] = {} + for name in ordered_layer_names: + layer = forward_context[name] + if ( + not isinstance(layer, QwenGatedDeltaNetAttention) + or layer._lq_layout is None + ): + continue + layer._lq_group = None + if name in layer_to_group: + by_group.setdefault(layer_to_group[name], []).append(layer) + for layers in by_group.values(): + ref = layers[0].kv_cache[1] + if len(layers) < 2 or not all( + b.device == ref.device + and b.dtype == ref.dtype + and b.shape == ref.shape + and b.stride() == ref.stride() + for b in (layer.kv_cache[1] for layer in layers) + ): + continue + group = _LqNormalizeGroup(layers) + for layer in layers: + layer._lq_group = group # Scratch buffers reused across GDN prefill calls. The key MUST carry a role @@ -1078,16 +1098,6 @@ def _init_leapquant(self, vllm_config: VllmConfig) -> LeapQuantLayout | None: ) return layout - def bind_kv_cache(self, kv_cache: torch.Tensor) -> None: - super().bind_kv_cache(kv_cache) - if self._lq_layout is None: - return - root = self.prefix.rsplit(".layers.", 1)[0] - group = _LQ_NORMALIZE_GROUPS.setdefault(root, _LqNormalizeGroup()) - group.members[self.layer_idx] = self - group.rebuild() - self._lq_group = group - def _init_gdn_state_arenas(self) -> None: """Permanent arenas on the layer device, before first BeginCapture.""" conv_shape, ssm_shape = self.get_state_shape()[:2] @@ -2251,21 +2261,20 @@ def _lq_normalize( ``None``) and fold full windows. Runs before every decode / verify launch, the MonoKernel one included, and before prefill reads. - On the paged blobs the group leader normalizes every layer of the - group in one launch and the other layers skip; decode arenas are - per-layer copies, so they are normalized per layer.""" + On the paged blobs the leader of the layer's KV-cache group normalizes + every layer of that group in one launch and the others skip; decode + arenas are per-layer copies, so they are normalized per layer.""" layout = self._lq_layout assert layout is not None group = self._lq_group offsets = None - if group is not None and group.leader is not None and blob is self.kv_cache[1]: + if group is not None and blob is self.kv_cache[1]: if group.leader is not self: return assert blob.data_ptr() == group.leader_ptr, "stale LeapQuant group" offsets = group.offsets - assert offsets is not None logger.info_once( - "LeapQuant GDN: one normalize launch covers %d layers", + "LeapQuant GDN: one normalize launch per KV-cache group (%d layers)", offsets.numel(), ) torch.ops._rocm_C.gdn_lq_normalize_rdna2( diff --git a/vllm/v1/worker/utils.py b/vllm/v1/worker/utils.py index 99903e990833..a9ba4da8a9f5 100644 --- a/vllm/v1/worker/utils.py +++ b/vllm/v1/worker/utils.py @@ -13,6 +13,9 @@ from vllm.config import CacheConfig from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention +from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( + share_leapquant_normalize_groups, +) from vllm.model_executor.layers.mamba.mamba_mixer2 import share_replayssm_ring_trackers from vllm.model_executor.layers.utils import warmup_rocm_skinny_gemm_workspaces from vllm.model_executor.models.interfaces import MultiModalEmbeddings @@ -885,7 +888,8 @@ def bind_kv_cache_to_layers( num_attn_module: int = 1, kv_cache_groups: Sequence[KVCacheGroupSpec] | None = None, ) -> None: - """Bind layer caches and share ReplaySSM trackers in model-layer order.""" + """Bind layer caches, then share ReplaySSM trackers and LeapQuant GDN + normalize launches per KV-cache group, in model-layer order.""" # Bind kv_caches to forward context. Each layer's bind_kv_cache unpacks # its raw allocation into the per-layer view(s) it needs (e.g. Mamba # splits conv/ssm), so the kv_caches dict can hold a single tensor per @@ -897,6 +901,9 @@ def bind_kv_cache_to_layers( kv_caches, key=lambda name: extract_layer_index(name, num_attn_module) ) share_replayssm_ring_trackers(ordered_layer_names, forward_context, kv_cache_groups) + share_leapquant_normalize_groups( + ordered_layer_names, forward_context, kv_cache_groups + ) def clear_layer_kv_caches(layers: Iterable[Any]) -> None: From 7984d42825b189269a51c9bb50c21598a6daf5be Mon Sep 17 00:00:00 2001 From: BlivionIaG Date: Fri, 13 Nov 2026 22:26:12 +0100 Subject: [PATCH 06/18] test(rdna2): cover LeapQuant normalization in padded KV pages [skip ci] Signed-off-by: BlivionIaG --- .../kernels/mamba/test_gdn_leapquant_rdna2.py | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/tests/kernels/mamba/test_gdn_leapquant_rdna2.py b/tests/kernels/mamba/test_gdn_leapquant_rdna2.py index 0384da2bae09..e81f1ef84bc5 100644 --- a/tests/kernels/mamba/test_gdn_leapquant_rdna2.py +++ b/tests/kernels/mamba/test_gdn_leapquant_rdna2.py @@ -230,17 +230,31 @@ def test_trajectory_tracks_fp32_with_rejections(spec): ) -def test_grouped_normalize_matches_per_layer(): - """One launch over a group of layers (layer_offsets) must leave every - layer's blobs exactly as normalizing each layer on its own does, folds - and NULL slots included.""" +@pytest.mark.parametrize("page_kib", [None, 512, 1024]) +def test_grouped_normalize_matches_per_layer(page_kib): + """Grouped normalization must match per-layer normalization even with + padded KV pages, without writing the conv state or unused page bytes.""" L = _layout(3) A_log, dt_bias = _params() n, layers = 3, 3 idx = torch.tensor([1, 0, 2], dtype=torch.int32, device="cuda") blobs = [] + pages = [] for layer in range(layers): - blob = torch.zeros(n, HV, L.nbytes, dtype=torch.uint8, device="cuda") + if page_kib is None: + blob = torch.zeros(n, HV, L.nbytes, dtype=torch.uint8, device="cuda") + else: + page = torch.full( + (n, page_kib * 1024), 165, dtype=torch.uint8, device="cuda" + ) + # The recurrent state follows the conv window in a unified page. + blob = page.as_strided( + (n, HV, L.nbytes), + (page.stride(0), L.nbytes, 1), + storage_offset=25 * 1024, + ) + blob.zero_() + pages.append(page) blob[0].fill_(7) _store(L, torch.randn(n, HV, V, K, device="cuda") * 0.5, blob, idx) # 8 steps commit 7 * 2 = 14 records; committing 3 (request 0) or 2 @@ -267,6 +281,10 @@ def test_grouped_normalize_matches_per_layer(): cursor = L.views(got[1:])["header"][..., 0] folded += int((cursor < 8).sum()) assert folded > 0, "no fold exercised" + for page in pages: + assert (page[:, : 25 * 1024] == 165).all(), "conv state overwritten" + end = 25 * 1024 + HV * L.nbytes + assert (page[:, end:] == 165).all(), "page padding overwritten" def test_normalize_groups_follow_kv_cache_groups(): From 433e745d7428dff0089c4857564e2962d2715bde Mon Sep 17 00:00:00 2001 From: BlivionIaG Date: Fri, 13 Nov 2026 22:28:41 +0100 Subject: [PATCH 07/18] docs(rdna2): record fixed LeapQuant receipt and acceptance caveat [skip ci] Signed-off-by: BlivionIaG --- docs/rdna2/leapquant-gdn-2026-11-13.md | 228 +++++++++++++++++++++++++ 1 file changed, 228 insertions(+) diff --git a/docs/rdna2/leapquant-gdn-2026-11-13.md b/docs/rdna2/leapquant-gdn-2026-11-13.md index 6d1aca48ddc0..2986bd1ec3f2 100644 --- a/docs/rdna2/leapquant-gdn-2026-11-13.md +++ b/docs/rdna2/leapquant-gdn-2026-11-13.md @@ -78,3 +78,231 @@ bash tools/rdna/serve_rdna.sh RECIPE=flashnext-mtp2 VLLM_RDNA_GDN_LEAPQUANT=1 or uncomment `VLLM_RDNA_GDN_LEAPQUANT=1` in `tools/rdna/recipes/flashnext-mtp2.env`. The worker log prints `GDN recurrent state: LeapQuant INT8 (...)`. + +## Results (flashnext-mtp2, 4x V620 TP4, 2026-11-13) + +Base = flag off, head = flag on, same build (`_rocm_C` md5 identical; the +trees differ only in the recipe line). Receipts: +`~/w4a8_runs/leapquant/speed-20261113-023315` (MonoKernel off) and +`speed-20261113-062600-mono` (MonoKernel on, `EXTRA_ARGS=--monokernel`), each +`pr_receipt.sh` ABBA over 1k/512, 16k/1k, 32k/2k x c=1,4,8, plus the quality +boots `quality-20261113-005043` (on/off/off2) and +`quality-20261113-013837-mono` (on/off). + +### Quality: gated checks inside the floor; decode-path KL needs more samples + +| check (MonoKernel off) | off -> on | floor (off -> off2) | +|---|---|---| +| `topk_kl` docs, 64 windows: KL / top-1 | 0.01198, 0.01202 / 0.967 | 0.01168 [upper CI 0.01394] / 0.968 | +| long-context PPL (6 x ~15k tokens) | -0.01 %, +0.15 % | +0.10 % | +| decode-path KL (`decode_kl.py`), 32 x 512 tokens | 4.19e-3; 6.94e-3 | 4.77e-3 | +| decode-path KL, 8 x 2048 tokens at 15-22k context | 8.2e-3; 1.31e-2 | 7.3e-3 | +| concurrency probe / greedy / prefix cache | 0 FAIL / 3/3 / 3.1x | same | + +The two decode-path KL figures per row are two on/off boot pairs of the same +build (`quality-20261113-005043`, `quality-20261113-101622`). The second +pair had no off2 boot, so it has no floor of its own. The metric averages +only the ~800 (512) / ~170 (2048) positions before the greedy paths +diverge, so it is heavy-tailed (p99 0.14-0.18 nats). That one build gives +4.2e-3 and 6.9e-3 means that a single pair does not resolve a 1.5x +difference. In the second pair: long-context PPL -0.25 %; gsm8k / needle / +humaneval (160 each) 96.2 / 100 / 96.2 % -> 96.2 / 100 / 96.9 % +(pooled 97.5 -> 97.7 %); PPL probe 4.81 -> 4.83; concurrency 0 FAIL. + +MonoKernel on, off -> on: decode-path KL 4.84e-3 (512) and 7.7e-3 (2048), +long-context PPL +0.23 %, gsm8k / needle / humaneval (160 each, pooled 480) +97.7 % -> 97.7 %, concurrency 0 FAIL, prefix cache 3.2x. The worker log shows +`mono gdn_rdna2: verify N rows -> fused kernel` with the flag on. + +### Memory: no KV capacity gain on this recipe + +KV capacity is 266,945 tokens with the flag on and off. With +`BLOCK_SIZE=1024` the attention page sets the unified page size: vLLM pads +the GDN page by 29.13 % (FP32 state) or by 182.78 % (LeapQuant) to match it, +so the smaller state becomes padding and frees no KV blocks. One block, one +GDN layer, one rank: + +| content | FP32 state | LeapQuant | +|---|---:|---:| +| recurrent state, 12 heads | 768 KiB | 337.1 KiB | +| conv window (2560 ch x 5 taps, fp16) | 25 KiB | 25 KiB | +| padding to the attention page | 231 KiB | 661.9 KiB | +| total (= attention page, 1024 tokens) | 1024 KiB | 1024 KiB | + +The attention bytes per token stay constant when its page and block size +shrink together. The fixed per-request Mamba reservation does not: smaller +pages reduce its byte cost. LeapQuant makes a 512-token page feasible; the +793 KiB FP32 state cannot fit it. + +`MambaSpec.max_memory_usage_bytes` reserves `2 + num_speculative_blocks + +num_prefill_checkpoint_blocks` pages in align mode. This HIP GDN path has +no internal prefill checkpoint exporter (`_checkpoint_alignment=None`), so +the reservation is **4 pages per GDN group**, not 5 (MTP2, checkpoint +blocks=0). For the 3 groups that is 12 pool blocks per request. An actual +CPU-only `MambaSpec` calculation gives 4 MiB per layer at block 1024 and +2 MiB at block 512; state contents are 812,032 and 370,816 bytes. +The LeapQuant kernel suite passes 13 tests, including grouped normalize +on 512 KiB and 1 MiB padded pages. These cases check equality with dense +per-layer normalization and preservation of neighboring conv/padding bytes. + +LeapQuant recurrence and conv update use the home block +(`spec_state_indices[:, 0]`). Removing speculative blocks is separate work: +verify lengths currently depend on the index width, and V2 requires all +Mamba groups to share speculative-block scheduling parameters. Neither is +changed by the 512 experiment. + +The 512-token experiment is included in this PR (user decision 2026-11-13), +with MTP2 and FULL_AND_PIECEWISE unchanged. The existing launcher accepts +`BLOCK_SIZE=512`; runtime bring-up and ABBA validation are pending. +Packing two LeapQuant layers into one page is reserved for a second PR. +That could reduce cached-prefix state bytes without shortening the +checkpoint interval, but needs allocator changes. + +### Speed (first build): no gain + +MonoKernel off, head vs base mean (ABBA verdict in parentheses where both +head boots fall on one side of both base boots): + +| cell | TTFT p50 | prefill aggregate | ITL p50 | ITL p99 | output aggregate | +|---|---:|---:|---:|---:|---:| +| 1k/512 c=1 | +0.5 % | -0.2 % | -0.9 % (win) | +5.7 % (regr.) | -1.0 % | +| 1k/512 c=4 | -0.5 % | +0.0 % | -0.1 % | -72.2 % | +1.1 % | +| 1k/512 c=8 | -0.1 % | -0.1 % | -0.6 % | +0.1 % | -0.9 % | +| 16k/1k c=1 | -0.2 % | +0.1 % | -0.4 % | +5.9 % (regr.) | -2.0 % | +| 16k/1k c=4 | -0.9 % | -0.0 % | +1.3 % (regr.) | +0.4 % | +0.9 % | +| 16k/1k c=8 | +24.9 % (regr.) | -0.1 % | +1.4 % (regr.) | +0.1 % | +4.8 % (win) | +| 32k/2k c=1 | -0.2 % (win) | +0.3 % (win) | -1.2 % (win) | +5.7 % (regr.) | -3.4 % | +| 32k/2k c=4 | +0.4 % | -0.1 % | -0.8 % (win) | +0.0 % | -1.0 % | +| 32k/2k c=8 | -6.2 % (win) | -2.3 % (regr.) | +1.2 % (regr.) | +0.3 % | +1.2 % | + +MonoKernel on, same columns: + +| cell | TTFT p50 | prefill aggregate | ITL p50 | ITL p99 | output aggregate | +|---|---:|---:|---:|---:|---:| +| 1k/512 c=1 | -0.4 % | +1.0 % | +0.1 % | +6.5 % (regr.) | -3.5 % | +| 1k/512 c=4 | -0.7 % (win) | -0.2 % (regr.) | +0.0 % | -71.9 % | -1.2 % | +| 1k/512 c=8 | +0.0 % | -0.4 % | +0.3 % | +0.1 % | -4.1 % (regr.) | +| 16k/1k c=1 | -0.1 % (win) | -0.0 % | +0.3 % | +6.5 % (regr.) | +1.1 % | +| 16k/1k c=4 | -2.7 % (win) | -0.0 % | -0.5 % | +0.3 % (regr.) | -3.1 % (regr.) | +| 16k/1k c=8 | -9.3 % | -0.1 % | -0.5 % | +0.0 % | -2.7 % | +| 32k/2k c=1 | -0.2 % (win) | +0.2 % (win) | +0.3 % (regr.) | +6.1 % (regr.) | -4.4 % (regr.) | +| 32k/2k c=4 | +0.1 % | -0.0 % | +0.5 % | +0.3 % (regr.) | +3.5 % | +| 32k/2k c=8 | +2.7 % (regr.) | -9.5 % (regr.) | +0.9 % (regr.) | +0.1 % (regr.) | -3.9 % | + +* Output aggregate follows MTP acceptance (1.68-2.60 per boot), not the + flag. +* **c=1 ITL p99 +6 %** (23.6 -> 25.0 ms) is the fold: all 36 layers + refold on the same step every ~16 committed tokens, one layer after the + other. +* 16k/1k c=8 TTFT p50 is bimodal per boot (MonoKernel-on base boots: 11.7 s + and 7.0 s); prefill aggregate does not move. + +### Efficiency pass: one normalize launch per KV-cache group + +Per decode step on one V620 (36 layers, HIP-graph replay, +`~/w4a8_runs/leapquant/mbench_mono.py`), the first build spent, at n=1: +~114 us on 36 normalize launches (~3 us each, mostly launch gaps), ~100 us +on folds (40 us per fold workgroup, one layer after the other) and ~55 us on +replaying the window's records. + +* **Grouped normalize.** The GDN layers of one KV-cache group share a block + table and one attention metadata, so the group's first LeapQuant layer + normalizes all of its layers' blobs in one launch (`layer_offsets`, grid.y + = layer) and the others skip. The groups are built at KV-cache bind + (`share_leapquant_normalize_groups`, next to + `share_replayssm_ring_trackers`). Flash-Next TP4 has 3 GDN groups of 12 + layers: 3 launches per step instead of 36, and each group's folds run + concurrently instead of back to back. Decode arenas (non-spec pure + decode) stay per layer. Same bytes as per-layer normalize + (`test_grouped_normalize_matches_per_layer`, + `test_normalize_groups_follow_kv_cache_groups`). +* **Replay loads.** The record replay loops unroll by 2 so two records' + loads issue together (unroll 4 was slower at n=8). Same arithmetic order. +* Tried and dropped: LDS instead of shuffle reductions in the fold (40 us -> + 39 us at n=1, 96 us -> 110 us at n=8). Power iterations 2 -> 0 saves 20-40 + us per step but changes the compensators; not taken, quality not measured. + +| step | FP32 MonoKernel | LeapQuant, first build | LeapQuant, efficiency pass | +|---|---:|---:|---:| +| decode n=1 | 341 us | 550 us | 368 us | +| MTP verify n=1 (acc 2) | 520 us | 838 us | 589 us | +| decode n=8 | 1503 us | 1290 us | 1135 us | +| MTP verify n=8 (acc 2) | 3289 us | 2147 us | 1857 us | + +(`GROUP=0` vs `GROUP=3`. One launch over all 36 layers was 5-8 us faster, +but it is wrong: see below.) + +Prefill adds materialize + store at chunk boundaries: 1.9 ms per +2048-token chunk (`mbench_prefill.py`). + +#### Bug found in the first efficiency build: one group across KV-cache groups + +The first version (`8ba674ad8b`) grouped every GDN layer of the model under +the first one. vLLM's hybrid allocator spreads Flash-Next's 36 GDN layers +over 3 KV-cache groups (`kv cache group sizes (1024, 1024, 1024, 1024, 8, +1024)`), each with its own block table, so one request sits at different +state slots per group. The leader committed the other groups' records at +its own group's slots, which belong to other requests in those groups. +Their own slots were never committed. + +* Symptom: decode fell into repetition loops, which MTP accepts easily. + The MonoKernel-on ABBA receipt showed fake speed "wins" (c=1 output + +37 %, ITL p50 -4 to -8 % at c=4/8). Outputs whose last 600 chars compress + below 0.12 (zlib), out of 32 at c=8: base boots 2-7, flag-on boots 10-21. + On the first build, the flag-on boots matched base (1-6). +* The gated checks missed it. Prompt KL and teacher-forced PPL do not run + decode (head KL 0.01178/0.01172 vs floor CI 0.01413, PPL +0.47/+0.15 %). + The concurrency probe hinted at it: 1 FAIL and 10 soft divergences, vs + 0-5 soft on every other boot. +* Isolation: the tree3 and tree2 kernels are bitwise identical over 200 + MTP steps (decode, normalize, MonoKernel). Disabling only the Python + grouping restores base behaviour. +* Fix: `d879da8c93` builds one group per KV-cache group. + `test_normalize_groups_follow_kv_cache_groups` uses two groups with + different slots per request. + +### Speed (efficiency build `d879da8c93`, MonoKernel on) + +ABBA receipt `~/w4a8_runs/leapquant/speed-20261113-152500-mono`, same +columns and rule as above: + +| cell | TTFT p50 | prefill aggregate | ITL p50 | ITL p99 | output aggregate | +|---|---:|---:|---:|---:|---:| +| 1k/512 c=1 | +0.7 % (regr.) | +0.1 % | -1.0 % (win) | -0.6 % (win) | -19.3 % (regr.) | +| 1k/512 c=4 | -0.8 % (win) | -0.1 % | +0.1 % | +258.3 % | +6.2 % (win) | +| 1k/512 c=8 | -1.7 % (win) | +0.3 % (win) | -5.4 % (win) | -0.3 % (win) | -1.3 % | +| 16k/1k c=1 | -0.1 % | +0.1 % (win) | -1.2 % (win) | -0.3 % | -6.8 % | +| 16k/1k c=4 | +0.7 % | +0.1 % | -0.8 % | +0.0 % | +3.1 % | +| 16k/1k c=8 | -2.3 % | +0.1 % | -1.4 % (win) | -0.2 % (win) | -1.4 % | +| 32k/2k c=1 | -0.2 % (win) | +0.2 % (win) | -0.1 % | +0.3 % (regr.) | +7.0 % (win) | +| 32k/2k c=4 | +1.5 % (regr.) | +0.0 % | -1.0 % | -0.1 % | +1.4 % | +| 32k/2k c=8 | +1.7 % | +2.1 % (win) | +0.1 % | -0.1 % (win) | +4.6 % (win) | + +* **The c=1 ITL p99 fold spike is gone** (+6 % in the first build, now + -0.6 / -0.3 / +0.3 %): each KV-cache group's folds run in one launch. +* ITL p50 moves -0.1 to -1.4 %, and -5.4 % at 1k/512 c=8 (45.2 vs 47.8 ms). +* At 1k/512 c=1 (4 requests), acceptance length from each benchmark JSON + is A1=2.034, B1=1.774, B2=1.622, A2=2.269 tokens per step. Both flag-on + boots have lower acceptance and output throughput despite similar ITL. + The -19.3 % throughput result remains a regression to investigate. + Other cells move in both directions; pooling their acceptance logs + cannot establish parity for this cell. +* 1k/512 c=4 ITL p99 +258 % is one boot (B2 380.9 ms vs 60.9-62.6 ms on the + others). The first build had the same stall on a base boot (-72 %). +* At 32k/2k c=8 every arm logs `Running: 5-6 reqs, Waiting: 2-3 reqs` + (TTFT p50 ~85 s). This is consistent with KV admission pressure + [INFERENCE]; the 512 experiment will check admission and throughput. +* Gates: quality PASS (head KL 0.01158 / 0.01178 vs floor 0.01376, top-1 + 0.967 vs 0.964, PPL +0.055 / -0.077 %), concurrency probe PASS with 0 + failing on both flag-on boots, stall PASS. Repetition loops (`loopscan.py`, + out of 32 per c=8 cell): base 0-5, flag on 1-5. + +### Triton census + +`VLLM_TRITON_LAUNCH_COUNT=1` over the quality boots: the flag-on and +flag-off servers launch the same Triton kernels (MonoKernel off: 10, +MonoKernel on: 12). LeapQuant adds none; every LeapQuant op is HIP. The +baseline ones are the recipe's (`_causal_conv1d_update/_fwd` because the +recipe sets `VLLM_CAUSAL_CONV1D_RDNA2_*=0`, `_mamba_align_block_table_kernel`, +`_qsa_mqa_paged_kernel`, the PLE kernels, and under the MonoKernel +`_fused_qk_rmsnorm_rope_gate_kernel` and `_num_nans_kernel`). From 3c3f6120c165515531bd88642c2176211da5252f Mon Sep 17 00:00:00 2001 From: BlivionIaG Date: Fri, 13 Nov 2026 23:08:04 +0100 Subject: [PATCH 08/18] docs(rdna2): record LeapQuant MonoKernel-off efficiency receipt [skip ci] Signed-off-by: BlivionIaG --- docs/rdna2/leapquant-gdn-2026-11-13.md | 28 ++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/rdna2/leapquant-gdn-2026-11-13.md b/docs/rdna2/leapquant-gdn-2026-11-13.md index 2986bd1ec3f2..478e63f042a2 100644 --- a/docs/rdna2/leapquant-gdn-2026-11-13.md +++ b/docs/rdna2/leapquant-gdn-2026-11-13.md @@ -297,6 +297,34 @@ columns and rule as above: failing on both flag-on boots, stall PASS. Repetition loops (`loopscan.py`, out of 32 per c=8 cell): base 0-5, flag on 1-5. +### Speed (efficiency build `d879da8c93`, MonoKernel off) + +Receipt `~/w4a8_runs/leapquant/speed-20261113-191345`, completed 23:05. +Quality, overlap, stall hard gate and all boots pass; no invalid cells. +Speed is mixed (5 winning cells, 4 with at least one regression): + +| cell | TTFT p50 | prefill aggregate | ITL p50 | ITL p99 | output aggregate | +|---|---:|---:|---:|---:|---:| +| 1k/512 c=1 | -1.0 % | +2.5 % | -1.3 % | -1.8 % | +5.9 % | +| 1k/512 c=4 | -0.7 % | +0.1 % | -0.8 % | +267.9 % | +15.0 % | +| 1k/512 c=8 | -0.5 % | +0.4 % | -3.5 % | -0.2 % | +2.8 % | +| 16k/1k c=1 | -0.2 % | +0.1 % | -1.1 % | -1.1 % | +8.8 % | +| 16k/1k c=4 | -0.3 % | +0.2 % | -2.5 % | -0.1 % | -2.0 % | +| 16k/1k c=8 | +0.8 % | +0.1 % | -2.3 % | -0.1 % | -1.5 % | +| 32k/2k c=1 | -0.2 % | +0.2 % | -0.9 % | -0.6 % | +5.2 % | +| 32k/2k c=4 | -0.9 % | +0.0 % | -1.2 % | +0.0 % | +1.4 % | +| 32k/2k c=8 | +3.3 % | -0.6 % | -2.0 % | -0.1 % | -7.2 % | + +* Both head boots pass KL against the floor (0.01186/0.01205 vs 0.01490), + PPL (+0.121/+0.350 %, limit 2 %) and concurrency (0 failing). +* At 1k c=1, acceptance length is A1=1.814, B1=1.865, B2=1.909, + A2=1.787: the MonoKernel-on acceptance regression does not reproduce + in this MonoKernel-off receipt. This does not establish the cause. +* Loop scan: c=8 base 2-5/32, head 1-4/32. At 16k c=1 and 32k c=1, + B1 flags 2/4 while the other boots flag 0/4; retain these as quality + concerns for the larger decode study rather than treating gate PASS + as complete decode-quality evidence. + ### Triton census `VLLM_TRITON_LAUNCH_COUNT=1` over the quality boots: the flag-on and From e20053608db5412e804eb51b843eb8bfa5e2d596 Mon Sep 17 00:00:00 2001 From: BlivionIaG Date: Sat, 14 Nov 2026 00:05:12 +0100 Subject: [PATCH 09/18] docs(rdna2): document LeapQuant 512-page smoke and throughput trade-offs [skip ci] Signed-off-by: BlivionIaG --- docs/rdna2/leapquant-gdn-2026-11-13.md | 42 ++++++++++++++++++++++---- tools/rdna/recipes/flashnext-mtp2.env | 3 ++ 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/docs/rdna2/leapquant-gdn-2026-11-13.md b/docs/rdna2/leapquant-gdn-2026-11-13.md index 478e63f042a2..04ff479e5dba 100644 --- a/docs/rdna2/leapquant-gdn-2026-11-13.md +++ b/docs/rdna2/leapquant-gdn-2026-11-13.md @@ -79,6 +79,13 @@ or uncomment `VLLM_RDNA_GDN_LEAPQUANT=1` in `tools/rdna/recipes/flashnext-mtp2.env`. The worker log prints `GDN recurrent state: LeapQuant INT8 (...)`. +Experimental 512-token pages (MTP2 stays enabled; throughput trade-offs below): + +```bash +bash tools/rdna/serve_rdna.sh RECIPE=flashnext-mtp2 \ + VLLM_RDNA_GDN_LEAPQUANT=1 BLOCK_SIZE=512 EXTRA_ARGS=--monokernel +``` + ## Results (flashnext-mtp2, 4x V620 TP4, 2026-11-13) Base = flag off, head = flag on, same build (`_rocm_C` md5 identical; the @@ -114,7 +121,7 @@ long-context PPL +0.23 %, gsm8k / needle / humaneval (160 each, pooled 480) 97.7 % -> 97.7 %, concurrency 0 FAIL, prefix cache 3.2x. The worker log shows `mono gdn_rdna2: verify N rows -> fused kernel` with the flag on. -### Memory: no KV capacity gain on this recipe +### Memory at block 1024: no KV capacity gain KV capacity is 266,945 tokens with the flag on and off. With `BLOCK_SIZE=1024` the attention page sets the unified page size: vLLM pads @@ -152,11 +159,34 @@ Mamba groups to share speculative-block scheduling parameters. Neither is changed by the 512 experiment. The 512-token experiment is included in this PR (user decision 2026-11-13), -with MTP2 and FULL_AND_PIECEWISE unchanged. The existing launcher accepts -`BLOCK_SIZE=512`; runtime bring-up and ABBA validation are pending. -Packing two LeapQuant layers into one page is reserved for a second PR. -That could reduce cached-prefix state bytes without shortening the -checkpoint interval, but needs allocator changes. +with MTP2 and FULL_AND_PIECEWISE unchanged. Packing two LeapQuant layers +into one page remains reserved for a second PR. + +### 512-token blocks: bring-up passes, speed not yet qualified + +Paired smoke run `~/w4a8_runs/leapquant/b512-20261113-230540`, one +1024-token and one 512-token boot, LeapQuant + MonoKernel on in both. +Runtime group sizes resolve to `(512, 512, 512, 512, 8, 512)`, not a fallback. +Padding is 41.39 % of state contents. Capacity rises from 266,945 to +275,523 reported tokens (+3.21 %); this is the max-context-equivalent +capacity metric, not the concurrency gain for every context length. + +All c=8 cells complete 32 requests; the 512 concurrency probe passes +(0 failing, 0 soft), after-load probe passes, no KFD eviction increments, +no fault markers and no new amdgpu dmesg faults. Loop scan, 1024 -> 512: +1k 3/32 -> 5/32, 16k 2/32 -> 1/32, 32k 5/32 -> 3/32. + +| cell | TTFT p50, 1024 -> 512 | ITL p50, 1024 -> 512 | output tok/s, 1024 -> 512 | acceptance length, 1024 -> 512 | +|---|---:|---:|---:|---:| +| 1k/512 c=8 | 0.539 -> 0.675 s | 46.31 -> 45.74 ms | 277.14 -> 260.55 | 2.123 -> 2.042 | +| 16k/1k c=8 | 8.363 -> 7.007 s | 48.59 -> 47.14 ms | 115.75 -> 106.18 | 2.353 -> 2.103 | +| 32k/2k c=8 | 85.272 -> 65.793 s | 43.91 -> 46.36 ms | 83.41 -> 64.66 | 2.288 -> 2.459 | + +The 32k TTFT improves 22.8 %, but output throughput falls 22.5 % despite +higher acceptance. ITL p99 rises from 435.96 to 648.89 ms (+48.8 %). +This is a capacity/latency trade-off, not a clean speed win. The full +ABBA matrix is running at `speed512-20261114-000314-mono`; do not promote +512 as the default from this single pair. ### Speed (first build): no gain diff --git a/tools/rdna/recipes/flashnext-mtp2.env b/tools/rdna/recipes/flashnext-mtp2.env index 63facabf6116..683d59d662f6 100644 --- a/tools/rdna/recipes/flashnext-mtp2.env +++ b/tools/rdna/recipes/flashnext-mtp2.env @@ -90,3 +90,6 @@ TUNED_CONFIG=1 # LeapQuant INT8 GDN recurrent state (arXiv 2609.38166): opt-in, lossy. See # docs/rdna2/leapquant-gdn-2026-11-13.md for the on/off receipt. # VLLM_RDNA_GDN_LEAPQUANT=1 +# Experimental smaller pages: pass VLLM_RDNA_GDN_LEAPQUANT=1 BLOCK_SIZE=512. +# MTP=2 and FULL_AND_PIECEWISE stay enabled. See the doc above for the +# capacity/throughput trade-off; keep the default block size at 1024. From 66c3acaff0fafebc3e495856cc1c3bb0342f8328 Mon Sep 17 00:00:00 2001 From: BlivionIaG Date: Sat, 14 Nov 2026 01:02:01 +0100 Subject: [PATCH 10/18] docs(rdna2): diagnose 512-page admission pressure and record benchmark pause [skip ci] Signed-off-by: BlivionIaG --- docs/rdna2/leapquant-gdn-2026-11-13.md | 67 ++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/docs/rdna2/leapquant-gdn-2026-11-13.md b/docs/rdna2/leapquant-gdn-2026-11-13.md index 04ff479e5dba..f1442a4c3ff1 100644 --- a/docs/rdna2/leapquant-gdn-2026-11-13.md +++ b/docs/rdna2/leapquant-gdn-2026-11-13.md @@ -184,9 +184,70 @@ no fault markers and no new amdgpu dmesg faults. Loop scan, 1024 -> 512: The 32k TTFT improves 22.8 %, but output throughput falls 22.5 % despite higher acceptance. ITL p99 rises from 435.96 to 648.89 ms (+48.8 %). -This is a capacity/latency trade-off, not a clean speed win. The full -ABBA matrix is running at `speed512-20261114-000314-mono`; do not promote -512 as the default from this single pair. +This is not a clean speed win. The full ABBA attempt at +`speed512-20261114-000314-mono` was stopped at the user's request before +the remaining arms completed. Its partial receipt is not an ABBA result; +queued KL/acceptance boots were cancelled. Results are preserved and +GPU testing remains paused. + +#### Slowdown investigation (2026-11-14, CPU diagnostics only) + +Reconstructed the saved streams, including every chunk's arrival time. +At 32k c=8, mean request ITL rises from 122.32 to 218.69 ms. Across the +merged streams, quiet intervals over 100 ms occur almost equally often +(1367 vs 1365), but their median rises 428 -> 635 ms and their summed +duration rises 575.04 -> 845.53 s. The whole benchmark grows by 227.87 s. +The extra time is concentrated in these long steps, not ordinary decode. + +Eight individual 512-mode streams pause for 27-55 s; other requests emit +282-2489 chunks during each pause. No merged-stream gap exceeds 0.661 s. +These are per-request starvation/replay signatures, not a whole-device +freeze. KFD eviction counters do not move. A QSA JIT warning exists, but +cannot explain the multi-request progress during the long individual gaps. +Absence of a preemption warning is not proof of no preemption: +`Scheduler._preempt_request` updates counters/events without logging one. + +CPU-only diagnostic: `~/w4a8_runs/leapquant/investigate_scheduler512.py`. +It drives the actual Scheduler and KV managers with representative six-group +specs and fixed two-token acceptance, using synchronous synthetic output +updates rather than the deployed async MTP runner. Pool counts 278/556 +model the reported capacities. Architecture metadata is supplied for config +construction; no GPU or model inference runs. These counts are a mechanism +reproduction, not measurements of deployed preemptions or GPU speed: + +| modeled setting | preemptions | prefill tokens processed | longest inter-output gap, scheduler steps | +|---|---:|---:|---:| +| 1024, 278 pool blocks, cap 8 | 6 | 1,232,896 | 1 | +| 512, 556 pool blocks, cap 8 | 17 | 1,552,896 | 687 | +| 512, same pool, cap 6 | 0 | 1,048,576 | 1 | +| 512, same pool, cap 8, watermark 5 % | 0 | 1,048,576 | 1 | +| 512, 600 pool blocks, cap 8 | 0 | 1,048,576 | 1 | + +All cases have the same 32 x 32768 unique input tokens (1,048,576); +extra tokens are repeated work. A cap of 7 behaves like cap 8. +A per-request prefill cap of 1024 alone does not solve it (18 preemptions): +multiple prefills can still fill a step. Watermarks 8 % and 10 % also +eliminate the reproduced preemptions. + +Likely live mechanism [INFERENCE]: smaller pages admit more long requests, +but full-input reservation does not reserve their future output growth. +Near cache saturation, a request is preempted and replays tokens while +others continue. Alignment also changes common mixed-prefill payloads +from 1024 to 1536 tokens, lengthening each prefill-carrying step. +The real scheduler's admission and checkpoint paths are unchanged by +LeapQuant, but the smaller block exposes a different operating point. + +Separate short-prompt cost: directly executing +`Scheduler._mamba_block_aligned_split` for a 1024-token prompt produces +`[1024]` at block 1024 and `[512, 512]` at block 512. The latter incurs +another prefill boundary/store-materialize cycle; this explains a concrete +source of additional startup work, not proof of the observed acceptance +difference's cause. + +Smallest next focused check, when testing resumes: existing `--watermark 0.05` +with block 512, keeping MTP2, cap 8 and FULL_AND_PIECEWISE unchanged, while +recording the actual preemption counter and per-step scheduling trace. +This is **not GPU-validated** and is not enabled in the recipe. ### Speed (first build): no gain From ff38538bccfd4792951fd501675f49c1df5fb7b4 Mon Sep 17 00:00:00 2001 From: BlivionIaG Date: Sat, 14 Nov 2026 03:26:49 +0100 Subject: [PATCH 11/18] fix(rdna2): reserve LeapQuant decode growth and in-flight state Signed-off-by: BlivionIaG --- docs/rdna2/leapquant-gdn-2026-11-13.md | 161 +++++++- tests/v1/core/test_leapquant_scheduling.py | 387 +++++++++++++++++++ vllm/v1/core/kv_cache_coordinator.py | 16 +- vllm/v1/core/kv_cache_manager.py | 30 +- vllm/v1/core/sched/scheduler.py | 77 +++- vllm/v1/core/single_type_kv_cache_manager.py | 25 ++ vllm/v1/kv_cache_interface.py | 23 +- 7 files changed, 688 insertions(+), 31 deletions(-) create mode 100644 tests/v1/core/test_leapquant_scheduling.py diff --git a/docs/rdna2/leapquant-gdn-2026-11-13.md b/docs/rdna2/leapquant-gdn-2026-11-13.md index f1442a4c3ff1..fcc35424b038 100644 --- a/docs/rdna2/leapquant-gdn-2026-11-13.md +++ b/docs/rdna2/leapquant-gdn-2026-11-13.md @@ -123,7 +123,7 @@ long-context PPL +0.23 %, gsm8k / needle / humaneval (160 each, pooled 480) ### Memory at block 1024: no KV capacity gain -KV capacity is 266,945 tokens with the flag on and off. With +The original on/off boots both reported 266,945 capacity tokens. With `BLOCK_SIZE=1024` the attention page sets the unified page size: vLLM pads the GDN page by 29.13 % (FP32 state) or by 182.78 % (LeapQuant) to match it, so the smaller state becomes padding and frees no KV blocks. One block, one @@ -141,13 +141,20 @@ shrink together. The fixed per-request Mamba reservation does not: smaller pages reduce its byte cost. LeapQuant makes a 512-token page feasible; the 793 KiB FP32 state cannot fit it. -`MambaSpec.max_memory_usage_bytes` reserves `2 + num_speculative_blocks + -num_prefill_checkpoint_blocks` pages in align mode. This HIP GDN path has -no internal prefill checkpoint exporter (`_checkpoint_alignment=None`), so -the reservation is **4 pages per GDN group**, not 5 (MTP2, checkpoint -blocks=0). For the 3 groups that is 12 pool blocks per request. An actual -CPU-only `MambaSpec` calculation gives 4 MiB per layer at block 1024 and +The original `MambaSpec.max_memory_usage_bytes` formula reserved +`2 + num_speculative_blocks + num_prefill_checkpoint_blocks` pages in align +mode: **4 pages per GDN group** at MTP2, checkpoint blocks=0 (12 planning +blocks for the 3 groups). That formula gave 4 MiB/layer at block 1024 and 2 MiB at block 512; state contents are 812,032 and 370,816 bytes. +HIP has no internal prefill checkpoint exporter (`_checkpoint_alignment=None`). + +This is not a bound on live asynchronous residency: two unresolved prefill +chunks can retain three home states plus two draft slots, **5 live pages** +per group. The extra page is a retained input state, not a checkpoint export. +LeapQuant startup sizing now matches the concurrency-aware admission bound: +5 MiB/layer at 1024 and 2.5 MiB at 512 with two unresolved batches. The +flag-off planning formula is unchanged. + The LeapQuant kernel suite passes 13 tests, including grouped normalize on 512 KiB and 1 MiB padded pages. These cases check equality with dense per-layer normalization and preservation of neighboring conv/padding bytes. @@ -187,8 +194,8 @@ higher acceptance. ITL p99 rises from 435.96 to 648.89 ms (+48.8 %). This is not a clean speed win. The full ABBA attempt at `speed512-20261114-000314-mono` was stopped at the user's request before the remaining arms completed. Its partial receipt is not an ABBA result; -queued KL/acceptance boots were cancelled. Results are preserved and -GPU testing remains paused. +queued KL/acceptance boots were cancelled. The full matrix and powered +quality study remain paused; only the targeted scheduling check below resumed. #### Slowdown investigation (2026-11-14, CPU diagnostics only) @@ -222,6 +229,8 @@ reproduction, not measurements of deployed preemptions or GPU speed: | 512, same pool, cap 6 | 0 | 1,048,576 | 1 | | 512, same pool, cap 8, watermark 5 % | 0 | 1,048,576 | 1 | | 512, 600 pool blocks, cap 8 | 0 | 1,048,576 | 1 | +| 1024, same pool/cap, bounded-growth admission | 0 | 1,048,576 | 1 | +| 512, same pool/cap, bounded-growth admission | 0 | 1,048,576 | 1 | All cases have the same 32 x 32768 unique input tokens (1,048,576); extra tokens are repeated work. A cap of 7 behaves like cap 8. @@ -234,8 +243,8 @@ but full-input reservation does not reserve their future output growth. Near cache saturation, a request is preempted and replays tokens while others continue. Alignment also changes common mixed-prefill payloads from 1024 to 1536 tokens, lengthening each prefill-carrying step. -The real scheduler's admission and checkpoint paths are unchanged by -LeapQuant, but the smaller block exposes a different operating point. +The original LeapQuant build leaves admission and checkpoint scheduling +unchanged; the smaller block exposes a different operating point. Separate short-prompt cost: directly executing `Scheduler._mamba_block_aligned_split` for a 1024-token prompt produces @@ -244,10 +253,132 @@ another prefill boundary/store-materialize cycle; this explains a concrete source of additional startup work, not proof of the observed acceptance difference's cause. -Smallest next focused check, when testing resumes: existing `--watermark 0.05` -with block 512, keeping MTP2, cap 8 and FULL_AND_PIECEWISE unchanged, while -recording the actual preemption counter and per-step scheduling trace. -This is **not GPU-validated** and is not enabled in the recipe. +#### Scheduling implementation (2026-11-14) + +The user rejected a watermark/cadence workaround. The opt-in LeapQuant +scheduler now changes admission and work ordering instead: + +* **Reserve bounded growth before admission.** The full-sequence check uses + prompt length + requested maximum output + an in-flight verification tail, + clamped to the model context limit. The tail includes every configured + concurrent batch, not just one final verify. Draft KV lookahead is separate. + Already admitted requests' unallocated growth is subtracted from free blocks. + These are allocator-derived promises, not eagerly allocated KV pages. +* **Account for physical recurrent peaks.** A future single-step allocation + alone undercounts Mamba align mode: initial state + two draft slots is three + pages. A synchronous boundary transition needs four; two unresolved + prefill chunks can keep five live pages. The peak bound includes + `1 + concurrent_batches + speculative_blocks + checkpoint_blocks`, + limited by the logical state columns. The coordinator takes the larger + of its normal prediction and this peak, subtracting held/adopted pages + and accounting for evictable prefix hits. +* **Ready decoders first under FCFS.** Their target and draft rows consume + the input budget before running prefills; prefills use what remains. + Explicit priority policy is unchanged. No sleep, watermark increase, + fixed sequence cap, cadence change, or reduction in speculative width. +* **No reservation ledger.** Growth is derived from live group tables. + Allocated pages stop being promises; early EOS and cancellation release + the request's remaining promise. Shared cached prefixes are not charged twice. + +Enabled only with the existing RDNA LeapQuant flag. Flag-off admission stays +unchanged; explicitly disabling `scheduler_reserve_full_isl` also disables +the new growth-admission gate. MTP2, speculative state blocks, +FULL_AND_PIECEWISE and prefix checkpoints are unchanged. +Declaring a very large `max_tokens` can now reduce admitted concurrency +even if EOS would arrive early: this is a worst-case growth promise, +not an output-length predictor. + +The retained 32-request CPU reproduction with the new policy, cap 8, no +prefill cap and zero watermark processes only the 1,048,576 unique prompt +tokens at both page sizes (last two rows above), with no preemptions and +one-step inter-output gaps. At 512 it takes 6,299 synthetic steps, including +695 prefill-carrying steps; these are not GPU timing measurements. + +`tests/v1/core/test_leapquant_scheduling.py`: 21 cases pass with real sync +and async scheduler/allocator objects and synthetic model outputs; the +two-in-flight test skips its inapplicable sync variant. Cases cover pressure +at both block sizes, recurrent peaks, early EOS/cancellation, prefix sharing, +full speculative width ahead of prefill, final verifies beyond the output +limit, context clamping, two unresolved verifies crossing a page boundary, +and retained input states across pipelined prefill chunks. CPU platform config +normally forces async off; the fixture preserves GPU concurrency metadata +without executing a model. +Five original cases failed before the scheduler change. Recurrent-peak and +two-batch-tail regressions each failed against the preceding implementation +before their accounting fixes. +A separate two-batch prefill reproduction (4k prompts, 8k batch budget, +per-request prefill cap 1024, 53 pool blocks) observes five live recurrent +states and two preemptions under the one-batch peak estimate. Concurrency-aware +accounting eliminates the preemptions; pools 53/54/60/61/65 all drain without +preemption. This is an allocator correctness check, not a new prefill-cap recipe. + +Startup sizing must use that same peak, or a cache can pass the initial +max-context check yet never admit a bounded full-context request. +The live-residency regression also checks that planned bytes cover the +observed five pages; it failed at 2 MiB planned versus 2.5 MiB live before +the sizing correction. A separate actual `check_enough_kv_cache_memory` +smoke at context 4096 requires 29 usable pages, rejects the old 26-page +total budget and accepts 30 total pages (including the null page). + +The broader cache/alignment run passes 261 existing cases; three fixture +failures also reproduce on unmodified `tree4` (circular-group Mamba mode, +and the two CPU-platform Flash-Next TP4 state-grid cases). They are unchanged. + +This fixes the reproduced admission/replay mechanism, not every prefill cost: +512-token checkpoints still split a 1k prompt into two forwards, and a +mixed-prefill step can still take longer than pure decode. No acceptance +parity or end-to-end speed improvement follows from CPU scheduling alone. + +#### Targeted live scheduling check (2026-11-14) + +Receipt `~/w4a8_runs/leapquant/scheduler512-final-20261114-025040`, +b4 HIP 2-5, resolved buses d9/dc/df/e2. Same 512-token pages, MTP2, +MonoKernel, FULL_AND_PIECEWISE, cap 8 and 32 x 32768/2048 requests +as the original 512 smoke. This measures the complete in-flight scheduling +policy (`tree6`), before the startup-sizing-only correction. It is not ABBA. + +| metric | original 512 smoke | complete pipeline admission + decoder-first | +|---|---:|---:| +| output tok/s | 64.66 | 104.13 | +| duration, s | 1013.56 | 629.36 | +| TTFT p50, s | 65.79 | 53.07 | +| mean request ITL, ms | 218.69 | 107.62 | +| ITL p50, ms | 46.36 | 45.08 | +| ITL p99, ms | 648.89 | 644.54 | +| longest inter-chunk gap after first token, s | 54.65 | 0.667 | +| MTP acceptance length | 2.459 | 2.325 | +| loop-scan flagged requests | 3/32 | 5/32 | + +All 32 complete. Fresh-boot Prometheus counter +`vllm:num_preemptions_total{engine="0",model_name="flash-next"} = 0`. +No request has an inter-chunk gap above one second; no KFD eviction +increments, GPU fault markers or new dmesg faults. Peak junction is 83 C. +Greedy, PPL (4.83), prefix reuse (2.9x), concurrency +(0 failing, 16 near-tie, 2 soft) and +after-load probes pass. + +Output throughput recovers despite lower acceptance in this boot; the long +request gaps disappear without reducing speculative width. +The ~644 ms p99 prefill-carrying steps remain. Loop-scan flags and the +unresolved controlled acceptance/decode-quality studies prevent a quality +or merge-readiness claim. + +Earlier policy snapshots, receipts `scheduler512-20261114-020245` and +`scheduler512-20261114-022815`, measured 102.48 and 105.79 output tok/s, +respectively; both also completed 32/32 with zero actual preemptions and +no inter-chunk request gap over one second. These are developmental checks, +not additional final-build repeats or an ABBA qualification. + +Final startup-only smoke: +`~/w4a8_runs/leapquant/scheduler512-sizing-20261114-031237` (`tree7`, +including the startup-sizing correction). Real 512-token groups remain +`(512, 512, 512, 512, 8, 512)` with the fixed 3.75 GiB KV budget. +Correctly accounting for retained states changes the reported capacity to +**273,456 tokens**, not 275,523 (+2.44 % versus the original 266,945-token +1024 baseline). This corrects the capacity estimate; no speed cell ran here. +Greedy, PPL 4.83, prefix reuse 2.9x, concurrency (0 failing, 16 near-tie, +1 soft), after-load and zero-preemption gates all pass. No KFD evictions +or new GPU faults; peak junction 68 C. b4 was released afterward. ### Speed (first build): no gain diff --git a/tests/v1/core/test_leapquant_scheduling.py b/tests/v1/core/test_leapquant_scheduling.py new file mode 100644 index 000000000000..a3f3222d8832 --- /dev/null +++ b/tests/v1/core/test_leapquant_scheduling.py @@ -0,0 +1,387 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Admission and decode progress with LeapQuant's smaller hybrid-cache pages.""" + +import pytest +import torch +from transformers import OPTConfig + +from vllm.platforms.cpu import CpuPlatform +from vllm.v1.core.sched.async_scheduler import AsyncScheduler +from vllm.v1.core.sched.scheduler import Scheduler +from vllm.v1.kv_cache_interface import ( + CircularBufferSpec, + FullAttentionSpec, + KVCacheGroupSpec, + MambaSpec, +) +from vllm.v1.outputs import DraftTokenIds, ModelRunnerOutput +from vllm.v1.request import RequestStatus + +from .utils import create_requests, create_scheduler + +pytestmark = pytest.mark.cpu_test + + +@pytest.fixture(params=[False, True], ids=["sync", "async"]) +def make_scheduler(tmp_path, monkeypatch, request): + async_scheduling = request.param + + class SchedulerOnlyPlatform(CpuPlatform): + @classmethod + def check_and_update_config(cls, vllm_config): + super().check_and_update_config(vllm_config) + # No model executes here. Keep the GPU scheduler's concurrency + # metadata instead of CPU's unconditional async=False override. + vllm_config.scheduler_config.async_scheduling = async_scheduling + + monkeypatch.setattr("vllm.platforms.current_platform", SchedulerOnlyPlatform()) + OPTConfig( + architectures=["OPTForCausalLM"], max_position_embeddings=262144 + ).save_pretrained(tmp_path) + + def make(*, hybrid=False, block_size=16, **kwargs): + class HybridScheduler(Scheduler): + def __init__(self, vllm_config, kv_cache_config, **scheduler_kwargs): + vllm_config.cache_config.mamba_cache_mode = "align" + vllm_config.cache_config.mamba_block_size = block_size + page_bytes = block_size * 1024 + state = MambaSpec( + block_size=block_size, + shapes=((2560, 5), (12, 28768)), + dtypes=(torch.float16, torch.uint8), + page_size_padded=page_bytes, + mamba_cache_mode="align", + num_speculative_blocks=2, + ) + kv_cache_config.kv_cache_groups = [ + KVCacheGroupSpec( + ["attn"], + FullAttentionSpec( + block_size=block_size, + num_kv_heads=2, + head_size=128, + dtype=torch.float16, + ), + ), + *[KVCacheGroupSpec([f"gdn{i}"], state) for i in range(3)], + KVCacheGroupSpec( + ["ring"], + CircularBufferSpec( + block_size=8, + num_kv_heads=2, + head_size=128, + dtype=torch.float16, + page_size_padded=page_bytes, + ), + ), + KVCacheGroupSpec(["ple"], state), + ] + super().__init__( + vllm_config=vllm_config, + kv_cache_config=kv_cache_config, + **scheduler_kwargs, + ) + + class HybridAsyncScheduler(HybridScheduler, AsyncScheduler): + """Use the same cache layout with real async placeholder accounting.""" + + hybrid_cls = HybridAsyncScheduler if async_scheduling else HybridScheduler + + scheduler = create_scheduler( + model=str(tmp_path), + skip_tokenizer_init=True, + enable_prefix_caching=True, + block_size=block_size, + scheduler_cls=hybrid_cls if hybrid else None, + async_scheduling=async_scheduling, + speculative_method=( + "ngram_gpu" if kwargs.get("num_speculative_tokens") else None + ), + **kwargs, + ) + # Exercise the policy with real allocators, without requiring a GPU or + # model weights. Production enables it only for RDNA LeapQuant. + scheduler._lq_decode_growth_admission = True + if hybrid: + scheduler.need_mamba_block_aligned_split = True + scheduler.mamba_state_block_size = block_size + return scheduler + + return make + + +def _update(scheduler, output, samples): + ids = list(output.num_scheduled_tokens) + scheduler.update_from_output( + output, + ModelRunnerOutput( + req_ids=ids, + req_id_to_index={rid: i for i, rid in enumerate(ids)}, + sampled_token_ids=[samples[rid] for rid in ids], + logprobs=None, + prompt_logprobs_dict={}, + pooler_output=[], + ), + ) + + +@pytest.mark.parametrize(("block_size", "num_blocks"), [(512, 556), (1024, 278)]) +def test_hybrid_decode_growth_does_not_preempt_or_replay( + make_scheduler, block_size, num_blocks +): + scheduler = make_scheduler( + hybrid=True, + block_size=block_size, + num_blocks=num_blocks, + max_num_seqs=8, + max_num_batched_tokens=2048, + max_model_len=262144, + num_speculative_tokens=2, + ) + requests = create_requests( + 12, num_tokens=32768, max_tokens=2048, ignore_eos=True, block_size=block_size + ) + pending = iter(requests) + active = set() + for _ in range(8): + request = next(pending) + scheduler.add_request(request) + active.add(request.request_id) + + last_sample: dict[str, int] = {} + prefill_rows = 0 + for step in range(20000): + output = scheduler.schedule() + assert output.num_scheduled_tokens, "admission must not deadlock" + samples: dict[str, list[int]] = {} + for rid, num_tokens in output.num_scheduled_tokens.items(): + request = scheduler.requests[rid] + start = request.num_computed_tokens - num_tokens + prefill_rows += max(0, min(num_tokens, request.num_prompt_tokens - start)) + if request.num_computed_tokens < request.num_tokens: + samples[rid] = [] + continue + count = 2 if rid in output.scheduled_spec_decode_tokens else 1 + samples[rid] = [100] * count + if rid in last_sample: + assert step - last_sample[rid] == 1, "a ready decoder lost its step" + last_sample[rid] = step + _update(scheduler, output, samples) + live = [rid for rid in active if rid in scheduler.requests] + scheduler.update_draft_token_ids( + DraftTokenIds(live, [[101, 102] for _ in live]) + ) + finished = active - set(scheduler.requests) + active -= finished + for _ in finished: + request = next(pending, None) + if request is not None: + scheduler.add_request(request) + active.add(request.request_id) + if not active: + break + else: + pytest.fail("bounded requests did not drain") + + assert sum(request.num_preemptions for request in requests) == 0 + assert prefill_rows == 12 * 32768 + assert all(request.num_output_tokens == 2048 for request in requests) + + +def test_admission_includes_recurrent_boundary_peak(make_scheduler): + scheduler = make_scheduler( + hybrid=True, + block_size=512, + num_blocks=39, + max_num_seqs=2, + max_num_batched_tokens=2048, + max_model_len=4096, + num_speculative_tokens=2, + ) + requests = create_requests( + 2, num_tokens=512, max_tokens=1024, ignore_eos=True, block_size=512 + ) + for request in requests: + scheduler.add_request(request) + # Initial state + two draft slots fits for both requests, but each also + # needs a spare state at a block boundary. Their combined peaks do not fit. + output = scheduler.schedule() + assert set(output.num_scheduled_tokens) == {requests[0].request_id} + + +@pytest.mark.parametrize("finish_reason", ["stop", "abort"]) +def test_output_reservation_released_on_early_finish(make_scheduler, finish_reason): + scheduler = make_scheduler( + num_blocks=15, max_num_seqs=2, max_num_batched_tokens=64, max_model_len=256 + ) + requests = create_requests( + 2, + num_tokens=32, + max_tokens=128, + stop_token_ids=[100] if finish_reason == "stop" else None, + ) + for request in requests: + scheduler.add_request(request) + output = scheduler.schedule() + assert set(output.num_scheduled_tokens) == {requests[0].request_id} + _update(scheduler, output, {requests[0].request_id: [100]}) + if finish_reason == "abort": + scheduler.finish_requests( + requests[0].request_id, RequestStatus.FINISHED_ABORTED + ) + output = scheduler.schedule() + assert set(output.num_scheduled_tokens) == {requests[1].request_id} + assert requests[1].num_preemptions == 0 + + +def test_ready_decode_gets_full_speculative_width_before_prefill(make_scheduler): + scheduler = make_scheduler( + num_blocks=100, + max_num_seqs=2, + max_num_batched_tokens=16, + max_model_len=256, + long_prefill_token_threshold=8, + num_speculative_tokens=2, + ) + (prefill,) = create_requests(1, num_tokens=64, req_ids=["prefill"]) + (decoder,) = create_requests(1, num_tokens=1, req_ids=["decoder"]) + scheduler.add_request(prefill) + scheduler.add_request(decoder) + output = scheduler.schedule() + _update(scheduler, output, {"prefill": [], "decoder": [100]}) + scheduler.update_draft_token_ids(DraftTokenIds(["decoder"], [[101, 102]])) + scheduler.scheduler_config.long_prefill_token_threshold = 0 + + output = scheduler.schedule() + assert output.num_scheduled_tokens["decoder"] == 3 + assert output.scheduled_spec_decode_tokens["decoder"] == [101, 102] + assert output.num_scheduled_tokens["prefill"] > 0 + _update(scheduler, output, {"prefill": [], "decoder": [100, 100]}) + assert decoder.num_output_tokens == 3 + + +def test_admission_growth_clamps_to_context_limit(make_scheduler): + scheduler = make_scheduler( + num_blocks=17, max_num_seqs=1, max_num_batched_tokens=32, max_model_len=256 + ) + (request,) = create_requests(1, num_tokens=32, max_tokens=1024) + scheduler.add_request(request) + assert scheduler.schedule().num_scheduled_tokens == {request.request_id: 32} + + +def test_admission_shares_cached_prompt_pages(make_scheduler): + scheduler = make_scheduler( + num_blocks=13, max_num_seqs=3, max_num_batched_tokens=256, max_model_len=256 + ) + (warm,) = create_requests(1, num_tokens=64, stop_token_ids=[100], req_ids=["warm"]) + scheduler.add_request(warm) + output = scheduler.schedule() + _update(scheduler, output, {"warm": [100]}) + requests = create_requests(3, num_tokens=64, same_prompt=True) + for request in requests: + scheduler.add_request(request) + # Three separate 80-token allocations do not fit. Shared cached prefixes + # do, and must not be charged again as future growth. + output = scheduler.schedule() + assert set(output.num_scheduled_tokens) == {r.request_id for r in requests} + + +def test_final_verify_reserves_rows_beyond_output_limit(make_scheduler): + scheduler = make_scheduler( + num_blocks=6, + max_num_seqs=2, + max_num_batched_tokens=64, + max_model_len=256, + num_speculative_tokens=2, + ) + requests = create_requests(2, num_tokens=30, max_tokens=2, ignore_eos=True) + for request in requests: + scheduler.add_request(request) + # Each final three-row verify crosses 32, even though only one of its + # samples is needed. Five usable pages cannot promise both requests. + output = scheduler.schedule() + first_id = requests[0].request_id + assert set(output.num_scheduled_tokens) == {first_id} + _update(scheduler, output, {first_id: [100]}) + scheduler.update_draft_token_ids(DraftTokenIds([first_id], [[101, 102]])) + output = scheduler.schedule() + assert output.num_scheduled_tokens[first_id] == 3 + _update(scheduler, output, {first_id: [100, 100]}) + assert requests[0].num_output_tokens == 2 + assert requests[0].num_preemptions == 0 + assert requests[1].request_id in scheduler.schedule().num_scheduled_tokens + + +def test_async_inflight_verifies_keep_their_growth_promise(make_scheduler): + scheduler = make_scheduler( + num_blocks=6, + max_num_seqs=2, + max_num_batched_tokens=64, + max_model_len=256, + num_speculative_tokens=2, + ) + if not scheduler.scheduler_config.async_scheduling: + pytest.skip("requires two concurrently scheduled batches") + requests = create_requests(2, num_tokens=27, max_tokens=3, ignore_eos=True) + for request in requests: + scheduler.add_request(request) + first_id = requests[0].request_id + prefill = scheduler.schedule() + # A single final verify fits in two pages, but two unresolved verifies + # can reach row 33. Each request must be promised three pages, not two. + assert set(prefill.num_scheduled_tokens) == {first_id} + verify1 = scheduler.schedule() + _update(scheduler, prefill, {first_id: [100]}) + verify2 = scheduler.schedule() + assert verify2.num_scheduled_tokens[first_id] == 3 + assert requests[0].num_computed_tokens == 33 + _update(scheduler, verify1, {first_id: [100]}) + _update(scheduler, verify2, {first_id: [100]}) + assert requests[0].num_output_tokens == 3 + assert requests[0].num_preemptions == 0 + assert requests[1].request_id in scheduler.schedule().num_scheduled_tokens + + +def test_concurrent_prefill_chunks_reserve_live_recurrent_states( + make_scheduler, monkeypatch +): + from vllm.model_executor.layers.mamba.ops import leapquant_gdn + + monkeypatch.setattr(leapquant_gdn, "leapquant_gdn_enabled", lambda: True) + scheduler = make_scheduler( + hybrid=True, + block_size=512, + num_blocks=53, + max_num_seqs=8, + max_num_batched_tokens=8192, + max_model_len=16384, + num_speculative_tokens=2, + long_prefill_token_threshold=1024, + ) + requests = create_requests( + 2, num_tokens=4096, max_tokens=16, ignore_eos=True, block_size=512 + ) + for request in requests: + scheduler.add_request(request) + chunk1 = scheduler.schedule() + if not scheduler.scheduler_config.async_scheduling: + assert set(chunk1.num_scheduled_tokens) == {r.request_id for r in requests} + return + first_id = requests[0].request_id + # Two sync requests fit, but two unresolved chunks retain one more state + # per recurrent group. Promising both async requests would overcommit. + assert set(chunk1.num_scheduled_tokens) == {first_id} + chunk2 = scheduler.schedule() + _update(scheduler, chunk1, {first_id: []}) + chunk3 = scheduler.schedule() + state_manager = scheduler.kv_cache_manager.coordinator.single_type_managers[1] + live_pages = sum(not b.is_null for b in state_manager.req_to_blocks[first_id]) + assert live_pages == 5 + spec = state_manager.kv_cache_spec + assert spec.max_memory_usage_bytes(scheduler.vllm_config) >= ( + live_pages * spec.page_size_bytes + ) + _update(scheduler, chunk2, {first_id: []}) + _update(scheduler, chunk3, {first_id: []}) + assert requests[0].num_preemptions == 0 diff --git a/vllm/v1/core/kv_cache_coordinator.py b/vllm/v1/core/kv_cache_coordinator.py index e145579d6187..8d8a1203e819 100644 --- a/vllm/v1/core/kv_cache_coordinator.py +++ b/vllm/v1/core/kv_cache_coordinator.py @@ -206,6 +206,7 @@ def get_num_blocks_to_allocate( num_tokens_main_model: int, apply_admission_cap: bool = False, prefill_end: int = 0, + num_admission_batches: int = 0, ) -> int: """Get the number of device blocks needed to be allocated for the request. @@ -232,6 +233,8 @@ def get_num_blocks_to_allocate( retention Mamba reserves a prefill checkpoint only on the chunk reaching it; under dense retention every chunk publishes a state, so it is ignored (0). + num_admission_batches: If nonzero, reserve recurrent state peaks for + this many unresolved batches as well as the predicted allocation. Returns: The number of blocks to allocate. @@ -254,7 +257,7 @@ def get_num_blocks_to_allocate( apply_admission_cap=apply_admission_cap, ) else: - num_blocks_to_allocate += manager.get_num_blocks_to_allocate( + group_blocks = manager.get_num_blocks_to_allocate( request_id, num_tokens, new_computed_blocks[i], @@ -264,6 +267,17 @@ def get_num_blocks_to_allocate( apply_admission_cap=apply_admission_cap, prefill_end=prefill_end, ) + if num_admission_batches and isinstance(manager, MambaManager): + group_blocks = max( + group_blocks, + manager.get_num_blocks_for_peak( + request_id, + num_tokens_main_model, + new_computed_blocks[i], + num_admission_batches, + ), + ) + num_blocks_to_allocate += group_blocks return num_blocks_to_allocate def allocate_new_computed_blocks( diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index b3703fb89873..f84af1c4bf7e 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -391,6 +391,8 @@ def allocate_slots( reserved_blocks: int = 0, has_scheduled_reqs: bool = True, skip_zeroing_group_ids: tuple[int, ...] = (), + num_admission_tokens: int | None = None, + num_admission_batches: int = 1, ) -> KVCacheBlocks | None: """Add slots for a request with new tokens to append. @@ -425,6 +427,12 @@ def allocate_slots( this step, controls whether watermark is applied. skip_zeroing_group_ids: Groups whose external-token blocks will be written by an async load and must not be zeroed concurrently. + num_admission_tokens: Optional bounded sequence length, including future + output, for full_sequence_must_fit. This gate also leaves + reserved_blocks available and accounts for speculative lookahead. + None retains the existing input-sequence admission policy. + num_admission_batches: Maximum unresolved batches per request for the + bounded-output admission gate's recurrent-state peak accounting. Blocks layout: ``` @@ -530,11 +538,21 @@ def allocate_slots( if full_sequence_must_fit: # First check and fail if the full request sequence won't fit. - full_num_tokens = min(request.num_tokens, self.max_model_len) + full_num_tokens = min( + request.num_tokens + if num_admission_tokens is None + else num_admission_tokens, + self.max_model_len, + ) + full_num_tokens_with_lookahead = ( + full_num_tokens + if num_admission_tokens is None + else min(full_num_tokens + num_lookahead_tokens, self.max_model_len) + ) num_blocks_to_allocate = self.coordinator.get_num_blocks_to_allocate( request_id=request.request_id, - num_tokens=full_num_tokens, + num_tokens=full_num_tokens_with_lookahead, new_computed_blocks=new_computed_block_list, num_encoder_tokens=num_encoder_tokens, total_computed_tokens=total_computed_tokens, @@ -542,9 +560,15 @@ def allocate_slots( num_tokens_main_model=full_num_tokens, apply_admission_cap=True, prefill_end=prefill_end, + num_admission_batches=( + num_admission_batches if num_admission_tokens is not None else 0 + ), ) required_blocks = num_blocks_to_allocate + watermark_blocks - if required_blocks > self.block_pool.get_num_free_blocks(): + available_blocks = self.block_pool.get_num_free_blocks() + if num_admission_tokens is not None: + available_blocks -= reserved_blocks + if required_blocks > available_blocks: return None num_tokens_main_model = total_computed_tokens + num_new_tokens diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index ddbd8985eb6e..f7f4c9156e2c 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -27,6 +27,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats from vllm.logger import init_logger +from vllm.model_executor.layers.mamba.ops.leapquant_gdn import leapquant_gdn_enabled from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry from vllm.multimodal.encoder_budget import MultiModalBudget from vllm.multimodal.inputs import MultiModalKwargsItem @@ -434,6 +435,19 @@ def __init__( self.scheduler_reserve_full_isl = ( self.scheduler_config.scheduler_reserve_full_isl ) + self._lq_decode_growth_admission = ( + kv_cache_config.has_mamba_layers and leapquant_gdn_enabled() + ) + # Final verifies can extend beyond the output limit. Each additional + # concurrent batch can still hold a full, unresolved target query. + self._lq_admission_tail_tokens = self.num_spec_tokens + ( + vllm_config.max_concurrent_batches - 1 + ) * (1 + self.num_spec_tokens) + if self._lq_decode_growth_admission: + logger.info( + "LeapQuant scheduling: bounded output admission=%s, decoder-first FCFS", + self.scheduler_reserve_full_isl, + ) # Adaptive prefill scheduling. Off unless VLLM_RDNA_DYNAMIC_PREFILL=1; when it # is off every hook below is skipped and the static config is used unchanged. self._dyn_prefill = maybe_create( @@ -956,7 +970,15 @@ def schedule(self, throttle_prefills: bool = False) -> SchedulerOutput: long_prefill_token_threshold, input_budget // num_eligible_reqs ) - if self._mixed_step is not None and len(self.running) > 1: + if ( + self._lq_decode_growth_admission + and self.policy == SchedulingPolicy.FCFS + and len(self.running) > 1 + ): + # Reserve the ready decoders' target and draft rows before prefills + # consume the shared input budget. Do not override explicit priority. + self.running.sort(key=lambda r: r.is_prefill_chunk) + elif self._mixed_step is not None and len(self.running) > 1: # Decodes and nearly finished prompts first (stable): FCFS order # lets a long running prefill take the whole token budget, and the # requests behind it (just-admitted short prompts, decoders) then @@ -1644,7 +1666,32 @@ def skip_request(from_queue: RequestQueue) -> None: ) reserved_blocks = 0 - if load_kv_async: + num_admission_tokens = None + if self._lq_decode_growth_admission and self.scheduler_reserve_full_isl: + num_admission_tokens = min( + request.num_prompt_tokens + + request.max_tokens + + self._lq_admission_tail_tokens, + self.max_model_len, + ) + # Promise future growth, not just the prompt's current chunk. + # Query live group accounting: allocated pages cease to be + # reservations, and finishing/aborting releases the promise. + reserved_blocks = sum( + self._request_remaining_blocks(other, reserve_output=True) + for other in itertools.chain( + self.running, + ( + req + for req in self._inflight_prefills + if req.status != RequestStatus.RUNNING + ), + ) + if other is not request + ) + if load_kv_async: + reserved_blocks += self._spec_decode_step_blocks + elif load_kv_async: # An async load holds its blocks for the whole transfer with # no forward progress and isn't preemptible here. Admit it # only if it fits in (free - other in-flight reservations) @@ -1675,6 +1722,8 @@ def skip_request(from_queue: RequestQueue) -> None: full_sequence_must_fit=self.scheduler_reserve_full_isl, reserved_blocks=reserved_blocks, has_scheduled_reqs=bool(self.running), + num_admission_tokens=num_admission_tokens, + num_admission_batches=self.vllm_config.max_concurrent_batches, ) if new_blocks is None: @@ -3672,11 +3721,24 @@ def _connector_finished( ) return delay_free or partial_tail_delay, kv_xfer_params - def _request_remaining_blocks(self, request: Request) -> int: - """Blocks `request` still needs to hold its full sequence and be promoted.""" - full_num_tokens = min(request.num_tokens, self.max_model_len) + def _request_remaining_blocks( + self, request: Request, *, reserve_output: bool = False + ) -> int: + """Physical growth still needed, using each group's recycling rules.""" + full_num_tokens = min( + request.num_prompt_tokens + + request.max_tokens + + self._lq_admission_tail_tokens + if reserve_output + else request.num_tokens, + self.max_model_len, + ) num_tokens = full_num_tokens - if self.num_spec_tokens: + if reserve_output: + num_tokens = min( + full_num_tokens + self.num_lookahead_tokens, self.max_model_len + ) + elif self.num_spec_tokens: num_tokens += 1 + self.num_spec_tokens + self.num_lookahead_tokens return self.kv_cache_manager.coordinator.get_num_blocks_to_allocate( request_id=request.request_id, @@ -3687,6 +3749,9 @@ def _request_remaining_blocks(self, request: Request) -> int: num_local_computed_tokens=request.num_computed_tokens, num_tokens_main_model=full_num_tokens, apply_admission_cap=True, + num_admission_batches=( + self.vllm_config.max_concurrent_batches if reserve_output else 0 + ), prefill_end=max(request.num_prompt_tokens, request.num_tokens - 1), ) diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index dd2ba5aa73e5..03927e7ad68b 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -1689,6 +1689,31 @@ def _needs_internal_checkpoint( ) ) + def get_num_blocks_for_peak( + self, + request_id: str, + num_tokens_main_model: int, + new_computed_blocks: Sequence[KVCacheBlock], + num_admission_batches: int, + ) -> int: + """Additional pages at an align-mode state transition, not just this step.""" + if self.mamba_cache_mode != "align": + return 0 + spec = self.kv_cache_spec + assert isinstance(spec, MambaSpec) + peak = min( + cdiv(num_tokens_main_model, self.block_size) + spec.num_speculative_blocks, + 1 + + num_admission_batches + + spec.num_speculative_blocks + + spec.num_prefill_checkpoint_blocks, + ) + held = sum(not block.is_null for block in self.req_to_blocks[request_id]) + adopted = sum(not block.is_null for block in new_computed_blocks) + return max(0, peak - held - adopted) + self._get_num_evictable_blocks( + new_computed_blocks + ) + def get_num_blocks_to_allocate( self, request_id: str, diff --git a/vllm/v1/kv_cache_interface.py b/vllm/v1/kv_cache_interface.py index 011b148623ec..bf92288763e0 100644 --- a/vllm/v1/kv_cache_interface.py +++ b/vllm/v1/kv_cache_interface.py @@ -1138,8 +1138,20 @@ def block_table_token_alignment(self) -> int | None: def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int: if vllm_config.cache_config.mamba_cache_mode == "align": + from vllm.model_executor.layers.mamba.ops.leapquant_gdn import ( + leapquant_gdn_enabled, + ) + + # LeapQuant admission accounts for retained inputs of unresolved + # batches. Startup capacity checks must use the same physical peak. + concurrent_batches = ( + vllm_config.max_concurrent_batches if leapquant_gdn_enabled() else 1 + ) return self.page_size_bytes * ( - 2 + self.num_speculative_blocks + self.num_prefill_checkpoint_blocks + 1 + + concurrent_batches + + self.num_speculative_blocks + + self.num_prefill_checkpoint_blocks ) else: return self.page_size_bytes * (1 + self.num_speculative_blocks) @@ -1148,11 +1160,10 @@ def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int: # Mamba state is replicated across DCP/PCP ranks, never sharded, so # no CP scaling applies. if vllm_config.cache_config.mamba_cache_mode == "align": - # Block table rows are position-indexed over the full sequence - # even though only 2 + num_speculative_blocks state blocks are - # resident at a time (earlier states are nulled out by - # remove_skipped_blocks), so the row length must cover max_len - # rather than max_memory_usage_bytes. + # Block table rows span the full sequence even though only a bounded + # set of states is resident (see max_memory_usage_bytes). Earlier + # states are nulled by remove_skipped_blocks, so the row length must + # cover max_len rather than just the resident-state memory budget. return cdiv(max_len, self.block_size) + self.num_speculative_blocks return cdiv(self.max_memory_usage_bytes(vllm_config), self.page_size_bytes) From 6d6fc194846b367df55e2ce55efddbcb9ae358ec Mon Sep 17 00:00:00 2001 From: BlivionIaG Date: Sat, 14 Nov 2026 14:56:15 +0100 Subject: [PATCH 12/18] fix(bench): include first divergent prediction in decode KL Signed-off-by: BlivionIaG --- .buildkite/test_areas/misc.yaml | 3 ++ docs/rdna2/leapquant-gdn-2026-11-13.md | 5 +++ tests/tools/rdna/test_decode_kl.py | 62 ++++++++++++++++++++++++++ tools/rdna/port_v031/decode_kl.py | 15 ++++--- 4 files changed, 78 insertions(+), 7 deletions(-) create mode 100644 tests/tools/rdna/test_decode_kl.py diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index dea972df6deb..27eb4bfaa016 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -809,6 +809,8 @@ steps: - tests/tools/test_check_agent_files.py - tools/pre_commit/check_trust_remote_code.py - tests/tools/test_check_trust_remote_code.py + - tools/rdna/port_v031/decode_kl.py + - tests/tools/rdna/test_decode_kl.py - tests/tool_parsers - tests/tokenizers_ - tests/parser @@ -822,6 +824,7 @@ steps: - pytest -v -s tools/test_check_test_tethering.py - pytest -v -s tools/test_check_agent_files.py - pytest -v -s tools/test_check_trust_remote_code.py + - pytest -v -s tools/rdna/test_decode_kl.py - label: ":computer: (CPU) Reasoning + Renderers" key: cpu-reasoning-renderers diff --git a/docs/rdna2/leapquant-gdn-2026-11-13.md b/docs/rdna2/leapquant-gdn-2026-11-13.md index fcc35424b038..f398d4840470 100644 --- a/docs/rdna2/leapquant-gdn-2026-11-13.md +++ b/docs/rdna2/leapquant-gdn-2026-11-13.md @@ -98,6 +98,11 @@ boots `quality-20261113-005043` (on/off/off2) and ### Quality: gated checks inside the floor; decode-path KL needs more samples +**Historical decode-KL values below exclude the first divergent prediction.** +That prediction still uses identical input context and is now included by +`decode_kl.py`; later predictions remain excluded. Do not compare these old +values directly with the corrected metric. Prefill KL and PPL are unaffected. + | check (MonoKernel off) | off -> on | floor (off -> off2) | |---|---|---| | `topk_kl` docs, 64 windows: KL / top-1 | 0.01198, 0.01202 / 0.967 | 0.01168 [upper CI 0.01394] / 0.968 | diff --git a/tests/tools/rdna/test_decode_kl.py b/tests/tools/rdna/test_decode_kl.py new file mode 100644 index 000000000000..726d1d86548a --- /dev/null +++ b/tests/tools/rdna/test_decode_kl.py @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Decode KL compares predictions conditioned on the same input prefix.""" + +import importlib.util +import math +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[3] +_spec = importlib.util.spec_from_file_location( + "decode_kl", ROOT / "tools/rdna/port_v031/decode_kl.py" +) +assert _spec is not None and _spec.loader is not None +decode_kl = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(decode_kl) + + +def _run(ids: list[int], probabilities: list[tuple[float, float]]) -> dict: + return { + "meta": {"gen_tokens": 3}, + "runs": [ + { + "ids": ids, + "top": [{"1": math.log(p), "2": math.log(q)} for p, q in probabilities], + } + ], + } + + +@pytest.mark.parametrize("divergence", [0, 1]) +def test_first_divergent_prediction_still_has_the_same_input(divergence: int): + ref = _run([1, 1, 1], [(0.75, 0.25)] * 3) + cand = _run( + [1] * divergence + [2] * (3 - divergence), + [(0.75, 0.25)] * divergence + + [(0.25, 0.75)] + + [(0.01, 0.99)] * (2 - divergence), + ) + + result = decode_kl.pair(ref, cand) + + assert result["positions"] == divergence + 1 + # Earlier predictions agree exactly; later predictions use different inputs. + divergent_kl = 0.5 * math.log(3) + assert result["kl_mean"] == pytest.approx(divergent_kl / (divergence + 1)) + assert result["kl_max"] == pytest.approx(divergent_kl) + assert result["confident_flips"] == 1 + + +def test_identical_generated_tokens_compare_every_prediction(): + ref = _run([1, 1, 1], [(0.75, 0.25)] * 3) + cand = _run([1, 1, 1], [(0.70, 0.30)] * 3) + + result = decode_kl.pair(ref, cand) + + expected_kl = 0.75 * math.log(0.75 / 0.70) + 0.25 * math.log(0.25 / 0.30) + assert result["positions"] == 3 + assert result["kl_mean"] == pytest.approx(expected_kl) + assert result["identical"] == 1 + assert result["confident_flips"] == 0 diff --git a/tools/rdna/port_v031/decode_kl.py b/tools/rdna/port_v031/decode_kl.py index b0a2d4fdf827..439dd9a37476 100755 --- a/tools/rdna/port_v031/decode_kl.py +++ b/tools/rdna/port_v031/decode_kl.py @@ -8,10 +8,10 @@ change that only acts across decode steps (a lossy recurrent-state format such as the LeapQuant GDN state, which a single-chunk prefill never stores) is invisible there. This tool greedy-generates from fixed passages with the -top-k logprobs of every generated token. Up to the first token where two runs -disagree, both runs decoded the same context, so the per-position KL there is a -decode-path KL; the divergence point itself is reported, and a second run of -the reference configuration (``--floor``) gives the noise floor of both. +top-k logprobs of every generated token. Through the first token where two runs +disagree, both predictions use the same preceding context, so their per-position +KL is a decode-path KL. Later predictions are excluded. A second run of the +reference configuration (``--floor``) gives the noise floor. python decode_kl.py collect --url http://127.0.0.1:18096 --model flash-next \\ --out A.json [--passages 32] [--passage-chars 6000] [--gen-tokens 512] \\ @@ -20,8 +20,8 @@ ``--workers`` requests run concurrently, so decode batches hold several requests as in serving. KL(ref || cand) is taken on the partition of the -vocabulary into the tokens of both top-k lists plus one cell for the rest -(a lower bound on the full KL, as in ``topk_kl.py``). +vocabulary into the shared tokens of both top-k lists plus one cell for the rest +(a lower bound on the full KL). """ import argparse @@ -144,7 +144,8 @@ def pair(ref: dict, cand: dict) -> dict: n = min(len(r["ids"]), len(c["ids"])) d = next((i for i in range(n) if r["ids"][i] != c["ids"][i]), n) divergence.append(d) - for i in range(d): + # The first differing token is predicted from the same preceding tokens. + for i in range(min(d + 1, n)): kl, cov = kl_partition(r["top"][i], c["top"][i]) kls.append(kl) covs.append(cov) From db40b66ae61bba089c7cf915ca85bde31fdcc68d Mon Sep 17 00:00:00 2001 From: BlivionIaG Date: Sat, 14 Nov 2026 17:11:23 +0100 Subject: [PATCH 13/18] fix(scheduler): recover adaptive prefill fitting after cold steps Signed-off-by: BlivionIaG --- docs/rdna2/mixed-batch-stall-probe.md | 7 ++++ tests/v1/core/test_decode_stall_bound.py | 45 +++++++++++++++++++++++- vllm/v1/core/sched/mixed_step.py | 34 ++++++++++++------ 3 files changed, 75 insertions(+), 11 deletions(-) diff --git a/docs/rdna2/mixed-batch-stall-probe.md b/docs/rdna2/mixed-batch-stall-probe.md index 3bbf9ec5c888..bc4c510ebe5a 100644 --- a/docs/rdna2/mixed-batch-stall-probe.md +++ b/docs/rdna2/mixed-batch-stall-probe.md @@ -314,6 +314,13 @@ The fit mode depends on the model (`MixedStepController.fit_all_steps_default`): With one bucket of samples the cap can also drop below its initial 512 tokens. +The fit uses the last 64 eligible observations, with the last eight samples +per token bucket reduced to a median. The smallest selectable chunk also +feeds the fit: ignoring its measurements can lock the policy at that chunk +forever. A cold bucket expires even if the policy no longer selects it. +If the fitted intercept is negative, the slope is refitted through the +origin; clamping only the intercept would overpredict work cost. + ### Where it is on **Dense 27B at TP=4 (`27b-exl3`): on, budget 250 ms.** Off in `full` (27B at TP=2): steps are ~2x slower there, and the cap cut 16k/1k c=8 aggregate output 22.7 -> 19.6 tok/s (TTFT 135 -> 200 s). 16k injection diff --git a/tests/v1/core/test_decode_stall_bound.py b/tests/v1/core/test_decode_stall_bound.py index 4da55cbd8358..8815b941ef4d 100644 --- a/tests/v1/core/test_decode_stall_bound.py +++ b/tests/v1/core/test_decode_stall_bound.py @@ -128,11 +128,54 @@ def test_fit_mode_default_dense_vs_moe(monkeypatch): def test_outlier_step_is_ignored(): c = _trained() cap = c.prefill_cap() - c.observe(512, 6, 25.0) # a JIT compile, clipped to 1.5x the bucket mean + c.observe(512, 6, 25.0) # One cold sample against four warm observations. c.observe(512, 6, 40.0) # above the 30 s sanity bound: dropped assert abs(c.prefill_cap() - cap) <= 64 +def test_controller_recovers_after_cold_fit_selects_smallest_chunk(): + c = MixedStepController( + 0.45, 0.0, 2048, granularity=64, initial_cap=512, max_overhead=1.0 + ) + c.observe(512, 8, 2.6) + c.observe(128, 8, 0.14) + assert c.prefill_cap(block_size=512) == 64 + + for _ in range(96): + cap = c.prefill_cap(block_size=512) + c.observe(cap, 8, 0.11 + 0.00015 * cap) + + cap = c.prefill_cap(block_size=512) + assert cap >= 512 + assert 0.11 + 0.00015 * cap <= 0.45 + assert not c.defer_prefill() + + +def test_controller_refits_cost_with_nonnegative_intercept(): + c = MixedStepController(0.45, 0.0, 1024, granularity=64, max_overhead=0.0) + for tokens, duration in ((128, 0.1), (256, 1.0)): + for _ in range(4): + c.observe(tokens, 8, duration) + + # Constrained least squares: refit the slope through the origin, rather + # than keeping a slope fitted against an impossible negative fixed cost. + slope = (128 * 0.1 + 256 * 1.0) / (128**2 + 256**2) + cap = c.prefill_cap() + assert slope * cap <= 0.45 + assert slope * (cap + 64) > 0.45 + + +def test_controller_tracks_sustained_cost_change(): + c = _trained(budget_s=0.25, share=0.0, max_overhead=0.0) + for _ in range(96): + cap = c.prefill_cap() + c.observe(cap, 8, 0.03 + 0.0001 * cap) + + cap = c.prefill_cap() + assert cap >= 1024 + assert 0.03 + 0.0001 * cap <= 0.25 + + @pytest.fixture(autouse=True) def _long_model_len(monkeypatch): # opt-125m derives max_model_len=2048; these tests need long prompts. diff --git a/vllm/v1/core/sched/mixed_step.py b/vllm/v1/core/sched/mixed_step.py index a37f00d1ae5e..7778113a1d2c 100644 --- a/vllm/v1/core/sched/mixed_step.py +++ b/vllm/v1/core/sched/mixed_step.py @@ -53,10 +53,10 @@ def __init__( self.granularity = max(1, granularity) self.initial_cap = initial_cap self.max_overhead = max_overhead - # Recent durations per prefill-token bucket (bucket -> last 8 samples). - # The fit uses each bucket's median, so a one-off step (a JIT compile, - # a TunableOp miss, a slow first request) cannot move the cap. - self._buckets: dict[int, deque[float]] = {} + # Bound history across buckets as well as within each bucket. Otherwise + # a cold bucket that the policy stops selecting can bias the fit forever. + # Per-bucket medians still limit the influence of isolated slow steps. + self._history: deque[tuple[int, float]] = deque(maxlen=64) self._decode_s = 0.035 self._debt_s = 0.0 # Which steps feed the fit; see fit_all_steps_default(). @@ -83,12 +83,12 @@ def observe( # prefix-cache tails) dragged Flash-Next's fitted fixed cost to # 330-800 ms, above the budget, which pinned the cap at 2048. return - if num_prefill_tokens < 2 * self.granularity: - # Tiny prefills (a short prompt, the tail of a long one) are - # dominated by per-step overheads; they would distort the slope. + if num_prefill_tokens < self.granularity: return + # Include the smallest selectable chunk: ignoring it leaves no feedback + # when a cold fit selects that chunk, so the controller cannot recover. b = round(num_prefill_tokens / self.granularity) - self._buckets.setdefault(b, deque(maxlen=8)).append(elapsed_s) + self._history.append((b, elapsed_s)) @staticmethod def fit_all_steps_default(is_moe: bool) -> bool: @@ -108,9 +108,15 @@ def fit_all_steps_default(is_moe: bool) -> bool: # ---------------------------------------------------------------- policy def _points(self) -> list[tuple[float, float, int]]: + buckets: dict[int, deque[float]] = {} + for b, elapsed_s in self._history: + samples = buckets.get(b) + if samples is None: + samples = buckets[b] = deque(maxlen=8) + samples.append(elapsed_s) return [ (b * self.granularity, statistics.median(v), len(v)) - for b, v in self._buckets.items() + for b, v in buckets.items() ] def _fit(self) -> tuple[float, float] | None: @@ -126,7 +132,15 @@ def _fit(self) -> tuple[float, float] | None: slope = sum(n * (x - mx) * (y - my) for x, y, n in pts) / sxx if slope <= 0.0: return None - return max(0.0, my - slope * mx), slope + intercept = my - slope * mx + if intercept < 0.0: + # Constrained least squares at a=0 needs a new slope. Clamping + # only the intercept overpredicts cost and underschedules work. + slope = sum(n * x * y for x, y, n in pts) / sum( + n * x * x for x, _, n in pts + ) + intercept = 0.0 + return intercept, slope def prefill_cap(self, block_size: int | None = None) -> int: """Max prefill tokens for a step that carries decodes.""" From 88be1d16954416fc1f681eda9c8debbf7e5b6b13 Mon Sep 17 00:00:00 2001 From: BlivionIaG Date: Sat, 14 Nov 2026 18:34:55 +0100 Subject: [PATCH 14/18] feat(rdna2): qualify 512-token LeapQuant with 32-record windows Signed-off-by: BlivionIaG --- .../controller-cpu-gate.txt | 9 + .../controller-recovery-replay.json | 777 ++++++++++++ .../cpu-window-derived-summary.json | 44 + .../decode-quality-comparison.json | 966 +++++++++++++++ .../pilot-metrics.json | 237 ++++ .../qualified-comparison.json | 1057 +++++++++++++++++ docs/rdna2/leapquant-gdn-2026-11-13.md | 149 ++- .../kernels/mamba/test_gdn_leapquant_rdna2.py | 65 +- tools/rdna/recipes/flashnext-mtp2.env | 8 +- .../layers/mamba/ops/leapquant_gdn.py | 4 +- 10 files changed, 3275 insertions(+), 41 deletions(-) create mode 100644 bench_results/2026-11-14_leapquant-acceptance/controller-cpu-gate.txt create mode 100644 bench_results/2026-11-14_leapquant-acceptance/controller-recovery-replay.json create mode 100644 bench_results/2026-11-14_leapquant-acceptance/cpu-window-derived-summary.json create mode 100644 bench_results/2026-11-14_leapquant-acceptance/decode-quality-comparison.json create mode 100644 bench_results/2026-11-14_leapquant-acceptance/pilot-metrics.json create mode 100644 bench_results/2026-11-14_leapquant-acceptance/qualified-comparison.json diff --git a/bench_results/2026-11-14_leapquant-acceptance/controller-cpu-gate.txt b/bench_results/2026-11-14_leapquant-acceptance/controller-cpu-gate.txt new file mode 100644 index 000000000000..128a8aa6613f --- /dev/null +++ b/bench_results/2026-11-14_leapquant-acceptance/controller-cpu-gate.txt @@ -0,0 +1,9 @@ +exit=0 +.....................................s... [100%] +=============================== warnings summary =============================== +Apps/vllm/venv-7.14.0_0.32.0/lib64/python3.12/site-packages/torch/jit/_script.py:365: 14 warnings + /home/chenco_adm/Apps/vllm/venv-7.14.0_0.32.0/lib64/python3.12/site-packages/torch/jit/_script.py:365: DeprecationWarning: `torch.jit.script_method` is deprecated. Please switch to `torch.compile` or `torch.export`. + warnings.warn( + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +40 passed, 1 skipped, 14 warnings in 23.96s diff --git a/bench_results/2026-11-14_leapquant-acceptance/controller-recovery-replay.json b/bench_results/2026-11-14_leapquant-acceptance/controller-recovery-replay.json new file mode 100644 index 000000000000..330df7ca87d0 --- /dev/null +++ b/bench_results/2026-11-14_leapquant-acceptance/controller-recovery-replay.json @@ -0,0 +1,777 @@ +{ + "scope": "Synthetic component replay, not GPU performance: real controller classes before/after; two cold observations followed by cost .11+.00015*tokens. Decode share zero.", + "traces": { + "before": [ + { + "step": 0, + "cap": 64 + }, + { + "step": 1, + "cap": 64 + }, + { + "step": 2, + "cap": 64 + }, + { + "step": 3, + "cap": 64 + }, + { + "step": 4, + "cap": 64 + }, + { + "step": 5, + "cap": 64 + }, + { + "step": 6, + "cap": 64 + }, + { + "step": 7, + "cap": 64 + }, + { + "step": 8, + "cap": 64 + }, + { + "step": 9, + "cap": 64 + }, + { + "step": 10, + "cap": 64 + }, + { + "step": 11, + "cap": 64 + }, + { + "step": 12, + "cap": 64 + }, + { + "step": 13, + "cap": 64 + }, + { + "step": 14, + "cap": 64 + }, + { + "step": 15, + "cap": 64 + }, + { + "step": 16, + "cap": 64 + }, + { + "step": 17, + "cap": 64 + }, + { + "step": 18, + "cap": 64 + }, + { + "step": 19, + "cap": 64 + }, + { + "step": 20, + "cap": 64 + }, + { + "step": 21, + "cap": 64 + }, + { + "step": 22, + "cap": 64 + }, + { + "step": 23, + "cap": 64 + }, + { + "step": 24, + "cap": 64 + }, + { + "step": 25, + "cap": 64 + }, + { + "step": 26, + "cap": 64 + }, + { + "step": 27, + "cap": 64 + }, + { + "step": 28, + "cap": 64 + }, + { + "step": 29, + "cap": 64 + }, + { + "step": 30, + "cap": 64 + }, + { + "step": 31, + "cap": 64 + }, + { + "step": 32, + "cap": 64 + }, + { + "step": 33, + "cap": 64 + }, + { + "step": 34, + "cap": 64 + }, + { + "step": 35, + "cap": 64 + }, + { + "step": 36, + "cap": 64 + }, + { + "step": 37, + "cap": 64 + }, + { + "step": 38, + "cap": 64 + }, + { + "step": 39, + "cap": 64 + }, + { + "step": 40, + "cap": 64 + }, + { + "step": 41, + "cap": 64 + }, + { + "step": 42, + "cap": 64 + }, + { + "step": 43, + "cap": 64 + }, + { + "step": 44, + "cap": 64 + }, + { + "step": 45, + "cap": 64 + }, + { + "step": 46, + "cap": 64 + }, + { + "step": 47, + "cap": 64 + }, + { + "step": 48, + "cap": 64 + }, + { + "step": 49, + "cap": 64 + }, + { + "step": 50, + "cap": 64 + }, + { + "step": 51, + "cap": 64 + }, + { + "step": 52, + "cap": 64 + }, + { + "step": 53, + "cap": 64 + }, + { + "step": 54, + "cap": 64 + }, + { + "step": 55, + "cap": 64 + }, + { + "step": 56, + "cap": 64 + }, + { + "step": 57, + "cap": 64 + }, + { + "step": 58, + "cap": 64 + }, + { + "step": 59, + "cap": 64 + }, + { + "step": 60, + "cap": 64 + }, + { + "step": 61, + "cap": 64 + }, + { + "step": 62, + "cap": 64 + }, + { + "step": 63, + "cap": 64 + }, + { + "step": 64, + "cap": 64 + }, + { + "step": 65, + "cap": 64 + }, + { + "step": 66, + "cap": 64 + }, + { + "step": 67, + "cap": 64 + }, + { + "step": 68, + "cap": 64 + }, + { + "step": 69, + "cap": 64 + }, + { + "step": 70, + "cap": 64 + }, + { + "step": 71, + "cap": 64 + }, + { + "step": 72, + "cap": 64 + }, + { + "step": 73, + "cap": 64 + }, + { + "step": 74, + "cap": 64 + }, + { + "step": 75, + "cap": 64 + }, + { + "step": 76, + "cap": 64 + }, + { + "step": 77, + "cap": 64 + }, + { + "step": 78, + "cap": 64 + }, + { + "step": 79, + "cap": 64 + }, + { + "step": 80, + "cap": 64 + }, + { + "step": 81, + "cap": 64 + }, + { + "step": 82, + "cap": 64 + }, + { + "step": 83, + "cap": 64 + }, + { + "step": 84, + "cap": 64 + }, + { + "step": 85, + "cap": 64 + }, + { + "step": 86, + "cap": 64 + }, + { + "step": 87, + "cap": 64 + }, + { + "step": 88, + "cap": 64 + }, + { + "step": 89, + "cap": 64 + }, + { + "step": 90, + "cap": 64 + }, + { + "step": 91, + "cap": 64 + }, + { + "step": 92, + "cap": 64 + }, + { + "step": 93, + "cap": 64 + }, + { + "step": 94, + "cap": 64 + }, + { + "step": 95, + "cap": 64 + } + ], + "after": [ + { + "step": 0, + "cap": 64 + }, + { + "step": 1, + "cap": 64 + }, + { + "step": 2, + "cap": 64 + }, + { + "step": 3, + "cap": 64 + }, + { + "step": 4, + "cap": 64 + }, + { + "step": 5, + "cap": 64 + }, + { + "step": 6, + "cap": 64 + }, + { + "step": 7, + "cap": 64 + }, + { + "step": 8, + "cap": 64 + }, + { + "step": 9, + "cap": 64 + }, + { + "step": 10, + "cap": 64 + }, + { + "step": 11, + "cap": 64 + }, + { + "step": 12, + "cap": 64 + }, + { + "step": 13, + "cap": 64 + }, + { + "step": 14, + "cap": 64 + }, + { + "step": 15, + "cap": 64 + }, + { + "step": 16, + "cap": 64 + }, + { + "step": 17, + "cap": 64 + }, + { + "step": 18, + "cap": 64 + }, + { + "step": 19, + "cap": 64 + }, + { + "step": 20, + "cap": 64 + }, + { + "step": 21, + "cap": 64 + }, + { + "step": 22, + "cap": 64 + }, + { + "step": 23, + "cap": 64 + }, + { + "step": 24, + "cap": 64 + }, + { + "step": 25, + "cap": 64 + }, + { + "step": 26, + "cap": 64 + }, + { + "step": 27, + "cap": 64 + }, + { + "step": 28, + "cap": 64 + }, + { + "step": 29, + "cap": 64 + }, + { + "step": 30, + "cap": 64 + }, + { + "step": 31, + "cap": 64 + }, + { + "step": 32, + "cap": 64 + }, + { + "step": 33, + "cap": 64 + }, + { + "step": 34, + "cap": 64 + }, + { + "step": 35, + "cap": 64 + }, + { + "step": 36, + "cap": 64 + }, + { + "step": 37, + "cap": 64 + }, + { + "step": 38, + "cap": 64 + }, + { + "step": 39, + "cap": 64 + }, + { + "step": 40, + "cap": 64 + }, + { + "step": 41, + "cap": 64 + }, + { + "step": 42, + "cap": 64 + }, + { + "step": 43, + "cap": 64 + }, + { + "step": 44, + "cap": 64 + }, + { + "step": 45, + "cap": 64 + }, + { + "step": 46, + "cap": 64 + }, + { + "step": 47, + "cap": 64 + }, + { + "step": 48, + "cap": 64 + }, + { + "step": 49, + "cap": 64 + }, + { + "step": 50, + "cap": 64 + }, + { + "step": 51, + "cap": 64 + }, + { + "step": 52, + "cap": 64 + }, + { + "step": 53, + "cap": 64 + }, + { + "step": 54, + "cap": 64 + }, + { + "step": 55, + "cap": 64 + }, + { + "step": 56, + "cap": 64 + }, + { + "step": 57, + "cap": 64 + }, + { + "step": 58, + "cap": 64 + }, + { + "step": 59, + "cap": 64 + }, + { + "step": 60, + "cap": 64 + }, + { + "step": 61, + "cap": 64 + }, + { + "step": 62, + "cap": 64 + }, + { + "step": 63, + "cap": 1024 + }, + { + "step": 64, + "cap": 2048 + }, + { + "step": 65, + "cap": 2048 + }, + { + "step": 66, + "cap": 2048 + }, + { + "step": 67, + "cap": 2048 + }, + { + "step": 68, + "cap": 2048 + }, + { + "step": 69, + "cap": 2048 + }, + { + "step": 70, + "cap": 2048 + }, + { + "step": 71, + "cap": 2048 + }, + { + "step": 72, + "cap": 2048 + }, + { + "step": 73, + "cap": 2048 + }, + { + "step": 74, + "cap": 2048 + }, + { + "step": 75, + "cap": 2048 + }, + { + "step": 76, + "cap": 2048 + }, + { + "step": 77, + "cap": 2048 + }, + { + "step": 78, + "cap": 2048 + }, + { + "step": 79, + "cap": 2048 + }, + { + "step": 80, + "cap": 2048 + }, + { + "step": 81, + "cap": 2048 + }, + { + "step": 82, + "cap": 2048 + }, + { + "step": 83, + "cap": 2048 + }, + { + "step": 84, + "cap": 2048 + }, + { + "step": 85, + "cap": 2048 + }, + { + "step": 86, + "cap": 2048 + }, + { + "step": 87, + "cap": 2048 + }, + { + "step": 88, + "cap": 2048 + }, + { + "step": 89, + "cap": 2048 + }, + { + "step": 90, + "cap": 2048 + }, + { + "step": 91, + "cap": 2048 + }, + { + "step": 92, + "cap": 2048 + }, + { + "step": 93, + "cap": 2048 + }, + { + "step": 94, + "cap": 2048 + }, + { + "step": 95, + "cap": 2048 + } + ] + } +} diff --git a/bench_results/2026-11-14_leapquant-acceptance/cpu-window-derived-summary.json b/bench_results/2026-11-14_leapquant-acceptance/cpu-window-derived-summary.json new file mode 100644 index 000000000000..1117ffb30ab7 --- /dev/null +++ b/bench_results/2026-11-14_leapquant-acceptance/cpu-window-derived-summary.json @@ -0,0 +1,44 @@ +{ + "scope": "Synthetic CPU GDN recurrence with identical accepted prefixes; not MTP acceptance, GPU performance, or model quality", + "seeds": [ + 3, + 19, + 71 + ], + "steps": 96, + "max_query_len": 3, + "accepted_output_relative_error_mean_by_seed": { + "w16-r4": [ + 0.0031096385365951087, + 0.0021988107544833233, + 0.0029049378664846395 + ], + "w32-r4": [ + 0.002223798308326271, + 0.0016178085411411303, + 0.002092122359348044 + ], + "w16-r8": [ + 0.00263732947405515, + 0.001960258194837176, + 0.002483596682698893 + ] + }, + "final_state_relative_error_by_seed": { + "w16-r4": [ + 0.004618530627340078, + 0.0037725975271314383, + 0.004417973570525646 + ], + "w32-r4": [ + 0.0027301411610096693, + 0.002603197703137994, + 0.002697742311283946 + ], + "w16-r8": [ + 0.003944317810237408, + 0.0034788830671459436, + 0.0038036704063415527 + ] + } +} diff --git a/bench_results/2026-11-14_leapquant-acceptance/decode-quality-comparison.json b/bench_results/2026-11-14_leapquant-acceptance/decode-quality-comparison.json new file mode 100644 index 000000000000..037134f7c5d2 --- /dev/null +++ b/bench_results/2026-11-14_leapquant-acceptance/decode-quality-comparison.json @@ -0,0 +1,966 @@ +{ + "scope": "Same preceding tokens and common top-k vocabulary partition across all boots; includes first divergent prediction. Bootstrap resamples passages, conditional on these boots.", + "reference_boot": "A1", + "floor_boot": "A2", + "candidate_boots": ["C1", "C2"], + "bootstrap_resamples": 20000, + "bootstrap_seed": 20261114, + "bootstrap_aggregation": "Resample 128 passages with replacement; divide summed KL by summed common positions. Average C1/C2 KL on the same positions; form candidate/floor ratio for each resample. Report 2.5th and 97.5th percentiles.", + "passages": 128, + "positions": 2963, + "positions_per_passage_median": 7.0, + "coverage_mean": 0.9779889571208276, + "candidate_kl_mean": 0.013978320883045148, + "floor_kl_mean": 0.012623591772722023, + "candidate_minus_floor": 0.0013547291103231246, + "difference_95_interval": [ + -0.0021298224329201077, + 0.004268801295470227 + ], + "candidate_to_floor_ratio": 1.1073172465265013, + "ratio_95_interval": [ + 0.8361255862534297, + 1.3170242275327946 + ], + "per_passage": [ + { + "positions": 4, + "coverage_sum": 3.9727993944940563, + "A2": 0.0178642933431514, + "C1": 0.03567802484642042, + "C2": 0.011358808266781432 + }, + { + "positions": 4, + "coverage_sum": 3.9541087533969277, + "A2": 0.05449936924878972, + "C1": 0.05477843672934189, + "C2": 0.11074265495969897 + }, + { + "positions": 4, + "coverage_sum": 3.546481880018412, + "A2": 0.010910331080674754, + "C1": 0.020918911214386913, + "C2": 0.006463971876070791 + }, + { + "positions": 7, + "coverage_sum": 6.421659561715666, + "A2": 0.12087312616304491, + "C1": 0.1059350320182833, + "C2": 0.10527447962589667 + }, + { + "positions": 1, + "coverage_sum": 0.7378091825898727, + "A2": 0.036289618884239754, + "C1": 0.03292923912532339, + "C2": 0.024792401746171033 + }, + { + "positions": 512, + "coverage_sum": 511.0471278844821, + "A2": 9.752358463415261, + "C1": 14.302244683944467, + "C2": 15.355989295855467 + }, + { + "positions": 2, + "coverage_sum": 1.9519161247502126, + "A2": 0.008701149427533876, + "C1": 0.047158216867879875, + "C2": 0.010345972895830764 + }, + { + "positions": 36, + "coverage_sum": 35.10342675427606, + "A2": 0.3240804249447399, + "C1": 0.40899550103501653, + "C2": 0.635865278639246 + }, + { + "positions": 33, + "coverage_sum": 32.54106866914014, + "A2": 0.10691432287594563, + "C1": 0.07418261793712062, + "C2": 0.10773665931108804 + }, + { + "positions": 6, + "coverage_sum": 5.858210647752898, + "A2": 0.5497977868893692, + "C1": 0.9909480967616514, + "C2": 1.4584490468427926 + }, + { + "positions": 9, + "coverage_sum": 8.96204283873262, + "A2": 0.030033552632671944, + "C1": 0.027472180690612174, + "C2": 0.015005707891234819 + }, + { + "positions": 49, + "coverage_sum": 48.681286761872606, + "A2": 0.07275922715551, + "C1": 0.11992396859306448, + "C2": 0.054230842049855316 + }, + { + "positions": 34, + "coverage_sum": 33.86047891845735, + "A2": 0.07614627643394527, + "C1": 0.05899590520440753, + "C2": 0.05928773881284494 + }, + { + "positions": 3, + "coverage_sum": 2.3491849363134243, + "A2": 0.0038671283372510514, + "C1": 0.0023655950144297892, + "C2": 0.00655327718757872 + }, + { + "positions": 26, + "coverage_sum": 25.422770634063667, + "A2": 0.44814717427871376, + "C1": 0.3149461717208504, + "C2": 0.6704072159102439 + }, + { + "positions": 3, + "coverage_sum": 2.9560335087944036, + "A2": 0.9419378576603633, + "C1": 0.6070723836409947, + "C2": 0.906635635127621 + }, + { + "positions": 21, + "coverage_sum": 19.94475284867036, + "A2": 0.0946670312375547, + "C1": 0.12200987471238005, + "C2": 0.0842616116545189 + }, + { + "positions": 10, + "coverage_sum": 9.700766444200482, + "A2": 0.015247065979417178, + "C1": 0.016928108184116025, + "C2": 0.01364807994368252 + }, + { + "positions": 19, + "coverage_sum": 18.82683633033457, + "A2": 0.310560919848682, + "C1": 0.39790899301071014, + "C2": 0.2829920764565785 + }, + { + "positions": 49, + "coverage_sum": 48.92834155988535, + "A2": 0.06342010545793225, + "C1": 0.05538649407308687, + "C2": 0.09794933618534185 + }, + { + "positions": 4, + "coverage_sum": 3.796751577600939, + "A2": 0.06488423282813037, + "C1": 0.057637111827669035, + "C2": 0.0571206713613229 + }, + { + "positions": 9, + "coverage_sum": 8.517121125065048, + "A2": 0.04996573393734786, + "C1": 0.03791532817151723, + "C2": 0.03781219435715968 + }, + { + "positions": 13, + "coverage_sum": 12.994353555859611, + "A2": 0.20968265849529744, + "C1": 0.31128088552065286, + "C2": 0.14752590143633393 + }, + { + "positions": 4, + "coverage_sum": 3.5723001155872556, + "A2": 0.537436618290837, + "C1": 0.4409795184015311, + "C2": 0.3739552571021939 + }, + { + "positions": 107, + "coverage_sum": 106.59947901943656, + "A2": 0.14813338261753048, + "C1": 0.16019624593998494, + "C2": 0.21216884594167423 + }, + { + "positions": 2, + "coverage_sum": 1.9610243023349518, + "A2": 0.025042994311080407, + "C1": 0.03363437109050141, + "C2": 0.019059325217182187 + }, + { + "positions": 56, + "coverage_sum": 55.344294635193236, + "A2": 0.08217961147241554, + "C1": 0.053731088354273294, + "C2": 0.06531934565942042 + }, + { + "positions": 1, + "coverage_sum": 0.6516503965524738, + "A2": 0.05694222878072903, + "C1": 0.08700357596884016, + "C2": 0.036140804076496606 + }, + { + "positions": 36, + "coverage_sum": 35.181946686240494, + "A2": 0.1585449449208822, + "C1": 0.20581202323027445, + "C2": 0.18565515232457724 + }, + { + "positions": 71, + "coverage_sum": 67.93284955058058, + "A2": 0.2663149693208765, + "C1": 0.1910014131131895, + "C2": 0.37561521240250406 + }, + { + "positions": 1, + "coverage_sum": 0.9998452592193887, + "A2": 0.4611732972383165, + "C1": 0.5196493590948891, + "C2": 0.09425108976285335 + }, + { + "positions": 7, + "coverage_sum": 6.314147218312878, + "A2": 0.1713824046354495, + "C1": 0.11178975377829664, + "C2": 0.07984270965265544 + }, + { + "positions": 68, + "coverage_sum": 67.63016231496718, + "A2": 0.2886277308771134, + "C1": 0.3141000514377211, + "C2": 0.14324018305340663 + }, + { + "positions": 55, + "coverage_sum": 54.52721804519543, + "A2": 1.5068364506845564, + "C1": 1.9213156346647706, + "C2": 1.8786567173152267 + }, + { + "positions": 5, + "coverage_sum": 4.974303918266125, + "A2": 2.7336127380396436, + "C1": 1.0622233053718668, + "C2": 2.8005905007085174 + }, + { + "positions": 6, + "coverage_sum": 5.746035754799978, + "A2": 0.08634560653283081, + "C1": 0.07689424868337097, + "C2": 0.08076326424156414 + }, + { + "positions": 38, + "coverage_sum": 37.92754138170702, + "A2": 0.5042679744450601, + "C1": 0.5950977104020462, + "C2": 0.2748987909669319 + }, + { + "positions": 7, + "coverage_sum": 6.586447885248639, + "A2": 0.1791828653786279, + "C1": 0.17383181412787793, + "C2": 0.1776226755405661 + }, + { + "positions": 24, + "coverage_sum": 23.786307358832765, + "A2": 0.056079090130261085, + "C1": 0.08692653180950323, + "C2": 0.10241054067717872 + }, + { + "positions": 1, + "coverage_sum": 0.6666506812917955, + "A2": 0.021408487027145, + "C1": 0.007476328114028152, + "C2": 0.014199079410296887 + }, + { + "positions": 59, + "coverage_sum": 58.354248696579546, + "A2": 0.14956209534800022, + "C1": 0.13737354696157839, + "C2": 0.14328343093532736 + }, + { + "positions": 4, + "coverage_sum": 3.374662532869632, + "A2": 0.027390068088665297, + "C1": 0.04284711717587652, + "C2": 0.30411630687068286 + }, + { + "positions": 4, + "coverage_sum": 3.528793049375715, + "A2": 0.015632146319135336, + "C1": 0.020075464217569724, + "C2": 0.030826311369601095 + }, + { + "positions": 3, + "coverage_sum": 2.815154242955505, + "A2": 0.0532251808355271, + "C1": 0.07376831177038, + "C2": 0.05737499250602573 + }, + { + "positions": 4, + "coverage_sum": 3.615792632551603, + "A2": 0.04208920880135584, + "C1": 0.10433925420690474, + "C2": 0.12190833253685338 + }, + { + "positions": 3, + "coverage_sum": 2.09928044263485, + "A2": 0.06091174778765022, + "C1": 0.016752700543598918, + "C2": 0.017356889336220475 + }, + { + "positions": 1, + "coverage_sum": 0.8770896786220528, + "A2": 0.023431654004315232, + "C1": 0.03636513727665514, + "C2": 0.027276661131763497 + }, + { + "positions": 3, + "coverage_sum": 2.143038514444626, + "A2": 0.038767342844201344, + "C1": 0.02664248661237022, + "C2": 0.03306491370323673 + }, + { + "positions": 1, + "coverage_sum": 0.8613966977977099, + "A2": 0.048875503042604074, + "C1": 0.025006101319921592, + "C2": 0.03328554245204071 + }, + { + "positions": 66, + "coverage_sum": 64.85629041951422, + "A2": 0.18819619667596293, + "C1": 0.10489167202995538, + "C2": 0.17593281252884513 + }, + { + "positions": 8, + "coverage_sum": 7.791209250989193, + "A2": 0.3545511091799911, + "C1": 2.5201864665251623, + "C2": 0.11867212692983894 + }, + { + "positions": 23, + "coverage_sum": 22.9363480283245, + "A2": 0.01634317960654661, + "C1": 0.044879109648940714, + "C2": 0.026806101035063007 + }, + { + "positions": 1, + "coverage_sum": 0.9930082816954773, + "A2": 0.014602044349210204, + "C1": 0.028973763731989396, + "C2": 0.003731571072557142 + }, + { + "positions": 95, + "coverage_sum": 93.37818712529679, + "A2": 0.7197422545744305, + "C1": 0.42183250760355145, + "C2": 0.524164699386325 + }, + { + "positions": 25, + "coverage_sum": 24.97285540476757, + "A2": 0.06118698391953463, + "C1": 0.09346416219100348, + "C2": 0.11234188558658414 + }, + { + "positions": 29, + "coverage_sum": 28.28307902027324, + "A2": 0.09291835259337188, + "C1": 0.08463968975455974, + "C2": 0.20258368453526143 + }, + { + "positions": 8, + "coverage_sum": 7.178269931800223, + "A2": 0.11400325289184222, + "C1": 0.09073922409377118, + "C2": 0.1534966090074754 + }, + { + "positions": 192, + "coverage_sum": 188.29400096280628, + "A2": 0.6638103513371635, + "C1": 0.47711592291960486, + "C2": 0.5398318822492096 + }, + { + "positions": 6, + "coverage_sum": 5.617148428144767, + "A2": 0.34687758007843233, + "C1": 0.3047633210423539, + "C2": 0.1810412926683952 + }, + { + "positions": 72, + "coverage_sum": 69.71380331462099, + "A2": 0.46249757905688305, + "C1": 0.4641572017749209, + "C2": 0.5827984422437144 + }, + { + "positions": 2, + "coverage_sum": 1.5248176778021187, + "A2": 0.011925564329892387, + "C1": 0.04620324468440253, + "C2": 0.03728327922552247 + }, + { + "positions": 14, + "coverage_sum": 13.999417949615088, + "A2": 0.08715445879839567, + "C1": 0.022062976012824087, + "C2": 0.050893227753834265 + }, + { + "positions": 85, + "coverage_sum": 84.95650158577693, + "A2": 0.17295281813366356, + "C1": 0.5153320766607623, + "C2": 0.728103410093515 + }, + { + "positions": 20, + "coverage_sum": 17.89005817419127, + "A2": 0.1308647300182994, + "C1": 0.08975287190480763, + "C2": 0.1261097983295071 + }, + { + "positions": 17, + "coverage_sum": 16.032659390796045, + "A2": 0.12292696927166871, + "C1": 0.0437424177272866, + "C2": 0.12461271053102808 + }, + { + "positions": 1, + "coverage_sum": 0.8261786629544963, + "A2": 0.032207766610218194, + "C1": 0.053922206910272795, + "C2": 0.03634629704542237 + }, + { + "positions": 33, + "coverage_sum": 32.768520700836504, + "A2": 0.2518777433196097, + "C1": 0.19408354895880697, + "C2": 0.5370747153490856 + }, + { + "positions": 43, + "coverage_sum": 41.131622343774495, + "A2": 0.035256994816206366, + "C1": 0.042327542433245983, + "C2": 0.027585584665168344 + }, + { + "positions": 3, + "coverage_sum": 2.7084806225443536, + "A2": 0.0240862824862855, + "C1": 0.012046688514126025, + "C2": 0.0315194959747354 + }, + { + "positions": 153, + "coverage_sum": 151.6585094241312, + "A2": 0.3450031655033049, + "C1": 0.36006508365133444, + "C2": 0.27445630632672285 + }, + { + "positions": 8, + "coverage_sum": 7.992607063244257, + "A2": 1.7163778583285885, + "C1": 0.3717677971412019, + "C2": 0.34123120746970703 + }, + { + "positions": 1, + "coverage_sum": 0.8887896698458598, + "A2": 0.10197039972641331, + "C1": 0.027213209509380318, + "C2": 0.01981230190480204 + }, + { + "positions": 3, + "coverage_sum": 2.066884770834881, + "A2": 0.06605884371702016, + "C1": 0.04584713338462498, + "C2": 0.05180484150510463 + }, + { + "positions": 3, + "coverage_sum": 2.733816612521492, + "A2": 0.030341402471351187, + "C1": 0.028983457023720308, + "C2": 0.041168395909661606 + }, + { + "positions": 1, + "coverage_sum": 0.9199661266776685, + "A2": 0.05731816671572309, + "C1": 0.025561186420307386, + "C2": 0.03508577388925628 + }, + { + "positions": 13, + "coverage_sum": 12.818459557140034, + "A2": 0.11715790597949577, + "C1": 0.051156338131818885, + "C2": 0.03669409638402664 + }, + { + "positions": 1, + "coverage_sum": 0.9000033833979927, + "A2": 0.21246190915367164, + "C1": 0.25362228789345703, + "C2": 0.35942850711508445 + }, + { + "positions": 20, + "coverage_sum": 19.748685947589117, + "A2": 0.4575788283432587, + "C1": 0.24263144733164071, + "C2": 0.3465400658137373 + }, + { + "positions": 2, + "coverage_sum": 1.846592555344797, + "A2": 0.04292996026922499, + "C1": 0.03712427621326341, + "C2": 0.04539269500571876 + }, + { + "positions": 42, + "coverage_sum": 40.38599218353354, + "A2": 0.4101504468392339, + "C1": 0.3883999451462979, + "C2": 0.3940039756445787 + }, + { + "positions": 11, + "coverage_sum": 10.502239044211493, + "A2": 0.11395108241439637, + "C1": 0.06640467014422813, + "C2": 0.911459429876593 + }, + { + "positions": 76, + "coverage_sum": 75.99148638034198, + "A2": 0.22128923547875362, + "C1": 0.10768907853786813, + "C2": 0.08159150201988138 + }, + { + "positions": 17, + "coverage_sum": 15.928553489655641, + "A2": 0.2867142403780673, + "C1": 0.2295803402446423, + "C2": 0.2508164412677154 + }, + { + "positions": 1, + "coverage_sum": 0.8656338970659233, + "A2": 0.07221023506852448, + "C1": 0.06678562733586764, + "C2": 0.015872673266493813 + }, + { + "positions": 21, + "coverage_sum": 20.790454563143463, + "A2": 0.1750021482072327, + "C1": 0.2720456881661148, + "C2": 0.2827930632081654 + }, + { + "positions": 22, + "coverage_sum": 20.17998117167672, + "A2": 0.29194628238591586, + "C1": 0.26401415549751905, + "C2": 0.2542738798992169 + }, + { + "positions": 11, + "coverage_sum": 10.196853884481284, + "A2": 0.11150115144942117, + "C1": 0.22995706633325896, + "C2": 0.13709496612568586 + }, + { + "positions": 2, + "coverage_sum": 1.8678252470583327, + "A2": 0.011622264364689204, + "C1": 0.011244372050735406, + "C2": 0.008156922783742727 + }, + { + "positions": 1, + "coverage_sum": 0.769424341412789, + "A2": 0.10826589155152734, + "C1": 0.2835049299409679, + "C2": 0.11934244198971115 + }, + { + "positions": 13, + "coverage_sum": 12.999465353847075, + "A2": 0.030672930983044244, + "C1": 0.04377825972776813, + "C2": 0.04185706168459097 + }, + { + "positions": 3, + "coverage_sum": 2.099501870880395, + "A2": 0.04686217197068351, + "C1": 0.05563340075075718, + "C2": 0.09663056381991741 + }, + { + "positions": 1, + "coverage_sum": 0.9992377656948213, + "A2": 0.03746739329702824, + "C1": 0.09433037046561443, + "C2": 0.21300605215860166 + }, + { + "positions": 19, + "coverage_sum": 18.99502803869781, + "A2": 0.02019316236254729, + "C1": 0.017024546149511488, + "C2": 0.01227470670437906 + }, + { + "positions": 1, + "coverage_sum": 0.9808839353043716, + "A2": 0.008960887025922038, + "C1": 0.025493506784773343, + "C2": 0.035382398994092804 + }, + { + "positions": 1, + "coverage_sum": 0.971236423907021, + "A2": 0.06533569259524416, + "C1": 0.11564174230035684, + "C2": 0.030318764108644574 + }, + { + "positions": 6, + "coverage_sum": 4.781694258056697, + "A2": 0.2000105777894581, + "C1": 0.19873069151644537, + "C2": 0.06441505311316315 + }, + { + "positions": 3, + "coverage_sum": 2.916509698121274, + "A2": 0.04839639788990272, + "C1": 0.008858049098970321, + "C2": 0.0285193198674449 + }, + { + "positions": 17, + "coverage_sum": 15.902337556158448, + "A2": 0.23184869395812988, + "C1": 0.14241580439925383, + "C2": 0.16202084610057926 + }, + { + "positions": 1, + "coverage_sum": 0.9079956167281402, + "A2": 0.07294456193109987, + "C1": 0.11430193327737569, + "C2": 0.018405183962127106 + }, + { + "positions": 2, + "coverage_sum": 1.8810222206335014, + "A2": 0.01752798323007055, + "C1": 0.13994651937418356, + "C2": 0.05974518641373524 + }, + { + "positions": 3, + "coverage_sum": 2.4994548199481956, + "A2": 0.25058175024668006, + "C1": 0.3072032535726327, + "C2": 0.14785601983020885 + }, + { + "positions": 11, + "coverage_sum": 10.707614986296962, + "A2": 0.047355678905384745, + "C1": 0.11487514894275212, + "C2": 0.08581406815095527 + }, + { + "positions": 2, + "coverage_sum": 1.4440628095218735, + "A2": 0.09400090089944574, + "C1": 0.09028347511741547, + "C2": 0.07231832425420243 + }, + { + "positions": 1, + "coverage_sum": 0.9970782070526872, + "A2": 0.048805769272853135, + "C1": 0.09513574092873262, + "C2": 0.08589678889525502 + }, + { + "positions": 1, + "coverage_sum": 0.7764168577443212, + "A2": 0.0876384495340686, + "C1": 0.11390930437738686, + "C2": 0.20489728424626297 + }, + { + "positions": 7, + "coverage_sum": 6.780168378336463, + "A2": 0.11485685698315765, + "C1": 0.06624033343172038, + "C2": 0.15671968142341244 + }, + { + "positions": 11, + "coverage_sum": 10.851615794049067, + "A2": 0.01023852636106644, + "C1": 0.017584567015750206, + "C2": 0.02471448557624053 + }, + { + "positions": 49, + "coverage_sum": 46.5312193352377, + "A2": 0.16815303652621347, + "C1": 0.7540633752454279, + "C2": 0.10408344408084663 + }, + { + "positions": 8, + "coverage_sum": 7.10821081419681, + "A2": 0.2649216282052252, + "C1": 0.40100866260166523, + "C2": 0.19568605709950546 + }, + { + "positions": 15, + "coverage_sum": 14.978346885001372, + "A2": 0.08463063089165922, + "C1": 0.17068551252358238, + "C2": 0.07163568009741582 + }, + { + "positions": 10, + "coverage_sum": 9.88400411600468, + "A2": 0.034214483136096395, + "C1": 0.08959867774131518, + "C2": 0.09502345083741046 + }, + { + "positions": 5, + "coverage_sum": 3.6621878161665617, + "A2": 0.08445173961950495, + "C1": 0.056004165365340544, + "C2": 0.05713770637526386 + }, + { + "positions": 4, + "coverage_sum": 3.8017819320463033, + "A2": 0.011933144515487437, + "C1": 0.009006075799790614, + "C2": 0.025169388636962026 + }, + { + "positions": 27, + "coverage_sum": 26.363395431872448, + "A2": 0.22841583398118362, + "C1": 0.20356604602107395, + "C2": 0.22306644939923012 + }, + { + "positions": 5, + "coverage_sum": 3.9748616829325547, + "A2": 0.27514339586856573, + "C1": 0.17652802139137963, + "C2": 0.2624303397297822 + }, + { + "positions": 2, + "coverage_sum": 1.4943417772326395, + "A2": 0.11936968889532046, + "C1": 0.09429109260977293, + "C2": 0.0825154683102854 + }, + { + "positions": 1, + "coverage_sum": 0.851367442096343, + "A2": 0.6876804312192168, + "C1": 0.15842060193525231, + "C2": 0.3721095429584962 + }, + { + "positions": 1, + "coverage_sum": 0.8598187986382956, + "A2": 0.6910337314623263, + "C1": 0.6194334065145509, + "C2": 0.06637405680283595 + }, + { + "positions": 66, + "coverage_sum": 64.75006141487158, + "A2": 0.401211872630768, + "C1": 0.4054808974669435, + "C2": 0.5305317183132184 + }, + { + "positions": 1, + "coverage_sum": 0.7592798457543265, + "A2": 1.7875884836486287, + "C1": 0.9386593537911195, + "C2": 0.13753427453532524 + }, + { + "positions": 5, + "coverage_sum": 4.474426509097095, + "A2": 0.26575268385804474, + "C1": 0.02700067108431793, + "C2": 0.19915834698253773 + }, + { + "positions": 3, + "coverage_sum": 2.76919899458149, + "A2": 0.0037985857355489414, + "C1": 0.005746496482339342, + "C2": 0.004309535801124458 + }, + { + "positions": 1, + "coverage_sum": 0.8505368035665154, + "A2": 0.03785247952894294, + "C1": 0.2103305623281509, + "C2": 0.10198045049875805 + }, + { + "positions": 21, + "coverage_sum": 20.225002955485824, + "A2": 0.36273605113189006, + "C1": 0.3300819715753964, + "C2": 0.3258877030091189 + }, + { + "positions": 2, + "coverage_sum": 1.881839563193846, + "A2": 0.01676361045153678, + "C1": 0.021235737471012997, + "C2": 0.019212383027967513 + }, + { + "positions": 45, + "coverage_sum": 44.60808939102878, + "A2": 0.22472862228759563, + "C1": 0.2586128073956945, + "C2": 0.23115507711757124 + }, + { + "positions": 6, + "coverage_sum": 5.445679445699365, + "A2": 0.2544760552609757, + "C1": 0.7450461896901812, + "C2": 0.7224292648205173 + }, + { + "positions": 2, + "coverage_sum": 1.964096909430654, + "A2": 0.0043100257610376004, + "C1": 0.007361919201698916, + "C2": 0.005318127082300609 + } + ], + "corrected_pair_reports": { + "A2": { + "positions": 5238, + "kl_mean": 0.011112688904474985, + "kl_p50": 0.0005208433122413416, + "kl_p99": 0.17306065685855543, + "kl_max": 2.2881015592295406, + "coverage_mean": 0.980634700776285, + "divergence_median": 14.0, + "identical": 2, + "passages": 128, + "confident_flips": 6 + }, + "C1": { + "positions": 5871, + "kl_mean": 0.010848314092625383, + "kl_p50": 0.0005053332210726754, + "kl_p99": 0.1831704402887287, + "kl_max": 2.325543450304667, + "coverage_mean": 0.9809179234932481, + "divergence_median": 15.0, + "identical": 2, + "passages": 128, + "confident_flips": 2 + }, + "C2": { + "positions": 5292, + "kl_mean": 0.011200947049793596, + "kl_p50": 0.0005368146625853158, + "kl_p99": 0.21313361814283333, + "kl_max": 1.9570430229365317, + "coverage_mean": 0.9795936612382408, + "divergence_median": 13.0, + "identical": 2, + "passages": 128, + "confident_flips": 3 + } + }, + "gate": { + "criterion": "Upper 95% passage-bootstrap candidate/floor KL ratio <= 1.5 on common contexts and vocabulary partition", + "passed": true, + "limitations": "Conditional on these four boots and shared generated prefixes; lower-bound top-k KL, not a universal accuracy or losslessness guarantee." + } +} diff --git a/bench_results/2026-11-14_leapquant-acceptance/pilot-metrics.json b/bench_results/2026-11-14_leapquant-acceptance/pilot-metrics.json new file mode 100644 index 000000000000..9cbbb67917d6 --- /dev/null +++ b/bench_results/2026-11-14_leapquant-acceptance/pilot-metrics.json @@ -0,0 +1,237 @@ +{ + "protocol": "One boot per arm; dataset seed 12345, thinking sampling values T=1, top_p=.95, top_k=20, min_p=0, presence_penalty=0, repetition_penalty=1. Random/completions throughput corpus; not a model-quality corpus or an ABBA result. Effective-wave prefill is c*mean_input/p99_TTFT, not a separately timed GPU prefill rate.", + "rows": [ + { + "arm": "baseline-1024", + "cell": "1024x512-c1", + "completed": 16, + "prefill_per_request_tps": 2535.8593068582327, + "prefill_effective_wave_tps": 2487.7841698674456, + "decode_per_request_tps": 87.38243295241118, + "output_aggregate_tps": 81.89322790163844, + "ttft_mean_s": 0.40380789156188257, + "ttft_p99_s": 0.4116112693387549, + "itl_mean_ms": 23.699523071684613, + "itl_median_ms": 23.70007403078489, + "itl_p99_ms": 24.211331792757846, + "maximum_stream_gap_ms": 25.831288017798215, + "acceptance_length": 2.0729483282674774, + "acceptance_rate_percent": 53.64741641337386, + "acceptance_per_position": [ + 0.6332320162107397, + 0.4397163120567376 + ], + "accepted_tokens": 4236, + "draft_steps": 3948, + "generated_tokens": 8192, + "corruption_flags": [ + { + "request": 3, + "flags": [ + "repeat30:'\\n'" + ], + "tail": " issue\n- **Get help with a text** \u2192 resend the clean version\n- **Test the model** \u2192 no problem! Just let me know what you're curious about\n\nI'll be glad to assist once the request is legible. \ud83d\ude42\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 10, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "question, ask for help with a task, or just chat in English, Hindi, or any other language you prefer. \ud83d\ude0a\n\n\n\n\nuser\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 13, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "ems and doesn't form a coherent sentence in any language I can identify.\n\nIf you have a question or need help with something, please feel free to ask again in any language. I'm happy to help! \ud83d\ude0a\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + } + ] + }, + { + "arm": "baseline-1024", + "cell": "16384x1024-c1", + "completed": 4, + "prefill_per_request_tps": 2772.2956324236716, + "prefill_effective_wave_tps": 2759.7367047187445, + "decode_per_request_tps": 64.51790209865202, + "output_aggregate_tps": 47.04496041517348, + "ttft_mean_s": 5.9099036222469294, + "ttft_p99_s": 5.936798235855531, + "itl_mean_ms": 24.07906489787873, + "itl_median_ms": 24.114163010381162, + "itl_p99_ms": 24.594302339828573, + "maximum_stream_gap_ms": 25.78802895732224, + "acceptance_length": 1.554290053151101, + "acceptance_rate_percent": 27.71450265755505, + "acceptance_per_position": [ + 0.3337129840546697, + 0.2205770690964313 + ], + "accepted_tokens": 1460, + "draft_steps": 2634, + "generated_tokens": 4096, + "corruption_flags": [ + { + "request": 3, + "flags": [ + "empty" + ], + "tail": "\n\n" + } + ] + }, + { + "arm": "window16-512", + "cell": "1024x512-c1", + "completed": 16, + "prefill_per_request_tps": 2069.6267232318633, + "prefill_effective_wave_tps": 2017.7462693445857, + "decode_per_request_tps": 89.18041394368278, + "output_aggregate_tps": 82.24750192335168, + "ttft_mean_s": 0.4947752116386255, + "ttft_p99_s": 0.5074969115579734, + "itl_mean_ms": 23.567949428766543, + "itl_median_ms": 23.569382989080623, + "itl_p99_ms": 24.194220601348206, + "maximum_stream_gap_ms": 24.613327987026423, + "acceptance_length": 2.104884318766067, + "acceptance_rate_percent": 55.24421593830334, + "acceptance_per_position": [ + 0.6419023136246786, + 0.46298200514138815 + ], + "accepted_tokens": 4298, + "draft_steps": 3890, + "generated_tokens": 8192, + "corruption_flags": [ + { + "request": 0, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "rm a coherent message. This might be a copy-paste glitch, encoding error, or accidental paste.\n\nCould you re-send what you meant to ask? I'm happy to help once I can see the actual question. \ud83d\ude42\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 11, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "packskeletonSuggests\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + } + ] + }, + { + "arm": "window16-512", + "cell": "16384x1024-c1", + "completed": 4, + "prefill_per_request_tps": 2735.7049546287926, + "prefill_effective_wave_tps": 2725.457958331893, + "decode_per_request_tps": 89.65990646335841, + "output_aggregate_tps": 58.85331327987859, + "ttft_mean_s": 5.9889499312703265, + "ttft_p99_s": 6.011466788512771, + "itl_mean_ms": 23.95754781364661, + "itl_median_ms": 24.006497056689113, + "itl_p99_ms": 24.669888496864587, + "maximum_stream_gap_ms": 25.03556798910722, + "acceptance_length": 2.150656167979003, + "acceptance_rate_percent": 57.53280839895013, + "acceptance_per_position": [ + 0.6493438320209973, + 0.5013123359580053 + ], + "accepted_tokens": 2192, + "draft_steps": 1905, + "generated_tokens": 4096, + "corruption_flags": [] + }, + { + "arm": "window32-512", + "cell": "1024x512-c1", + "completed": 16, + "prefill_per_request_tps": 2067.5232733332214, + "prefill_effective_wave_tps": 2026.0172195081932, + "decode_per_request_tps": 91.00534860707369, + "output_aggregate_tps": 83.78733369776965, + "ttft_mean_s": 0.49527858438523253, + "ttft_p99_s": 0.5054251218301943, + "itl_mean_ms": 23.561729181167948, + "itl_median_ms": 23.567059019114822, + "itl_p99_ms": 24.281382472254336, + "maximum_stream_gap_ms": 24.73050798289478, + "acceptance_length": 2.148701809598741, + "acceptance_rate_percent": 57.43509047993706, + "acceptance_per_position": [ + 0.6637817991083137, + 0.4849200104904275 + ], + "accepted_tokens": 4380, + "draft_steps": 3813, + "generated_tokens": 8192, + "corruption_flags": [ + { + "request": 1, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "rbled structure and responding to the implicit intent of a new interaction.\n\nPlease provide a clear question or context, and I will be happy to assist you!\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 6, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "lease provide more context so I can help you better.\nuser\nassistant\nuser\nassistant\nuser\nassistant\nuser\nassistant\nuser\nassistant\nuser\nassistant\nuser\nassistant\nuser\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 10, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "ake main behtar tareeqay se aapki madad kar sakoon. \ud83d\ude0a\n\nKya aap kisi sawal ka jawab dhoond rahe hain? Ya kisi baat ko samajhna chahte hain?\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 11, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "c question \u2014 whether about programming, science, writing, or anything else \u2014 I'd be happy to help. Just send it in a clear format and I'll do my best to give you a solid answer! \ud83d\ude42\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + } + ] + }, + { + "arm": "window32-512", + "cell": "16384x1024-c1", + "completed": 4, + "prefill_per_request_tps": 2726.4142041299306, + "prefill_effective_wave_tps": 2719.300602188987, + "decode_per_request_tps": 71.14016899969934, + "output_aggregate_tps": 50.220932468641415, + "ttft_mean_s": 6.009358363517094, + "ttft_p99_s": 6.025078649565694, + "itl_mean_ms": 24.056982726879404, + "itl_median_ms": 24.09440401243046, + "itl_p99_ms": 24.595086637418717, + "maximum_stream_gap_ms": 27.68424100941047, + "acceptance_length": 1.71267252195734, + "acceptance_rate_percent": 35.633626097867, + "acceptance_per_position": [ + 0.41614387285654536, + 0.29652864910079463 + ], + "accepted_tokens": 1704, + "draft_steps": 2391, + "generated_tokens": 4096, + "corruption_flags": [ + { + "request": 3, + "flags": [ + "empty" + ], + "tail": "\n\n" + } + ] + } + ] +} diff --git a/bench_results/2026-11-14_leapquant-acceptance/qualified-comparison.json b/bench_results/2026-11-14_leapquant-acceptance/qualified-comparison.json new file mode 100644 index 000000000000..f2e1a33005c8 --- /dev/null +++ b/bench_results/2026-11-14_leapquant-acceptance/qualified-comparison.json @@ -0,0 +1,1057 @@ +{ + "protocol": { + "baseline_arms": [ + "A1", + "A2" + ], + "candidate_arms": [ + "C1", + "C2" + ], + "order": "A1, rejected B1, C1, C2, A2; the accepted controls bracket the repeated candidate, not an uninterrupted ABBA", + "baseline": "LeapQuant off, block1024, mixed-step budget0", + "candidate": "LeapQuant on, window32, block512, adaptive mixed-step budget450ms, decode_share0, max_overhead1", + "shared": "Flash-Next AWQ-W4A16, MTP2, greedy/one-hot drafting, MonoKernel, FULL_AND_PIECEWISE, TP4 b4 HIP2-5 (d9/dc/df/e2)", + "sampling": "Thinking T1/.95/k20/min0/pres0/rep1 except explicit greedy T0 cell. Random cells have dataset seed12345, no API seed; templated agentic cells API seed12345, EOS allowed. KL T0 seed12345, ignore EOS, 128x512 per boot.", + "aggregation": "Rates and latency summaries are means of two boot summaries, including per-boot p99 values; acceptance is pooled as 1+sum(accepted)/sum(draft_steps). Effective-wave prefill = concurrency*mean_input/p99_TTFT, not separately timed GPU throughput." + }, + "preemptions_total": { + "A1": 14.0, + "A2": 10.0, + "C1": 0.0, + "C2": 0.0 + }, + "capacity_tokens": { + "baseline": 266945, + "candidate": 273456, + "percent_delta": 2.439079211073447 + }, + "comparisons": { + "bench-1024x512-c1.json": { + "baseline": { + "prefill_per_request_tps": 2517.8562346563153, + "prefill_effective_wave_tps": 2466.7276482365137, + "decode_per_request_tps": 88.00703710900318, + "output_aggregate_tps": 82.38294444579583, + "ttft_mean_s": 0.4067079229389492, + "ttft_p99_s": 0.41513236717000834, + "itl_mean_ms": 23.675173532820306, + "itl_median_ms": 23.68379247491248, + "itl_p99_ms": 24.20597830612678, + "maximum_stream_gap_ms": 25.585468480130658, + "acceptance_length": 2.076864535768645, + "acceptance_length_per_boot": [ + 2.2285714285714286, + 1.9444048467569495 + ], + "output_aggregate_tps_per_boot": [ + 87.60096785165912, + 77.16492103993254 + ], + "completed": 32, + "corruption_flag_counts_per_boot": [ + 4, + 1 + ] + }, + "candidate": { + "prefill_per_request_tps": 2054.0647460859786, + "prefill_effective_wave_tps": 2013.7236082900133, + "decode_per_request_tps": 91.32320526413608, + "output_aggregate_tps": 84.0099027384095, + "ttft_mean_s": 0.49853082837580587, + "ttft_p99_s": 0.5085128714083111, + "itl_mean_ms": 23.596989390356235, + "itl_median_ms": 23.59654124302324, + "itl_p99_ms": 24.23233177163638, + "maximum_stream_gap_ms": 25.328585994429886, + "acceptance_length": 2.1580819391384534, + "acceptance_length_per_boot": [ + 2.2020962106960495, + 2.115762273901809 + ], + "output_aggregate_tps_per_boot": [ + 85.53603706166338, + 82.4837684151556 + ], + "completed": 32, + "corruption_flag_counts_per_boot": [ + 0, + 2 + ] + }, + "percent_delta": { + "prefill_per_request_tps": -18.42009413351766, + "prefill_effective_wave_tps": -18.36457463272677, + "decode_per_request_tps": 3.768071581623156, + "output_aggregate_tps": 1.9748727161410518, + "ttft_mean_s": 22.577112531599287, + "ttft_p99_s": 22.494151654539827, + "itl_mean_ms": -0.33023682954503863, + "itl_median_ms": -0.36840059285970783, + "itl_p99_ms": 0.10887172241631493, + "maximum_stream_gap_ms": -1.0040171275357501, + "acceptance_length": 3.9105777950871445 + } + }, + "bench-32768x2048-c8.json": { + "baseline": { + "prefill_per_request_tps": 407.2159994972895, + "prefill_effective_wave_tps": 1799.2981072950984, + "decode_per_request_tps": 20.210457017911544, + "output_aggregate_tps": 82.31701601501254, + "ttft_mean_s": 80.56362891624667, + "ttft_p99_s": 145.694371291867, + "itl_mean_ms": 113.33798366754365, + "itl_median_ms": 44.58355699898675, + "itl_p99_ms": 434.90863372571766, + "maximum_stream_gap_ms": 455.97916550468653, + "acceptance_length": 2.280999791129987, + "acceptance_length_per_boot": [ + 2.120590329471163, + 2.4676508247345033 + ], + "output_aggregate_tps_per_boot": [ + 78.54624888374738, + 86.08778314627769 + ], + "completed": 32, + "corruption_flag_counts_per_boot": [ + 3, + 4 + ] + }, + "candidate": { + "prefill_per_request_tps": 536.3846215419022, + "prefill_effective_wave_tps": 1909.8842110340095, + "decode_per_request_tps": 23.409080422075576, + "output_aggregate_tps": 100.45949176573534, + "ttft_mean_s": 61.11852181294125, + "ttft_p99_s": 137.31352753602695, + "itl_mean_ms": 97.2865884474271, + "itl_median_ms": 45.69040652131662, + "itl_p99_ms": 436.1885846001678, + "maximum_stream_gap_ms": 441.6392714774702, + "acceptance_length": 2.2767605878469928, + "acceptance_length_per_boot": [ + 2.2235190337246387, + 2.332621386871707 + ], + "output_aggregate_tps_per_boot": [ + 101.03564463216186, + 99.88333889930881 + ], + "completed": 32, + "corruption_flag_counts_per_boot": [ + 3, + 6 + ] + }, + "percent_delta": { + "prefill_per_request_tps": 31.719928049013824, + "prefill_effective_wave_tps": 6.146069030504142, + "decode_per_request_tps": 15.826576318037965, + "output_aggregate_tps": 22.039763622401075, + "ttft_mean_s": -24.13633467717846, + "ttft_p99_s": -5.752345599577646, + "itl_mean_ms": -14.162414665149148, + "itl_median_ms": 2.482640679286807, + "itl_p99_ms": 0.29430339505684966, + "maximum_stream_gap_ms": -3.144857289991465, + "acceptance_length": -0.18584847308968921 + } + }, + "greedy-1024x512-c1.json": { + "baseline": { + "prefill_per_request_tps": 2532.122175504175, + "prefill_effective_wave_tps": 2466.046708883563, + "decode_per_request_tps": 85.9373030177309, + "output_aggregate_tps": 80.61735697609461, + "ttft_mean_s": 0.4044041465349437, + "ttft_p99_s": 0.4152406490786234, + "itl_mean_ms": 23.48317176418994, + "itl_median_ms": 23.481810989324003, + "itl_p99_ms": 24.047488028299995, + "maximum_stream_gap_ms": 24.78781051468104, + "acceptance_length": 2.0206096507466373, + "acceptance_length_per_boot": [ + 2.0083456062837506, + 2.0330106726234796 + ], + "output_aggregate_tps_per_boot": [ + 80.2010718896197, + 81.03364206256953 + ], + "completed": 32, + "corruption_flag_counts_per_boot": [ + 2, + 3 + ] + }, + "candidate": { + "prefill_per_request_tps": 2053.2499418249477, + "prefill_effective_wave_tps": 2015.415586412402, + "decode_per_request_tps": 88.93250016609886, + "output_aggregate_tps": 81.98053170225919, + "ttft_mean_s": 0.498724023496834, + "ttft_p99_s": 0.5080926415292197, + "itl_mean_ms": 23.420604571939812, + "itl_median_ms": 23.42107647564262, + "itl_p99_ms": 24.07252019038424, + "maximum_stream_gap_ms": 28.539560007629916, + "acceptance_length": 2.084372613896666, + "acceptance_length_per_boot": [ + 2.021229326092323, + 2.151562910428159 + ], + "output_aggregate_tps_per_boot": [ + 79.73630601295496, + 84.22475739156343 + ], + "completed": 32, + "corruption_flag_counts_per_boot": [ + 0, + 1 + ] + }, + "percent_delta": { + "prefill_per_request_tps": -18.911892890155602, + "prefill_effective_wave_tps": -18.273422025942576, + "decode_per_request_tps": 3.4853283070216623, + "output_aggregate_tps": 1.6909196447221664, + "ttft_mean_s": 23.3231725663674, + "ttft_p99_s": 22.361007443906434, + "itl_mean_ms": -0.2664341634869727, + "itl_median_ms": -0.2586449303633165, + "itl_p99_ms": 0.10409470650234454, + "maximum_stream_gap_ms": 15.135461402397077, + "acceptance_length": 3.155629942006244 + } + }, + "agentic-c1.json": { + "baseline": { + "prefill_per_request_tps": 2347.1297079310843, + "prefill_effective_wave_tps": 2216.3898085901583, + "decode_per_request_tps": 82.19528940946867, + "output_aggregate_tps": 79.55212655892055, + "ttft_mean_s": 0.42482932043822075, + "ttft_p99_s": 0.4498894843272865, + "itl_mean_ms": 23.88193494006121, + "itl_median_ms": 23.886592985945754, + "itl_p99_ms": 24.37753264530329, + "maximum_stream_gap_ms": 28.060377022484317, + "acceptance_length": 1.9652126233702938, + "acceptance_length_per_boot": [ + 1.9672071636011617, + 1.9631901840490797 + ], + "output_aggregate_tps_per_boot": [ + 79.6473380126734, + 79.45691510516768 + ], + "completed": 32, + "corruption_flag_counts_per_boot": [ + 0, + 0 + ] + }, + "candidate": { + "prefill_per_request_tps": 1926.1411377091881, + "prefill_effective_wave_tps": 1865.506539459761, + "decode_per_request_tps": 81.86340643812206, + "output_aggregate_tps": 78.62088185685326, + "ttft_mean_s": 0.5176802224432322, + "ttft_p99_s": 0.5345063869157457, + "itl_mean_ms": 23.810569439084478, + "itl_median_ms": 23.81068651448004, + "itl_p99_ms": 24.383006834541447, + "maximum_stream_gap_ms": 26.77231698180549, + "acceptance_length": 1.9502265176683782, + "acceptance_length_per_boot": [ + 1.9588348579682235, + 1.9415545046683642 + ], + "output_aggregate_tps_per_boot": [ + 78.97712217658392, + 78.26464153712259 + ], + "completed": 32, + "corruption_flag_counts_per_boot": [ + 0, + 0 + ] + }, + "percent_delta": { + "prefill_per_request_tps": -17.93631467401874, + "prefill_effective_wave_tps": -15.831297715341586, + "decode_per_request_tps": -0.4037737122541074, + "output_aggregate_tps": -1.1706094385516619, + "ttft_mean_s": 21.856048426514853, + "ttft_p99_s": 18.808375286874224, + "itl_mean_ms": -0.2988262934131791, + "itl_median_ms": -0.31777856101276036, + "itl_p99_ms": 0.02245587901699153, + "maximum_stream_gap_ms": -4.590316230058944, + "acceptance_length": -0.7625691756556474 + } + }, + "agentic-c8.json": { + "baseline": { + "prefill_per_request_tps": 990.2480035889034, + "prefill_effective_wave_tps": 2795.340307872814, + "decode_per_request_tps": 39.128719457257375, + "output_aggregate_tps": 288.96763225084493, + "ttft_mean_s": 1.0081444940024085, + "ttft_p99_s": 2.856986685660086, + "itl_mean_ms": 50.9921333011215, + "itl_median_ms": 47.46040949248709, + "itl_p99_ms": 368.365953799512, + "maximum_stream_gap_ms": 684.3110420159064, + "acceptance_length": 1.9933893663235434, + "acceptance_length_per_boot": [ + 2.003533123028391, + 1.9832883080982597 + ], + "output_aggregate_tps_per_boot": [ + 291.4461060524896, + 286.48915844920026 + ], + "completed": 64, + "corruption_flag_counts_per_boot": [ + 0, + 0 + ] + }, + "candidate": { + "prefill_per_request_tps": 867.5457031541815, + "prefill_effective_wave_tps": 2469.017907224006, + "decode_per_request_tps": 39.22211535514063, + "output_aggregate_tps": 288.8587366316688, + "ttft_mean_s": 1.1509892123312966, + "ttft_p99_s": 3.2345931410902997, + "itl_mean_ms": 50.848705760316385, + "itl_median_ms": 46.39309199410491, + "itl_p99_ms": 273.59915793989785, + "maximum_stream_gap_ms": 395.14828348183073, + "acceptance_length": 1.9954854688528703, + "acceptance_length_per_boot": [ + 1.98421569831716, + 2.0072366314441243 + ], + "output_aggregate_tps_per_boot": [ + 290.5602171200135, + 287.15725614332416 + ], + "completed": 64, + "corruption_flag_counts_per_boot": [ + 0, + 0 + ] + }, + "percent_delta": { + "prefill_per_request_tps": -12.391067691125702, + "prefill_effective_wave_tps": -11.673798704571004, + "decode_per_request_tps": 0.2386888688889366, + "output_aggregate_tps": -0.037684365659884644, + "ttft_mean_s": 14.169071911684394, + "ttft_p99_s": 13.216948378706572, + "itl_mean_ms": -0.28127385837759533, + "itl_median_ms": -2.248858595607217, + "itl_p99_ms": -25.726263483946244, + "maximum_stream_gap_ms": -42.25604159217327, + "acceptance_length": 0.10515268942128397 + } + } + }, + "per_boot": { + "A1": { + "bench-1024x512-c1.json": { + "completed": 16, + "prefill_per_request_tps": 2531.9487217335723, + "prefill_effective_wave_tps": 2477.208815209293, + "decode_per_request_tps": 93.93601342033841, + "output_aggregate_tps": 87.60096785165912, + "ttft_mean_s": 0.40443157130721374, + "ttft_p99_s": 0.4133684628090123, + "itl_mean_ms": 23.68380159808031, + "itl_median_ms": 23.68311397731304, + "itl_p99_ms": 24.223012233851478, + "maximum_stream_gap_ms": 26.32979699410498, + "acceptance_length": 2.2285714285714286, + "acceptance_rate_percent": 61.42857142857143, + "acceptance_per_position": [ + 0.7006802721088435, + 0.527891156462585 + ], + "accepted_tokens": 4515, + "draft_steps": 3675, + "generated_tokens": 8192, + "corruption_flags": [ + { + "request": 7, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "lp with something, please feel free to type it again in a clear format, and I'll be happy to assist! \ud83d\ude0a\n\n(If this was a test or encoding issue \u2014 no worries. Just let me know how I can help.)code\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 10, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "hari), silakan tanyakan dengan jelas dan saya akan dengan senang hati membantu! \ud83d\ude0a\n\nApa yang bisa saya bantu hari ini?\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 13, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "\u0e31\u0e07\u0e01\u0e24\u0e29\u0e17\u0e35\u0e48\u0e0a\u0e31\u0e14\u0e40\u0e08\u0e19\u0e2d\u0e35\u0e01\u0e04\u0e23\u0e31\u0e49\u0e07\u0e2b\u0e19\u0e36\u0e48\u0e07 \u0e09\u0e31\u0e19\u0e22\u0e34\u0e19\u0e14\u0e35\u0e0a\u0e48\u0e27\u0e22\u0e40\u0e2b\u0e25\u0e37\u0e2d\u0e2d\u0e22\u0e48\u0e32\u0e07\u0e40\u0e15\u0e47\u0e21\u0e17\u0e35\u0e48\u0e04\u0e23\u0e31\u0e1a \ud83d\ude0a\n\nuser\nassistant\nuser\nassistant\nuser\nassistant\nuser\nassistant\nuser\nassistant\nuser\n\nuser\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 14, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "$__\n\n\nton\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + } + ] + }, + "bench-32768x2048-c8.json": { + "completed": 16, + "prefill_per_request_tps": 393.2120241293086, + "prefill_effective_wave_tps": 1792.6242996486765, + "decode_per_request_tps": 19.153110521916606, + "output_aggregate_tps": 78.54624888374738, + "ttft_mean_s": 83.33417593868944, + "ttft_p99_s": 146.23476879755324, + "itl_mean_ms": 110.68738756909222, + "itl_median_ms": 44.354368990752846, + "itl_p99_ms": 433.99033051449805, + "maximum_stream_gap_ms": 454.68425401486456, + "acceptance_length": 2.120590329471163, + "acceptance_rate_percent": 56.02951647355816, + "acceptance_per_position": [ + 0.6259304809372775, + 0.49465984853388567 + ], + "accepted_tokens": 17312, + "draft_steps": 15449, + "generated_tokens": 32768, + "corruption_flags": [ + { + "request": 9, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 10, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 12, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + } + ] + }, + "greedy-1024x512-c1.json": { + "completed": 16, + "prefill_per_request_tps": 2534.2266249505788, + "prefill_effective_wave_tps": 2470.150715421965, + "decode_per_request_tps": 85.45860258040771, + "output_aggregate_tps": 80.2010718896197, + "ttft_mean_s": 0.4040680458165298, + "ttft_p99_s": 0.41454960363626014, + "itl_mean_ms": 23.483567383932087, + "itl_median_ms": 23.481965996325016, + "itl_p99_ms": 24.02258750400506, + "maximum_stream_gap_ms": 25.15287499409169, + "acceptance_length": 2.0083456062837506, + "acceptance_rate_percent": 50.41728031418753, + "acceptance_per_position": [ + 0.6067746686303387, + 0.4015709376534119 + ], + "accepted_tokens": 4108, + "draft_steps": 4074, + "generated_tokens": 8192, + "corruption_flags": [ + { + "request": 12, + "flags": [ + "repeat30:'\u7684'" + ], + "tail": "\u6587\u6863\u6216\u7a0b\u5e8f\u4e2d\u9047\u5230\u8fd9\u6bb5\u6587\u5b57\uff0c\u5efa\u8bae\uff1a\n- \u68c0\u67e5\u6570\u636e\u6765\u6e90\u662f\u5426\u53ef\u9760\uff1b\n- \u5982\u679c\u662f\u81ea\u5df1\u751f\u6210\u7684\u5185\u5bb9\uff0c\u8bf7\u68c0\u67e5\u8f93\u5165\u6570\u636e\u6216\u6a21\u578b\u8bbe\u7f6e\uff1b\n- \u5982\u679c\u662f\u4ece\u7f51\u9875\u590d\u5236\u7684\uff0c\u53ef\u80fd\u662f\u8be5\u9875\u9762\u5b58\u5728\u6076\u610f\u811a\u672c\u6216\u5185\u5bb9\u6c61\u67d3\u3002\n\n\u5982\u9700\u5e2e\u52a9\u5206\u6790\u5177\u4f53\u6765\u6e90\u6216\u6280\u672f\u539f\u56e0\uff0c\u53ef\u4ee5\u63d0\u4f9b\u66f4\u591a\u4e0a\u4e0b\u6587\uff08\u5982\uff1a\u8fd9\u6bb5\u6587\u5b57\u51fa\u73b0\u5728\u54ea\u91cc\uff1f\u4f60\u5e0c\u671b\u7528\u5b83\u505a\u4ec0\u4e48\uff1f\uff09\u3002\u3010\u7cfb\u7edf\u63d0\u793a\u3011\u8fd9\u662f\u4e00\u4e2a\u6a21\u62df\u7684AI\u52a9\u624b\u56de\u590d\uff0c\u7528\u4e8e\u6f14\u793a\u5982\u4f55\u5904\u7406\u65e0\u610f\u4e49\u6216\u4e71\u7801\u8f93\u5165\u3002\u5b9e\u9645\u5e94\u7528\u4e2d\uff0c\u5e94\u7ed3\u5408\u5177\u4f53\u573a\u666f\u8fdb\u884c\u66f4\u7cbe\u51c6\u7684\u5904\u7406\u3002\n\n " + }, + { + "request": 14, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "n.\n\n### How can I help you?\n\nIf you have a **specific question**, a **code snippet** you need help with, or a **text** you want summarized or analyzed, please provide it clearly, and I will be happy to assist!\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + } + ] + }, + "agentic-c1.json": { + "completed": 16, + "prefill_per_request_tps": 2352.1262885439573, + "prefill_effective_wave_tps": 2221.5800383501382, + "decode_per_request_tps": 82.3116940883631, + "output_aggregate_tps": 79.6473380126734, + "ttft_mean_s": 0.42392494180967333, + "ttft_p99_s": 0.4488359558454249, + "itl_mean_ms": 23.88045701790443, + "itl_median_ms": 23.88645597966388, + "itl_p99_ms": 24.36973957985174, + "maximum_stream_gap_ms": 26.67556202504784, + "acceptance_length": 1.9672071636011617, + "acceptance_rate_percent": 48.360358180058086, + "acceptance_per_position": [ + 0.5992255566311714, + 0.3679816069699903 + ], + "accepted_tokens": 7993, + "draft_steps": 8264, + "generated_tokens": 16259, + "corruption_flags": [] + }, + "agentic-c8.json": { + "completed": 32, + "prefill_per_request_tps": 984.6571535100127, + "prefill_effective_wave_tps": 2795.1388468856894, + "decode_per_request_tps": 39.25146801841164, + "output_aggregate_tps": 291.4461060524896, + "ttft_mean_s": 1.0138363860369282, + "ttft_p99_s": 2.8571925895195456, + "itl_mean_ms": 51.04093233114975, + "itl_median_ms": 47.46358498232439, + "itl_p99_ms": 367.43880002468404, + "maximum_stream_gap_ms": 684.0341520146467, + "acceptance_length": 2.003533123028391, + "acceptance_rate_percent": 50.176656151419564, + "acceptance_per_position": [ + 0.6131230283911672, + 0.390410094637224 + ], + "accepted_tokens": 15906, + "draft_steps": 15850, + "generated_tokens": 31760, + "corruption_flags": [] + } + }, + "C1": { + "bench-1024x512-c1.json": { + "completed": 16, + "prefill_per_request_tps": 2061.820582042356, + "prefill_effective_wave_tps": 2017.8856071821822, + "decode_per_request_tps": 93.09953051092047, + "output_aggregate_tps": 85.53603706166338, + "ttft_mean_s": 0.4966484518190555, + "ttft_p99_s": 0.5074618681828724, + "itl_mean_ms": 23.601180256647353, + "itl_median_ms": 23.59938999870792, + "itl_p99_ms": 24.28793321596459, + "maximum_stream_gap_ms": 25.961947976611555, + "acceptance_length": 2.2020962106960495, + "acceptance_rate_percent": 60.104810534802475, + "acceptance_per_position": [ + 0.6898683149690943, + 0.5122278957269552 + ], + "accepted_tokens": 4473, + "draft_steps": 3721, + "generated_tokens": 8192, + "corruption_flags": [] + }, + "bench-32768x2048-c8.json": { + "completed": 16, + "prefill_per_request_tps": 547.8716110596232, + "prefill_effective_wave_tps": 1948.8099583881838, + "decode_per_request_tps": 23.513568188604783, + "output_aggregate_tps": 101.03564463216186, + "ttft_mean_s": 59.80963302081727, + "ttft_p99_s": 134.5149119705922, + "itl_mean_ms": 94.51705982031665, + "itl_median_ms": 45.511523028835654, + "itl_p99_ms": 435.83857964957133, + "maximum_stream_gap_ms": 441.87965797027573, + "acceptance_length": 2.2235190337246387, + "acceptance_rate_percent": 61.17595168623193, + "acceptance_per_position": [ + 0.6667571418877655, + 0.5567618918368732 + ], + "accepted_tokens": 18031, + "draft_steps": 14737, + "generated_tokens": 32768, + "corruption_flags": [ + { + "request": 3, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 13, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "\u0627\u0644\u0639\u0631\u0628\u064a\u0629, \u4e2d\u6587, \u65e5\u672c\u8a9e, \ud55c\uad6d\uc5b4, \u0e44\u0e17\u0e22, \u0939\u093f\u0928\u094d\u0926\u0940, \u0438\u043b\u0438 \u043b\u044e\u0431\u043e\u043c \u0434\u0440\u0443\u0433\u043e\u043c \u044f\u0437\u044b\u043a\u0435.)\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 14, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "ings.\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + } + ] + }, + "greedy-1024x512-c1.json": { + "completed": 16, + "prefill_per_request_tps": 2048.6802143371483, + "prefill_effective_wave_tps": 2007.0059869040074, + "decode_per_request_tps": 86.30385866056078, + "output_aggregate_tps": 79.73630601295496, + "ttft_mean_s": 0.4998339871854114, + "ttft_p99_s": 0.5102127281541471, + "itl_mean_ms": 23.385595103951783, + "itl_median_ms": 23.3892829855904, + "itl_p99_ms": 24.0378970047459, + "maximum_stream_gap_ms": 26.18515200447291, + "acceptance_length": 2.021229326092323, + "acceptance_rate_percent": 51.06146630461615, + "acceptance_per_position": [ + 0.6158973093063441, + 0.40533201678597874 + ], + "accepted_tokens": 4137, + "draft_steps": 4051, + "generated_tokens": 8192, + "corruption_flags": [] + }, + "agentic-c1.json": { + "completed": 16, + "prefill_per_request_tps": 1925.3397054598524, + "prefill_effective_wave_tps": 1866.2542239986853, + "decode_per_request_tps": 82.24060814100929, + "output_aggregate_tps": 78.97712217658392, + "ttft_mean_s": 0.5178956197560183, + "ttft_p99_s": 0.5342921597592067, + "itl_mean_ms": 23.810526338578484, + "itl_median_ms": 23.809028032701463, + "itl_p99_ms": 24.395217173732817, + "maximum_stream_gap_ms": 27.116021956317127, + "acceptance_length": 1.9588348579682235, + "acceptance_rate_percent": 47.94174289841117, + "acceptance_per_position": [ + 0.594607607125662, + 0.3642272508425614 + ], + "accepted_tokens": 7966, + "draft_steps": 8308, + "generated_tokens": 16278, + "corruption_flags": [] + }, + "agentic-c8.json": { + "completed": 32, + "prefill_per_request_tps": 853.6956755674167, + "prefill_effective_wave_tps": 2465.271467240546, + "decode_per_request_tps": 39.020059547615425, + "output_aggregate_tps": 290.5602171200135, + "ttft_mean_s": 1.1693643046000943, + "ttft_p99_s": 3.2395012501155724, + "itl_mean_ms": 50.808861953625325, + "itl_median_ms": 46.44924699096009, + "itl_p99_ms": 272.8392387984786, + "maximum_stream_gap_ms": 406.303427007515, + "acceptance_length": 1.98421569831716, + "acceptance_rate_percent": 49.210784915858, + "acceptance_per_position": [ + 0.6051467878638987, + 0.37906891045326124 + ], + "accepted_tokens": 16025, + "draft_steps": 16282, + "generated_tokens": 32306, + "corruption_flags": [] + } + }, + "C2": { + "bench-1024x512-c1.json": { + "completed": 16, + "prefill_per_request_tps": 2046.3089101296014, + "prefill_effective_wave_tps": 2009.5616093978444, + "decode_per_request_tps": 89.54688001735167, + "output_aggregate_tps": 82.4837684151556, + "ttft_mean_s": 0.5004132049325563, + "ttft_p99_s": 0.50956387463375, + "itl_mean_ms": 23.592798524065117, + "itl_median_ms": 23.593692487338558, + "itl_p99_ms": 24.176730327308178, + "maximum_stream_gap_ms": 24.695224012248218, + "acceptance_length": 2.115762273901809, + "acceptance_rate_percent": 55.78811369509044, + "acceptance_per_position": [ + 0.641860465116279, + 0.47390180878552973 + ], + "accepted_tokens": 4318, + "draft_steps": 3870, + "generated_tokens": 8192, + "corruption_flags": [ + { + "request": 3, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "id SDK).\n* A brief description of what you were trying to do when this output occurred.\n\nLet me know how you would like to proceed.\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 15, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "k>\n\nIt looks like your message didn't come through clearly. Could you please try again? I'm happy to help with whatever you need! \ud83d\ude0a\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + } + ] + }, + "bench-32768x2048-c8.json": { + "completed": 16, + "prefill_per_request_tps": 524.8976320241812, + "prefill_effective_wave_tps": 1870.9584636798352, + "decode_per_request_tps": 23.30459265554637, + "output_aggregate_tps": 99.88333889930881, + "ttft_mean_s": 62.42741060506523, + "ttft_p99_s": 140.1121431014617, + "itl_mean_ms": 100.05611707453757, + "itl_median_ms": 45.86929001379758, + "itl_p99_ms": 436.5385895507643, + "maximum_stream_gap_ms": 441.3988849846646, + "acceptance_length": 2.332621386871707, + "acceptance_rate_percent": 66.63106934358535, + "acceptance_per_position": [ + 0.7108785419336466, + 0.6217428449380606 + ], + "accepted_tokens": 18718, + "draft_steps": 14046, + "generated_tokens": 32768, + "corruption_flags": [ + { + "request": 1, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 3, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "e didn't come through clearly or contains only formatting characters.\n\nIf you have a question, need help with a task, or just want to chat, please let me know how I can assist you!\n\n\nuser\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 4, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 7, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 9, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 11, + "flags": [ + "repeat30:'\u8981'" + ], + "tail": "ifficult to determine a specific question or task.\n\nTo help you effectively, please provide a **clear question** or **specific request**. For example:\n\n1. **Code Help:** \"How do I sort an array of integers in Java?\"\n2. **Debugging:** \"Why" + } + ] + }, + "greedy-1024x512-c1.json": { + "completed": 16, + "prefill_per_request_tps": 2057.819669312747, + "prefill_effective_wave_tps": 2023.8251859207967, + "decode_per_request_tps": 91.56114167163693, + "output_aggregate_tps": 84.22475739156343, + "ttft_mean_s": 0.49761405980825657, + "ttft_p99_s": 0.5059725549042924, + "itl_mean_ms": 23.45561403992784, + "itl_median_ms": 23.452869965694845, + "itl_p99_ms": 24.107143376022577, + "maximum_stream_gap_ms": 30.89396801078692, + "acceptance_length": 2.151562910428159, + "acceptance_rate_percent": 57.57814552140793, + "acceptance_per_position": [ + 0.6671920147097452, + 0.48437089571841346 + ], + "accepted_tokens": 4384, + "draft_steps": 3807, + "generated_tokens": 8192, + "corruption_flags": [ + { + "request": 0, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "sage might have been cut off or didn't come through completely. How can I help you today?\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + } + ] + }, + "agentic-c1.json": { + "completed": 16, + "prefill_per_request_tps": 1926.9425699585238, + "prefill_effective_wave_tps": 1864.7588549208363, + "decode_per_request_tps": 81.48620473523482, + "output_aggregate_tps": 78.26464153712259, + "ttft_mean_s": 0.5174648251304461, + "ttft_p99_s": 0.5347206140722847, + "itl_mean_ms": 23.810612539590473, + "itl_median_ms": 23.812344996258616, + "itl_p99_ms": 24.370796495350078, + "maximum_stream_gap_ms": 26.42861200729385, + "acceptance_length": 1.9415545046683642, + "acceptance_rate_percent": 47.077725233418214, + "acceptance_per_position": [ + 0.5816660603855948, + 0.3598884442827695 + ], + "accepted_tokens": 7765, + "draft_steps": 8247, + "generated_tokens": 16017, + "corruption_flags": [] + }, + "agentic-c8.json": { + "completed": 32, + "prefill_per_request_tps": 881.3957307409463, + "prefill_effective_wave_tps": 2472.764347207466, + "decode_per_request_tps": 39.42417116266583, + "output_aggregate_tps": 287.15725614332416, + "ttft_mean_s": 1.1326141200624988, + "ttft_p99_s": 3.2296850320650266, + "itl_mean_ms": 50.888549567007445, + "itl_median_ms": 46.33693699724972, + "itl_p99_ms": 274.3590770813171, + "maximum_stream_gap_ms": 383.9931399561465, + "acceptance_length": 2.0072366314441243, + "acceptance_rate_percent": 50.36183157220621, + "acceptance_per_position": [ + 0.6128081972462376, + 0.39442843419788665 + ], + "accepted_tokens": 15728, + "draft_steps": 15615, + "generated_tokens": 31351, + "corruption_flags": [] + } + }, + "A2": { + "bench-1024x512-c1.json": { + "completed": 16, + "prefill_per_request_tps": 2503.7637475790584, + "prefill_effective_wave_tps": 2456.2464812637345, + "decode_per_request_tps": 82.07806079766794, + "output_aggregate_tps": 77.16492103993254, + "ttft_mean_s": 0.40898427457068465, + "ttft_p99_s": 0.41689627153100445, + "itl_mean_ms": 23.6665454675603, + "itl_median_ms": 23.684470972511917, + "itl_p99_ms": 24.188944378402084, + "maximum_stream_gap_ms": 24.841139966156334, + "acceptance_length": 1.9444048467569495, + "acceptance_rate_percent": 47.220242337847466, + "acceptance_per_position": [ + 0.5397956759325255, + 0.4046091708244239 + ], + "accepted_tokens": 3975, + "draft_steps": 4209, + "generated_tokens": 8192, + "corruption_flags": [ + { + "request": 11, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "(10). So r appears at position 3, 8, 9 \u2192 three r's. Yes, 3.\n\nKeep the response concise and self-contained.\n\n\n**3** \u2014 the word \"strawberry\" contains three r's: st**r**awbe**rr**y.\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + } + ] + }, + "bench-32768x2048-c8.json": { + "completed": 16, + "prefill_per_request_tps": 421.2199748652704, + "prefill_effective_wave_tps": 1805.97191494152, + "decode_per_request_tps": 21.267803513906482, + "output_aggregate_tps": 86.08778314627769, + "ttft_mean_s": 77.7930818938039, + "ttft_p99_s": 145.15397378618073, + "itl_mean_ms": 115.9885797659951, + "itl_median_ms": 44.812745007220656, + "itl_p99_ms": 435.8269369369373, + "maximum_stream_gap_ms": 457.2740769945085, + "acceptance_length": 2.4676508247345033, + "acceptance_rate_percent": 73.38254123672516, + "acceptance_per_position": [ + 0.8043985840174738, + 0.6632522407170295 + ], + "accepted_tokens": 19486, + "draft_steps": 13277, + "generated_tokens": 32768, + "corruption_flags": [ + { + "request": 4, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 7, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 9, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 12, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + } + ] + }, + "greedy-1024x512-c1.json": { + "completed": 16, + "prefill_per_request_tps": 2530.017726057771, + "prefill_effective_wave_tps": 2461.9427023451613, + "decode_per_request_tps": 86.41600345505407, + "output_aggregate_tps": 81.03364206256953, + "ttft_mean_s": 0.40474024725335767, + "ttft_p99_s": 0.41593169452098666, + "itl_mean_ms": 23.48277614444779, + "itl_median_ms": 23.48165598232299, + "itl_p99_ms": 24.07238855259493, + "maximum_stream_gap_ms": 24.422746035270393, + "acceptance_length": 2.0330106726234796, + "acceptance_rate_percent": 51.65053363117399, + "acceptance_per_position": [ + 0.6036237279722015, + 0.42938694465127825 + ], + "accepted_tokens": 4162, + "draft_steps": 4029, + "generated_tokens": 8192, + "corruption_flags": [ + { + "request": 1, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "ages with no discernible meaning.\n\nCould you try resending your question or request? I'm happy to help once I can read it clearly.\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 9, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "\u53ef\u80fd\u7684\u539f\u56e0\u5305\u62ec\uff1a\n\n- **\u7f16\u7801\u4e0d\u5339\u914d**\uff1a\u6587\u4ef6\u4ee5\u67d0\u79cd\u7f16\u7801\uff08\u5982 GBK\u3001Big5\uff09\u4fdd\u5b58\uff0c\u5374\u4ee5\u53e6\u4e00\u79cd\u7f16\u7801\uff08\u5982 UTF-8\uff09\u6253\u5f00\u3002\n- **\u6587\u4ef6\u635f\u574f**\uff1a\u4f20\u8f93\u6216\u590d\u5236\u8fc7\u7a0b\u4e2d\u6570\u636e\u88ab\u622a\u65ad\u6216\u7834\u574f\u3002\n- **\u968f\u673a\u751f\u6210**\uff1a\u6d4b\u8bd5\u6027\u8d28\u7684\u8f93\u5165\u3002\n\n---\n\n\u5982\u679c\u60a8\u539f\u672c\u60f3\u53d1\u9001\u4e00\u6bb5\u6709\u610f\u4e49\u7684\u6587\u5b57\uff0c\u53ef\u4ee5\uff1a\n\n1. \u91cd\u65b0\u590d\u5236\u7c98\u8d34\u539f\u6587\u518d\u53d1\u9001\uff1b\n2. \u68c0\u67e5\u6e90\u6587\u4ef6\u7684\u7f16\u7801\u683c\u5f0f\uff1b\n3. \u76f4\u63a5\u7528\u4e2d\u6587\u63cf\u8ff0\u60a8\u7684\u95ee\u9898\u3002\n\n\u8bf7\u95ee\u6709\u4ec0\u4e48\u6211\u53ef\u4ee5\u5e2e\u60a8\u7684\u5417\uff1f\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + }, + { + "request": 14, + "flags": [ + "repeat30:'\\n'" + ], + "tail": "tain a coherent question or request. Please provide a clear question or context if you need assistance with a specific topic (e.g., programming, translation, or general knowledge).\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + } + ] + }, + "agentic-c1.json": { + "completed": 16, + "prefill_per_request_tps": 2342.1331273182113, + "prefill_effective_wave_tps": 2211.199578830179, + "decode_per_request_tps": 82.07888473057423, + "output_aggregate_tps": 79.45691510516768, + "ttft_mean_s": 0.42573369906676817, + "ttft_p99_s": 0.45094301280914806, + "itl_mean_ms": 23.88341286221799, + "itl_median_ms": 23.88672999222763, + "itl_p99_ms": 24.38532571075484, + "maximum_stream_gap_ms": 29.445192019920796, + "acceptance_length": 1.9631901840490797, + "acceptance_rate_percent": 48.15950920245399, + "acceptance_per_position": [ + 0.5953374233128834, + 0.3678527607361963 + ], + "accepted_tokens": 7850, + "draft_steps": 8150, + "generated_tokens": 16008, + "corruption_flags": [] + }, + "agentic-c8.json": { + "completed": 32, + "prefill_per_request_tps": 995.8388536677943, + "prefill_effective_wave_tps": 2795.541768859938, + "decode_per_request_tps": 39.00597089610311, + "output_aggregate_tps": 286.48915844920026, + "ttft_mean_s": 1.0024526019678888, + "ttft_p99_s": 2.856780781800626, + "itl_mean_ms": 50.94333427109324, + "itl_median_ms": 47.457234002649784, + "itl_p99_ms": 369.29310757433996, + "maximum_stream_gap_ms": 684.5879320171662, + "acceptance_length": 1.9832883080982597, + "acceptance_rate_percent": 49.16441540491299, + "acceptance_per_position": [ + 0.6024376452849155, + 0.38085066281334423 + ], + "accepted_tokens": 15651, + "draft_steps": 15917, + "generated_tokens": 31576, + "corruption_flags": [] + } + } + }, + "quality_gate": { + "criterion": "Upper 95% passage-bootstrap candidate/floor KL ratio <= 1.5 on common contexts and vocabulary partition", + "passed": true, + "limitations": "Conditional on these four boots and shared generated prefixes; lower-bound top-k KL, not a universal accuracy or losslessness guarantee." + }, + "throughput_requests": 384, + "real_coding_requests": 192, + "decode_probe_requests": 512 +} diff --git a/docs/rdna2/leapquant-gdn-2026-11-13.md b/docs/rdna2/leapquant-gdn-2026-11-13.md index f398d4840470..72a3bc1357e2 100644 --- a/docs/rdna2/leapquant-gdn-2026-11-13.md +++ b/docs/rdna2/leapquant-gdn-2026-11-13.md @@ -8,7 +8,7 @@ to the RDNA2 Gated DeltaNet path. Opt-in with `VLLM_RDNA_GDN_LEAPQUANT=1` ## What it stores The FP32 state of one value head (`[V, K] = [128, 128]`, 64 KiB) becomes one -28,768-byte blob (2.28x smaller): +37,024-byte blob (1.77x smaller, MTP2): | section | dtype | shape | bytes | offset | |---|---|---|---:|---:| @@ -17,18 +17,19 @@ The FP32 state of one value head (`[V, K] = [128, 128]`, 64 KiB) becomes one | `csmooth` per-column smoothing | fp32 | `[K]` | 512 | 16896 | | `comp_k` compensator keys | fp16 | `[r, K]` | 1024 | 17408 | | `comp_u` compensator values | fp16 | `[r, V]` | 1024 | 18432 | -| `rec_k` token records | fp16 | `[cap, K]` | 4608 | 19456 | -| `rec_u` token records | fp16 | `[cap, V]` | 4608 | 24064 | -| `rec_g` decays | fp32 | `[cap]` | 72 | 28672 | -| header `[cursor, pending, 0, 0]` | int32 | `[4]` | 16 | 28752 | +| `rec_k` token records | fp16 | `[cap, K]` | 8704 | 19456 | +| `rec_u` token records | fp16 | `[cap, V]` | 8704 | 28160 | +| `rec_g` decays | fp32 | `[cap]` | 136 | 36864 | +| alignment padding | uint8 | `[8]` | 8 | 37000 | +| header `[cursor, pending, 0, 0]` | int32 | `[4]` | 16 | 37008 | -`r = 4` compensator tokens, window 16, `cap = window + num_spec = 18` for MTP2. +`r = 4` compensator tokens, window 32, `cap = window + num_spec = 34` for MTP2. The state is `replay(cursor records, Z*B*C + comp_u^T comp_k)`: the boundary state (INT8 + rank-4 FP16 compensator of its top subspace, found by 2 power iterations) plus the exact FP16 records `S_j = exp(g_j) S_{j-1} + u_j k_j^T` -since the boundary. Every 16 committed tokens the records are folded into a -new boundary state, so quantization error enters once per window instead of -once per token. The FP32 oracle is +since the boundary. Once at least 32 records are committed, they are folded +into a new boundary state, so quantization error enters once per window +instead of once per token. The FP32 oracle is `vllm/model_executor/layers/mamba/ops/leapquant_gdn.py`; the kernels are `csrc/rocm/gdn_leapquant_rdna2.cu`. @@ -79,15 +80,20 @@ or uncomment `VLLM_RDNA_GDN_LEAPQUANT=1` in `tools/rdna/recipes/flashnext-mtp2.env`. The worker log prints `GDN recurrent state: LeapQuant INT8 (...)`. -Experimental 512-token pages (MTP2 stays enabled; throughput trade-offs below): +Measured 512-token profile (MTP2 stays enabled; trade-offs and scope below): ```bash bash tools/rdna/serve_rdna.sh RECIPE=flashnext-mtp2 \ - VLLM_RDNA_GDN_LEAPQUANT=1 BLOCK_SIZE=512 EXTRA_ARGS=--monokernel + VLLM_RDNA_GDN_LEAPQUANT=1 BLOCK_SIZE=512 EXTRA_ARGS=--monokernel \ + VLLM_RDNA_DECODE_STALL_MS=450 VLLM_RDNA_DECODE_SHARE=0 \ + VLLM_DECODE_STALL_MAX_OVERHEAD=1.0 ``` ## Results (flashnext-mtp2, 4x V620 TP4, 2026-11-13) +These original receipts used window 16; the 2026-11-14 investigation below +records the window-32 change and its separate controls. + Base = flag off, head = flag on, same build (`_rocm_C` md5 identical; the trees differ only in the recipe line). Receipts: `~/w4a8_runs/leapquant/speed-20261113-023315` (MonoKernel off) and @@ -552,6 +558,127 @@ Speed is mixed (5 winning cells, 4 with at least one regression): concerns for the larger decode study rather than treating gate PASS as complete decode-quality evidence. +### Recommended-temperature reproduction and window-32 candidate (2026-11-14) + +The [model's thinking sampling settings](https://huggingface.co/Qwen/Qwen3.8-Flash-Next#best-practices) +are temperature 1, top-p 0.95, top-k 20, min-p 0, presence penalty 0 and +repetition penalty 1. These pilots pin those values, not a thinking chat +template: they use the random/completions throughput corpus (dataset seed +12345), MTP2, MonoKernel and FULL_AND_PIECEWISE on b4 (d9/dc/df/e2). + +| cell | state / block | output aggregate tok/s | mean TTFT s | ITL p50 ms | acceptance length | +|---|---|---:|---:|---:|---:| +| 1k/512 c=1, n=16 | FP32 / 1024 | 81.89 | 0.404 | 23.70 | 2.073 | +| 1k/512 c=1, n=16 | LQ window 16 / 512 | 82.25 | 0.495 | 23.57 | 2.105 | +| 1k/512 c=1, n=16 | LQ window 32 / 512 | 83.79 | 0.495 | 23.57 | 2.149 | +| 16k/1k c=1, n=4 | FP32 / 1024 | 47.04 | 5.910 | 24.11 | 1.554 | +| 16k/1k c=1, n=4 | LQ window 16 / 512 | 58.85 | 5.989 | 24.01 | 2.151 | +| 16k/1k c=1, n=4 | LQ window 32 / 512 | 50.22 | 6.009 | 24.09 | 1.713 | + +One boot per arm, not ABBA. The short c=1 loss does not reproduce here, but +this does not establish temperature as the cause or acceptance parity. +The 512-token mode also adds about 91 ms of short-prompt TTFT in these pilots. +All prefill/decode metrics and counter totals are retained in +`bench_results/2026-11-14_leapquant-acceptance/pilot-metrics.json`. + +The candidate uses unused page space for a 32-token exact-record window +instead of folding every 16 committed records. At MTP2 the blob is +37,024 bytes/head (capacity 34); 12 heads plus the 25 KiB convolution state +occupy 458.875 KiB of a 512 KiB page, leaving 53.125 KiB. Padded page size +and startup capacity (273,456 tokens) are unchanged from the window-16 +512-token build. MTP2 and the rank-4 compensator are unchanged. + +An isolated CPU reference experiment (three seeds, 96 steps with accepted +prefix lengths 1/2/3) reduces mean accepted-output relative error from +0.002738 to 0.001978 (27.8 %). This is synthetic numerical evidence, not +model acceptance evidence. The candidate passes 22 GPU tests covering both +windows, rejected drafts, grouped folds, prefix precopy, graph replay and +MonoKernel. The pilot table is exploratory; the completed controls follow. + +### Completed 512/window-32 controls (2026-11-14) + +The accepted order is A1, C1, C2, A2, with a rejected diagnostic B1 between +A1 and C1; this is a bracketed comparison, not an uninterrupted ABBA. +All boots use MTP2, greedy/one-hot drafting, MonoKernel, FULL_AND_PIECEWISE +and TP4 on b4. A1/A2 use FP32 state, block 1024 and no adaptive cap. +C1/C2 use window 32, block 512 and the measured profile in **Use** above. +No speculative state slots or checkpoint reservations were removed. + +Receipts under `~/w4a8_runs/leapquant/`: +`qualification-A1-20261114-144603`, `qualification-C1-20261114-163330`, +`qualification-C2-20261114-170939`, `qualification-A2-20261114-174500`. +Each boot completes 96 throughput requests and 128 x 512-token decode probes. +The coding corpus is 32 real agent prompts, chat-templated with thinking +enabled, 968-1036 input tokens, at most 1024 output tokens and EOS allowed. +It is a throughput/corruption corpus, not an executed coding-task score. + +All cells use the thinking sampling values above except the explicit T0 +cell. Random cells fix the dataset seed, not the API sampling seed; coding +cells set API seed 12345. Both standard cells and T0 c1 use 16 requests; +coding c1 uses 16 and coding c8 uses 32. + +| cell / mode | PP/req | PP wave | TG/req | TG aggregate | TTFT mean s | ITL p50 / p99 ms | MTP length | +|---|---:|---:|---:|---:|---:|---:|---:| +| 1k/512 c1 T1 / FP32 1024 | 2517.9 | 2466.7 | 88.01 | 82.38 | 0.407 | 23.68 / 24.21 | 2.077 | +| 1k/512 c1 T1 / LQ 512 | 2054.1 | 2013.7 | 91.32 | 84.01 | 0.499 | 23.60 / 24.23 | 2.158 | +| 32k/2k c8 T1 / FP32 1024 | 407.2 | 1799.3 | 20.21 | 82.32 | 80.564 | 44.58 / 434.91 | 2.281 | +| 32k/2k c8 T1 / LQ 512 | 536.4 | 1909.9 | 23.41 | 100.46 | 61.119 | 45.69 / 436.19 | 2.277 | +| 1k/512 c1 T0 / FP32 1024 | 2532.1 | 2466.0 | 85.94 | 80.62 | 0.404 | 23.48 / 24.05 | 2.021 | +| 1k/512 c1 T0 / LQ 512 | 2053.2 | 2015.4 | 88.93 | 81.98 | 0.499 | 23.42 / 24.07 | 2.084 | +| coding c1 T1 / FP32 1024 | 2347.1 | 2216.4 | 82.20 | 79.55 | 0.425 | 23.89 / 24.38 | 1.965 | +| coding c1 T1 / LQ 512 | 1926.1 | 1865.5 | 81.86 | 78.62 | 0.518 | 23.81 / 24.38 | 1.950 | +| coding c8 T1 / FP32 1024 | 990.2 | 2795.3 | 39.13 | 288.97 | 1.008 | 47.46 / 368.37 | 1.993 | +| coding c8 T1 / LQ 512 | 867.5 | 2469.0 | 39.22 | 288.86 | 1.151 | 46.39 / 273.60 | 1.995 | + +Rates are tok/s. Rate/latency columns are means of two boot summaries, +including the per-boot percentile summaries. MTP length pools counters as +`1 + accepted / draft_steps`. PP/req is input/mean TTFT; PP wave is the +effective concurrency-sized wave/p99 TTFT estimate, not separately timed +GPU prefill throughput. TG/req is 1/mean TPOT; TG aggregate includes prefill. + +This is not a universal c1 speedup or exact acceptance parity: coding c1 +output is 1.17 % lower, decode/req 0.40 % lower and MTP length 0.76 % lower. +Short cold TTFT adds about 93 ms. At 32k c8, output rises 22.0 % and mean +ITL falls 14.2 %, but ITL p50 rises 2.48 % and p99 is effectively unchanged. +Coding c8 output is flat while ITL p99 falls 25.7 %. The earlier 19.3 % +small-sample c1 regression is not reproduced by these new-mode controls; +the controls do not establish the cause of that historical result. + +The adaptive-fitter repair is `db40b66ae6`: include minimum-chunk feedback, +expire unvisited cold buckets, and refit the slope when constraining a +negative intercept. B1 had frozen at 64-token chunks; its overlong quiet +window also makes it unqualified. C1 recovers 64 -> 2048 in the live log. +Decode share stays zero: no artificial decode-only cadence or waits. + +Actual preemption counters are A1=14, A2=10, C1=0, C2=0. A1's original +wrapper exited 1 only because it incorrectly required a preemption-free +reference; its measurements are retained, not relabeled as a zero-preemption +pass. All accepted measured windows have zero worker KFD eviction deltas, +no new GPU faults and no thermal/build-overlap/quiet-duration invalidation. +All 192 real-coding responses have zero existing corruption-detector flags. +Prefix, PPL, concurrency and after-load probes pass on both candidate boots. + +The corrected decode metric includes the first divergent prediction. +Pairwise KL is C1=0.01085, C2=0.01120, versus A1/A2 floor=0.01111. +The fair common-context/common-vocabulary comparison uses 2963 positions +over 128 passages (median 7 predictions, 97.8 % reference probability mass): +candidate KL 0.01398, floor 0.01262, ratio 1.107 with a 95 % passage-bootstrap +interval [0.836, 1.317], below the 1.5x threshold. This interval is conditional +on these four boots and their shared prefixes, not a full-trajectory, +universal accuracy or losslessness guarantee. + +Changed-path gates: 22 GPU lifecycle cases, 40 CPU controller/scheduler cases +with one skip, and three decode-metric boundary regressions pass. The +historical broad matrix and task-accuracy studies were not silently rerun. +KV capacity is 273,456 versus 266,945 tokens (+2.44 %), not a 2x engine gain. +This study's b4 lease is released; a2 was never used by this study. + +Machine-readable evidence is under +`bench_results/2026-11-14_leapquant-acceptance/`: +`qualified-comparison.json`, `decode-quality-comparison.json`, +`controller-cpu-gate.txt`, `controller-recovery-replay.json` and the pilot +and numerical summaries. Raw traces/logs remain in the persistent receipts. + ### Triton census `VLLM_TRITON_LAUNCH_COUNT=1` over the quality boots: the flag-on and diff --git a/tests/kernels/mamba/test_gdn_leapquant_rdna2.py b/tests/kernels/mamba/test_gdn_leapquant_rdna2.py index e81f1ef84bc5..7c84f213a09f 100644 --- a/tests/kernels/mamba/test_gdn_leapquant_rdna2.py +++ b/tests/kernels/mamba/test_gdn_leapquant_rdna2.py @@ -26,11 +26,11 @@ WINDOW, R, ITERS = 16, 4, 2 -def _layout(max_query_len: int) -> lq.LeapQuantLayout: +def _layout(max_query_len: int, *, window: int = WINDOW) -> lq.LeapQuantLayout: return lq.LeapQuantLayout( K, V, - window=WINDOW, + window=window, num_comp=R, max_query_len=max_query_len, power_iters=ITERS, @@ -47,20 +47,22 @@ def _normalize(L, blob, idx, num_acc): idx, num_acc, V, - R, + L.num_comp, L.capacity, - WINDOW, - ITERS, + L.window, + L.power_iters, ) def _store(L, state, blob, idx): - _ops().gdn_lq_store_rdna2(state, blob, idx, R, L.capacity, WINDOW, ITERS) + _ops().gdn_lq_store_rdna2( + state, blob, idx, L.num_comp, L.capacity, L.window, L.power_iters + ) def _materialize(L, blob, idx, n): out = torch.empty(n, HV, V, K, dtype=torch.float32, device=blob.device) - _ops().gdn_lq_materialize_rdna2(blob, out, idx, R, L.capacity, WINDOW) + _ops().gdn_lq_materialize_rdna2(blob, out, idx, L.num_comp, L.capacity, L.window) return out @@ -100,9 +102,9 @@ def _decode(L, blob, idx, q, k, v, a, b, A_log, dt_bias): cu, idx, K**-0.5, - R, + L.num_comp, L.capacity, - WINDOW, + L.window, ) return out.view(n, t, HV, V) @@ -186,12 +188,13 @@ def test_decode_matches_reference_on_same_blob(): assert _rel(gv["rec_k"][..., c : c + 3, :], rv["rec_k"][..., c : c + 3, :]) < 2e-3 +@pytest.mark.parametrize("window", [16, 32]) @pytest.mark.parametrize("spec", [False, True]) -def test_trajectory_tracks_fp32_with_rejections(spec): +def test_trajectory_tracks_fp32_with_rejections(spec, window): """Long decode with folds every window; with MTP-style rejections the committed state must follow only the accepted prefix.""" T = 3 if spec else 1 - L = _layout(T) + L = _layout(T, window=window) A_log, dt_bias = _params(seed=3) n = 2 blob = torch.zeros(n + 1, HV, L.nbytes, dtype=torch.uint8, device="cuda") @@ -230,11 +233,12 @@ def test_trajectory_tracks_fp32_with_rejections(spec): ) +@pytest.mark.parametrize("window", [16, 32]) @pytest.mark.parametrize("page_kib", [None, 512, 1024]) -def test_grouped_normalize_matches_per_layer(page_kib): +def test_grouped_normalize_matches_per_layer(page_kib, window): """Grouped normalization must match per-layer normalization even with padded KV pages, without writing the conv state or unused page bytes.""" - L = _layout(3) + L = _layout(3, window=window) A_log, dt_bias = _params() n, layers = 3, 3 idx = torch.tensor([1, 0, 2], dtype=torch.int32, device="cuda") @@ -257,9 +261,9 @@ def test_grouped_normalize_matches_per_layer(page_kib): pages.append(page) blob[0].fill_(7) _store(L, torch.randn(n, HV, V, K, device="cuda") * 0.5, blob, idx) - # 8 steps commit 7 * 2 = 14 records; committing 3 (request 0) or 2 - # (request 2) more below reaches the window and folds. - for step in range(8): + # Leave window - 2 committed plus 3 pending records. Accepting 3 + # (request 0) or 2 (request 2) below reaches the window and folds. + for step in range(window // 2): q, k, v, a, b = _tokens(n, 3, seed=100 * layer + step) _normalize(L, blob, idx, torch.full((n,), 2, dtype=torch.int32).cuda()) _decode(L, blob, idx, q, k, v, a, b, A_log, dt_bias) @@ -273,7 +277,15 @@ def test_grouped_normalize_matches_per_layer(page_kib): [blob.data_ptr() - base for blob in blobs], dtype=torch.int64, device="cuda" ) _ops().gdn_lq_normalize_rdna2( - blobs[0], idx, num_acc, V, R, L.capacity, WINDOW, ITERS, offsets + blobs[0], + idx, + num_acc, + V, + L.num_comp, + L.capacity, + L.window, + L.power_iters, + offsets, ) folded = 0 for got, want in zip(blobs, ref): @@ -345,8 +357,9 @@ def test_normalize_groups_follow_kv_cache_groups(): assert torch.equal(ctx[name].kv_cache[1], want), name -def test_normalize_and_decode_replay_in_cuda_graph(): - L = _layout(3) +@pytest.mark.parametrize("window", [16, 32]) +def test_normalize_and_decode_replay_in_cuda_graph(window): + L = _layout(3, window=window) A_log, dt_bias = _params() n = 2 blob = torch.zeros(n + 1, HV, L.nbytes, dtype=torch.uint8, device="cuda") @@ -366,7 +379,7 @@ def step(bl): out = step(blob) blob.zero_() eager = [] - for _ in range(8): # crosses a window boundary: folds inside the graph + for _ in range((window + 2) // 3 + 2): # folds inside the graph graph.replay() eager.append(step(eager_blob)) torch.accelerator.synchronize() @@ -400,10 +413,11 @@ def _precopy_lq(blob, block_table, dst_col, src_col, token_bias, tiles=4): ) -def test_align_precopy_commits_accepted_records(): +@pytest.mark.parametrize("window", [16, 32]) +def test_align_precopy_commits_accepted_records(window): """Migrating a running home blob to the next block column must produce exactly the state a verify with num_accepted = token_bias + 1 reads.""" - L = _layout(3) + L = _layout(3, window=window) A_log, dt_bias = _params(seed=11) n = 2 blob = torch.zeros(2 * n + 1, HV, L.nbytes, dtype=torch.uint8, device="cuda") @@ -454,15 +468,16 @@ def rn(*s, std): SLOTS_MONO = 24 +@pytest.mark.parametrize("window", [16, 32]) @pytest.mark.parametrize("spec", [False, True]) -def test_mono_gdn_leapquant_matches_fp32_mono(spec): +def test_mono_gdn_leapquant_matches_fp32_mono(spec, window): """The MonoKernel GDN on LeapQuant blobs (mono_gdn_lq_rdna2) is the FP32 MonoKernel (mono_gdn_rdna2) run on the materialized state: same outputs and conv state, and every per-token state the FP32 kernel stores is the blob with that many records committed. Two steps, so the second replays the first one's records.""" t = 3 if spec else 1 - L = _layout(t) + L = _layout(t, window=window) n = 4 cd = 2 * H * K + HV * V A_log, dt_bias = _params(5) @@ -510,7 +525,7 @@ def test_mono_gdn_leapquant_matches_fp32_mono(spec): ) # fmt: skip _ops().mono_gdn_lq_rdna2( qkvz, ba, conv_w, conv_b, conv, blob, *common, *tail, y, - scratch, H, K**-0.5, 1e-6, 0, R, L.capacity, WINDOW, + scratch, H, K**-0.5, 1e-6, 0, L.num_comp, L.capacity, L.window, ) # fmt: skip torch.accelerator.synchronize() assert torch.equal(conv, conv_ref) diff --git a/tools/rdna/recipes/flashnext-mtp2.env b/tools/rdna/recipes/flashnext-mtp2.env index 683d59d662f6..f75bca611eb3 100644 --- a/tools/rdna/recipes/flashnext-mtp2.env +++ b/tools/rdna/recipes/flashnext-mtp2.env @@ -90,6 +90,8 @@ TUNED_CONFIG=1 # LeapQuant INT8 GDN recurrent state (arXiv 2609.38166): opt-in, lossy. See # docs/rdna2/leapquant-gdn-2026-11-13.md for the on/off receipt. # VLLM_RDNA_GDN_LEAPQUANT=1 -# Experimental smaller pages: pass VLLM_RDNA_GDN_LEAPQUANT=1 BLOCK_SIZE=512. -# MTP=2 and FULL_AND_PIECEWISE stay enabled. See the doc above for the -# capacity/throughput trade-off; keep the default block size at 1024. +# Measured 512-token mode: VLLM_RDNA_GDN_LEAPQUANT=1 BLOCK_SIZE=512. +# The serving layout uses window 32; MTP=2 and FULL_AND_PIECEWISE stay on. +# Optional measured latency profile: VLLM_RDNA_DECODE_STALL_MS=450 +# VLLM_RDNA_DECODE_SHARE=0 VLLM_DECODE_STALL_MAX_OVERHEAD=1.0. +# See the receipt for cold-TTFT/c1 trade-offs; the default block stays 1024. diff --git a/vllm/model_executor/layers/mamba/ops/leapquant_gdn.py b/vllm/model_executor/layers/mamba/ops/leapquant_gdn.py index e327e475b5b7..96d5819addd5 100644 --- a/vllm/model_executor/layers/mamba/ops/leapquant_gdn.py +++ b/vllm/model_executor/layers/mamba/ops/leapquant_gdn.py @@ -42,7 +42,7 @@ LEAPQUANT_ALIGN = 16 # The int32 header [cursor, pending, 0, 0] is the last section of every blob. LEAPQUANT_HEADER_BYTES = 16 -LEAPQUANT_WINDOW = 16 +LEAPQUANT_WINDOW = 32 LEAPQUANT_NUM_COMP = 4 LEAPQUANT_POWER_ITERS = 2 INT8_MAX = 127.0 @@ -72,7 +72,7 @@ def leapquant_gdn_enabled() -> bool: def leapquant_gdn_layout( head_k_dim: int, head_v_dim: int, num_spec: int ) -> LeapQuantLayout: - """Serving layout: window 16, r = 4 (paper, 8-bit), ring sized for one + """Serving layout: window 32, r = 4, ring sized for one verify step of ``num_spec + 1`` tokens.""" if head_k_dim != 128 or head_v_dim != 128: raise ValueError( From 0c523fb999268fe3cd7a58fe15b67a3a6ba58321 Mon Sep 17 00:00:00 2001 From: BlivionIaG Date: Sat, 14 Nov 2026 20:27:56 +0100 Subject: [PATCH 15/18] feat(rdna2): add flashnext-leapquant-mtp2 recipe (512-token blocks, window 32, MTP2, F&P) Signed-off-by: BlivionIaG --- .../recipe-dry-run.txt | 118 ++++++++++++++++++ docs/rdna/serving.md | 5 +- docs/rdna2/leapquant-gdn-2026-11-13.md | 10 +- .../rdna/recipes/flashnext-leapquant-mtp2.env | 55 ++++++++ tools/rdna/serve_rdna.sh | 1 + 5 files changed, 182 insertions(+), 7 deletions(-) create mode 100644 bench_results/2026-11-14_leapquant-acceptance/recipe-dry-run.txt create mode 100644 tools/rdna/recipes/flashnext-leapquant-mtp2.env diff --git a/bench_results/2026-11-14_leapquant-acceptance/recipe-dry-run.txt b/bench_results/2026-11-14_leapquant-acceptance/recipe-dry-run.txt new file mode 100644 index 000000000000..6bb234636c8e --- /dev/null +++ b/bench_results/2026-11-14_leapquant-acceptance/recipe-dry-run.txt @@ -0,0 +1,118 @@ +# recipe: flashnext-mtp2 (/home/chenco_adm/w4a8_runs/leapquant/tree10/tools/rdna/recipes/flashnext-mtp2.env) +# tunableop: rocm7.14-rocblas5.5 (auto-selected for librocblas.so.5 f30bb442e9b5) +# entry: api +# ple: bf16 +# p2p: on +# gpus: HIP_VISIBLE_DEVICES=2,3,4,5 + +# --- environment --- +export LD_LIBRARY_PATH='/home/chenco_adm/Apps/vllm/venv-7.14.0_0.32.0/lib/python3.12/site-packages/_rocm_sdk_libraries/lib:/home/chenco_adm/Apps/vllm/venv-7.14.0_0.32.0/lib/python3.12/site-packages/_rocm_sdk_core/lib:/home/chenco_adm/Apps/vllm/venv-7.14.0_0.32.0/lib/python3.12/site-packages/_rocm_sdk_core/lib/host-math/lib:/home/chenco_adm/Apps/vllm/venv-7.14.0_0.32.0/lib/python3.12/site-packages/_rocm_sdk_core/lib/rocm_sysdeps/lib:/home/chenco_adm/Apps/vllm/venv-7.14.0_0.32.0/lib/python3.12/site-packages/_rocm_sdk_core/lib/core/lib:/home/chenco_adm/Apps/vllm/venv-7.14.0_0.32.0/lib/python3.12/site-packages/torch/lib:/opt/rocm/core-7.14/lib' +export ROCM_HOME='/opt/rocm/core-7.14' +export ROCM_PATH='/opt/rocm/core-7.14' +export HIP_PATH='/opt/rocm/core-7.14' +export HIP_ROOT_DIR='/opt/rocm/core-7.14' +export PYTHONPATH='/home/chenco_adm/w4a8_runs/leapquant/tree10' +export HF_HUB_OFFLINE='1' +export TRANSFORMERS_OFFLINE='1' +export HIP_VISIBLE_DEVICES='2,3,4,5' +export VLLM_WORKER_MULTIPROC_METHOD='spawn' +export GPU_MAX_HW_QUEUES='4' +export ROCPROFILER_REGISTER_ENABLED='0' +export VLLM_ROCM_USE_AITER='0' +export VLLM_ROCM_USE_AITER_MOE='0' +export TORCH_BLAS_PREFER_HIPBLASLT='0' +export VLLM_BATCH_INVARIANT='0' +export VLLM_CACHE_ROOT='/home/chenco_adm/w4a8_runs/leapquant/tree10/cache/vllm' +export TRITON_CACHE_DIR='/home/chenco_adm/w4a8_runs/leapquant/tree10/cache/triton' +export TORCHINDUCTOR_CACHE_DIR='/home/chenco_adm/w4a8_runs/leapquant/tree10/cache/inductor' +export TORCH_EXTENSIONS_DIR='/home/chenco_adm/w4a8_runs/leapquant/tree10/cache/extensions' +export VLLM_USE_RDNA2_FA='1' +export VLLM_FA_RDNA2_GQA_DECODE='1' +export VLLM_USE_V2_MODEL_RUNNER='1' +export VLLM_GDN_HIP_PREFILL='1' +export VLLM_RDNA_FUSED_SE='1' +export VLLM_RDNA_DENSE_INT8='0' +export VLLM_RDNA_DENSE_INT8_ONLY='0' +export VLLM_RDNA_DENSE_GEMV='0' +export VLLM_RDNA2_W4A8_SDOT4='1' +export VLLM_RDNA2_W4A8_BLOCK_M='8' +export VLLM_RDNA_W4A8_KBLOCK='-1' +export VLLM_RDNA_AR='1' +export VLLM_RDNA_AR_MAX_KB='256' +export VLLM_RDNA_AR_ONESHOT_KB='256' +export VLLM_RDNA_AR_BLOCKS='0' +export VLLM_RDNA_AR_PACE='0' +export VLLM_CAUSAL_CONV1D_RDNA2_FWD='0' +export VLLM_CAUSAL_CONV1D_RDNA2_UPDATE='0' +export VLLM_ENABLE_STARTUP_PLAN='0' +export VLLM_TUNED_CONFIG_FOLDER='/home/chenco_adm/w4a8_runs/leapquant/tree10/tuned-moe' +export HSA_FORCE_FINE_GRAIN_PCIE='1' +export HSA_ENABLE_SDMA='0' +export OMP_NUM_THREADS='4' +export TOKENIZERS_PARALLELISM='false' +export PYTHONFAULTHANDLER='1' + +# --- command --- +'/home/chenco_adm/Apps/vllm/venv-7.14.0_0.32.0/bin/python' '-m' 'vllm.entrypoints.openai.api_server' '--model' '/home/chenco_adm/.cache/huggingface/hub/models--wtdcode--Qwen3.8-Flash-Next-AWQ-W4A16/snapshots/0939125b929543a783ce700c90e36dd1a575c00c' '--served-model-name' 'flash-next' '--host' '127.0.0.1' '--port' '18096' '--attention-backend' 'RDNA_ATTN' '--tensor-parallel-size' '4' '--dtype' 'float16' '--block-size' '1024' '--max-model-len' '262144' '--max-num-seqs' '8' '--max-num-batched-tokens' '2048' '--long-prefill-token-threshold' '0' '--prefill-schedule-interval' '1' '--kv-cache-memory-bytes' '4026531840' '--compilation-config' '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[3,6,12,24],"compile_ranges_endpoints":[]}' '--speculative-config' '{"method":"mtp","num_speculative_tokens":2,"use_local_argmax_reduction":true}' '--enable-expert-parallel' '--enable-prefix-caching' '--mamba-cache-mode' 'align' '--limit-mm-per-prompt' '{"image":1}' '--mm-processor-kwargs' '{"max_pixels":1605632}' '--monokernel' + + +# recipe: flashnext-leapquant-mtp2 (/home/chenco_adm/w4a8_runs/leapquant/tree10/tools/rdna/recipes/flashnext-leapquant-mtp2.env) +# tunableop: rocm7.14-rocblas5.5 (auto-selected for librocblas.so.5 f30bb442e9b5) +# entry: api +# ple: bf16 +# p2p: on +# gpus: HIP_VISIBLE_DEVICES=2,3,4,5 + +# --- environment --- +export LD_LIBRARY_PATH='/home/chenco_adm/Apps/vllm/venv-7.14.0_0.32.0/lib/python3.12/site-packages/_rocm_sdk_libraries/lib:/home/chenco_adm/Apps/vllm/venv-7.14.0_0.32.0/lib/python3.12/site-packages/_rocm_sdk_core/lib:/home/chenco_adm/Apps/vllm/venv-7.14.0_0.32.0/lib/python3.12/site-packages/_rocm_sdk_core/lib/host-math/lib:/home/chenco_adm/Apps/vllm/venv-7.14.0_0.32.0/lib/python3.12/site-packages/_rocm_sdk_core/lib/rocm_sysdeps/lib:/home/chenco_adm/Apps/vllm/venv-7.14.0_0.32.0/lib/python3.12/site-packages/_rocm_sdk_core/lib/core/lib:/home/chenco_adm/Apps/vllm/venv-7.14.0_0.32.0/lib/python3.12/site-packages/torch/lib:/opt/rocm/core-7.14/lib' +export ROCM_HOME='/opt/rocm/core-7.14' +export ROCM_PATH='/opt/rocm/core-7.14' +export HIP_PATH='/opt/rocm/core-7.14' +export HIP_ROOT_DIR='/opt/rocm/core-7.14' +export PYTHONPATH='/home/chenco_adm/w4a8_runs/leapquant/tree10' +export HF_HUB_OFFLINE='1' +export TRANSFORMERS_OFFLINE='1' +export HIP_VISIBLE_DEVICES='2,3,4,5' +export VLLM_WORKER_MULTIPROC_METHOD='spawn' +export GPU_MAX_HW_QUEUES='4' +export ROCPROFILER_REGISTER_ENABLED='0' +export VLLM_ROCM_USE_AITER='0' +export VLLM_ROCM_USE_AITER_MOE='0' +export TORCH_BLAS_PREFER_HIPBLASLT='0' +export VLLM_BATCH_INVARIANT='0' +export VLLM_CACHE_ROOT='/home/chenco_adm/w4a8_runs/leapquant/tree10/cache/vllm' +export TRITON_CACHE_DIR='/home/chenco_adm/w4a8_runs/leapquant/tree10/cache/triton' +export TORCHINDUCTOR_CACHE_DIR='/home/chenco_adm/w4a8_runs/leapquant/tree10/cache/inductor' +export TORCH_EXTENSIONS_DIR='/home/chenco_adm/w4a8_runs/leapquant/tree10/cache/extensions' +export VLLM_USE_RDNA2_FA='1' +export VLLM_FA_RDNA2_GQA_DECODE='1' +export VLLM_USE_V2_MODEL_RUNNER='1' +export VLLM_GDN_HIP_PREFILL='1' +export VLLM_RDNA_FUSED_SE='1' +export VLLM_RDNA_GDN_LEAPQUANT='1' +export VLLM_RDNA_DENSE_INT8='0' +export VLLM_RDNA_DENSE_INT8_ONLY='0' +export VLLM_RDNA_DENSE_GEMV='0' +export VLLM_RDNA2_W4A8_SDOT4='1' +export VLLM_RDNA2_W4A8_BLOCK_M='8' +export VLLM_RDNA_W4A8_KBLOCK='-1' +export VLLM_RDNA_AR='1' +export VLLM_RDNA_AR_MAX_KB='256' +export VLLM_RDNA_AR_ONESHOT_KB='256' +export VLLM_RDNA_AR_BLOCKS='0' +export VLLM_RDNA_AR_PACE='0' +export VLLM_CAUSAL_CONV1D_RDNA2_FWD='0' +export VLLM_CAUSAL_CONV1D_RDNA2_UPDATE='0' +export VLLM_ENABLE_STARTUP_PLAN='0' +export VLLM_TUNED_CONFIG_FOLDER='/home/chenco_adm/w4a8_runs/leapquant/tree10/tuned-moe' +export VLLM_RDNA_DECODE_STALL_MS='450' +export VLLM_RDNA_DECODE_SHARE='0' +export VLLM_DECODE_STALL_MAX_OVERHEAD='1.0' +export HSA_FORCE_FINE_GRAIN_PCIE='1' +export HSA_ENABLE_SDMA='0' +export OMP_NUM_THREADS='4' +export TOKENIZERS_PARALLELISM='false' +export PYTHONFAULTHANDLER='1' + +# --- command --- +'/home/chenco_adm/Apps/vllm/venv-7.14.0_0.32.0/bin/python' '-m' 'vllm.entrypoints.openai.api_server' '--model' '/home/chenco_adm/.cache/huggingface/hub/models--wtdcode--Qwen3.8-Flash-Next-AWQ-W4A16/snapshots/0939125b929543a783ce700c90e36dd1a575c00c' '--served-model-name' 'flash-next' '--host' '127.0.0.1' '--port' '18096' '--attention-backend' 'RDNA_ATTN' '--tensor-parallel-size' '4' '--dtype' 'float16' '--block-size' '512' '--max-model-len' '262144' '--max-num-seqs' '8' '--max-num-batched-tokens' '2048' '--long-prefill-token-threshold' '0' '--prefill-schedule-interval' '1' '--kv-cache-memory-bytes' '4026531840' '--compilation-config' '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[3,6,12,24],"compile_ranges_endpoints":[]}' '--speculative-config' '{"method":"mtp","num_speculative_tokens":2,"use_local_argmax_reduction":true}' '--enable-expert-parallel' '--enable-prefix-caching' '--mamba-cache-mode' 'align' '--limit-mm-per-prompt' '{"image":1}' '--mm-processor-kwargs' '{"max_pixels":1605632}' '--monokernel' diff --git a/docs/rdna/serving.md b/docs/rdna/serving.md index 02772d354cc5..c25e0f4d4214 100644 --- a/docs/rdna/serving.md +++ b/docs/rdna/serving.md @@ -19,7 +19,7 @@ Nothing in the launcher or the recipes bakes in a home directory, a host, or an ROCm variables, `VLLM_*`, `HSA_*`, `NCCL_*` / `RCCL_*`, and vLLM serve flags belong to the process that runs `serve_rdna.sh`. An API client or a gateway (OpenAI SDK, OpenCode, LiteLLM, and others) receives the base URL and the served model name. It does not receive those variables or flags. -`GPU_MAX_HW_QUEUES` is one of those launch-environment variables. [`rdna_launcher_common.sh`](../../tools/rdna/rdna_launcher_common.sh) defaults it to **2**. Recipes that omit the key keep that default (`27b-awq`, `27b-exl3`, `dsv4-flash`, `full`, and both `flashnext-pp3*` recipes). The five Flash-Next TP=4 recipes — `flashnext`, `flashnext-exl3`, `flashnext-mtp0`, `flashnext-mtp2`, `flashnext-dflash2` — override it to **4**; leave that recipe value. Do not change those recipes to 2. A value of 2 as a host-specific fix is only an unverified workaround reported in [ROCm/TheRock#9306](https://github.com/ROCm/TheRock/issues/9306). +`GPU_MAX_HW_QUEUES` is one of those launch-environment variables. [`rdna_launcher_common.sh`](../../tools/rdna/rdna_launcher_common.sh) defaults it to **2**. Recipes that omit the key keep that default (`27b-awq`, `27b-exl3`, `dsv4-flash`, `full`, and both `flashnext-pp3*` recipes). The Flash-Next TP=4 recipes — `flashnext`, `flashnext-exl3`, `flashnext-mtp0`, `flashnext-mtp2`, `flashnext-leapquant-mtp2`, `flashnext-dflash2` — override it to **4**; leave that recipe value. Do not change those recipes to 2. A value of 2 as a host-specific fix is only an unverified workaround reported in [ROCm/TheRock#9306](https://github.com/ROCm/TheRock/issues/9306). ## Pick a recipe @@ -29,6 +29,7 @@ ROCm variables, `VLLM_*`, `HSA_*`, `NCCL_*` / `RCCL_*`, and vLLM serve flags bel | Flash-Next AWQ-W4A16, same parsers, no MTP | [`flashnext`](../../tools/rdna/recipes/flashnext.env) | TP=4 | `0.0.0.0:18094`, name `flash-next` | | Flash-Next AWQ, plain decode, 262144 context | [`flashnext-mtp0`](../../tools/rdna/recipes/flashnext-mtp0.env) | TP=4 | `127.0.0.1:18096`, name `flash-next` | | Flash-Next AWQ, MTP=2 | [`flashnext-mtp2`](../../tools/rdna/recipes/flashnext-mtp2.env) | TP=4 | `127.0.0.1:18096`, name `flash-next` | +| Flash-Next AWQ, lossy LeapQuant GDN, 512-token blocks, MTP=2 + MonoKernel | [`flashnext-leapquant-mtp2`](../../tools/rdna/recipes/flashnext-leapquant-mtp2.env) | TP=4 | `127.0.0.1:18096`, name `flash-next` | | Flash-Next AWQ + Inco DFlash2 | [`flashnext-dflash2`](../../tools/rdna/recipes/flashnext-dflash2.env) | TP=4 | `127.0.0.1:18096`, name `flash-next` | | Flash-Next AWQ on 3 cards, no MTP | [`flashnext-pp3`](../../tools/rdna/recipes/flashnext-pp3.env) | TP=1, PP=3 | `127.0.0.1:18096`, name `flash-next` | | 3 cards, MTP=2 | [`flashnext-pp3-mtp2`](../../tools/rdna/recipes/flashnext-pp3-mtp2.env) | TP=1, PP=3 | `127.0.0.1:18097`, name `flash-next` | @@ -89,7 +90,7 @@ These are shell variables in the `.env` files. They become flags. They are not e | `ATTN` | `fa` → `RDNA_ATTN` and `VLLM_FA_RDNA2_GQA_DECODE=1`. `triton` → Triton AMD FA. `none` → the launcher does not pass `--attention-backend`. | Leave `fa` on the Flash-Next and 27B recipes. Only the 27B recipes run the GQA decode kernel; the Flash-Next full-attention layers use QSA ([validation](../rdna2/fa-gqa-decode-validation-2026-11-11.md)). | | `MAXLEN` | `--max-model-len`. Empty uses the model's own context. | Lower it when KV memory is tight. | | `MIN_MAXLEN` | Refuse to start when `MAXLEN` is set below this. | `full` sets 32768. | -| `BLOCK_SIZE` | `--block-size`. | 1024 on the current Flash-Next and 27B recipes; 16 on `full`. | +| `BLOCK_SIZE` | `--block-size`. | 1024 on Flash-Next and 27B recipes, except 512 on `flashnext-leapquant-mtp2`; 16 on `full`. | | `SEQS` | `--max-num-seqs`. | Lower it when a long context does not fit. | | `MAXBAT` | `--max-num-batched-tokens`. Empty: the launcher derives it from the KV and SSM dtypes (2048 for an FP16 KV cache, 4096 for `int4_fp16_block` with an FP16 SSM state, 6144 with the model's FP32 SSM state). A prefill chunk ends on a mamba block, so the budget covers two blocks. | Leave `27b-awq` empty. Pinning it puts a packed KV cache back on a one-block budget (see that recipe). | | `LPTH` | `--long-prefill-token-threshold`. `0` disables the long-prefill split. | The recipes that set it use 0. | diff --git a/docs/rdna2/leapquant-gdn-2026-11-13.md b/docs/rdna2/leapquant-gdn-2026-11-13.md index 72a3bc1357e2..8a992d060c09 100644 --- a/docs/rdna2/leapquant-gdn-2026-11-13.md +++ b/docs/rdna2/leapquant-gdn-2026-11-13.md @@ -80,13 +80,13 @@ or uncomment `VLLM_RDNA_GDN_LEAPQUANT=1` in `tools/rdna/recipes/flashnext-mtp2.env`. The worker log prints `GDN recurrent state: LeapQuant INT8 (...)`. -Measured 512-token profile (MTP2 stays enabled; trade-offs and scope below): +Dedicated 512-token recipe (MTP2, MonoKernel and FULL_AND_PIECEWISE stay +enabled; window 32/rank 4, adaptive 450 ms mixed-step budget, decode share +zero). The existing `flashnext-mtp2` recipe remains unchanged and opt-in +LeapQuant is lossy; cold-TTFT/c1 trade-offs are recorded below. ```bash -bash tools/rdna/serve_rdna.sh RECIPE=flashnext-mtp2 \ - VLLM_RDNA_GDN_LEAPQUANT=1 BLOCK_SIZE=512 EXTRA_ARGS=--monokernel \ - VLLM_RDNA_DECODE_STALL_MS=450 VLLM_RDNA_DECODE_SHARE=0 \ - VLLM_DECODE_STALL_MAX_OVERHEAD=1.0 +bash tools/rdna/serve_rdna.sh RECIPE=flashnext-leapquant-mtp2 ``` ## Results (flashnext-mtp2, 4x V620 TP4, 2026-11-13) diff --git a/tools/rdna/recipes/flashnext-leapquant-mtp2.env b/tools/rdna/recipes/flashnext-leapquant-mtp2.env new file mode 100644 index 000000000000..ad3b877e6a6d --- /dev/null +++ b/tools/rdna/recipes/flashnext-leapquant-mtp2.env @@ -0,0 +1,55 @@ +# Flash-Next AWQ-W4A16 + LeapQuant INT8 GDN, TP4/EP4 on gfx1030. +# Opt-in, lossy: docs/rdna2/leapquant-gdn-2026-11-13.md. +# Window 32 / rank 4 is the serving layout; MTP2 and full graphs stay enabled. +# Measured against flashnext-mtp2 + --monokernel; short cold TTFT is higher. + +SERVE_ENTRY=api +SERVED_NAME=flash-next +PORT=18096 +TP=4 +HOST=127.0.0.1 +MTP=2 +ATTN=fa +MAXLEN=262144 +BLOCK_SIZE=512 +SEQS=8 +MAXBAT=2048 +LPTH=0 +PREFILL_INTERVAL=1 +KV=4026531840 +COMPILE_MODE=3 +CG_MODE=FULL_AND_PIECEWISE +CG_CAPTURE_SIZES=1 +CG_EXTRA=',"compile_ranges_endpoints":[]' +FEATURES="expert-parallel prefix-caching mamba-align vision-cap" +EXTRA_ARGS=--monokernel + +VLLM_RDNA_GDN_LEAPQUANT=1 +VLLM_RDNA_DECODE_STALL_MS=450 +VLLM_RDNA_DECODE_SHARE=0 +VLLM_DECODE_STALL_MAX_OVERHEAD=1.0 +VLLM_USE_V2_MODEL_RUNNER=1 +VLLM_GDN_HIP_PREFILL=1 +VLLM_RDNA_FUSED_SE=1 +VLLM_RDNA_DENSE_INT8=0 +VLLM_RDNA_DENSE_INT8_ONLY=0 +VLLM_RDNA_DENSE_GEMV=0 +VLLM_RDNA_SMALLM_GEMM_MAX_BYTES=4194304 +VLLM_RDNA2_W4A8_SDOT4=1 +VLLM_RDNA2_W4A8_BLOCK_M=8 +VLLM_RDNA_W4A8_KBLOCK=-1 +VLLM_RDNA_AR=1 +VLLM_RDNA_AR_MAX_KB=256 +VLLM_RDNA_AR_ONESHOT_KB=256 +VLLM_RDNA_AR_BLOCKS=0 +VLLM_RDNA_AR_PACE=0 +HSA_FORCE_FINE_GRAIN_PCIE=1 +HSA_ENABLE_SDMA=0 +GPU_MAX_HW_QUEUES=4 +OMP_NUM_THREADS=4 +TOKENIZERS_PARALLELISM=false +PYTHONFAULTHANDLER=1 +VLLM_CAUSAL_CONV1D_RDNA2_FWD=0 +VLLM_CAUSAL_CONV1D_RDNA2_UPDATE=0 +VLLM_ENABLE_STARTUP_PLAN=0 +TUNED_CONFIG=1 diff --git a/tools/rdna/serve_rdna.sh b/tools/rdna/serve_rdna.sh index f4ad064ffab8..763d065596e9 100755 --- a/tools/rdna/serve_rdna.sh +++ b/tools/rdna/serve_rdna.sh @@ -449,6 +449,7 @@ RDSE_ENV_KEYS=( RDNA_DFLASH2_DRAFT_BF16 VLLM_USE_BREAKABLE_CUDAGRAPH VLLM_RDNA_FUSED_HC VLLM_RDNA_HC_PREFILL_FUSED VLLM_GDN_HIP_PREFILL VLLM_RDNA_FUSED_SE + VLLM_RDNA_GDN_LEAPQUANT VLLM_RDNA_DENSE_INT8 VLLM_RDNA_DENSE_INT8_ONLY VLLM_RDNA_DENSE_GEMV VLLM_RDNA2_W4A8_SDOT4 VLLM_RDNA2_W4A8_DEBUG VLLM_RDNA2_W4A8_BLOCK_M VLLM_RDNA_W4A8_KBLOCK From 7b4aff7fe30ad5426877f38b38eb5ba697d3d62e Mon Sep 17 00:00:00 2001 From: BlivionIaG Date: Sat, 14 Nov 2026 20:37:50 +0100 Subject: [PATCH 16/18] docs(rdna2): credit LeapQuant, ReplaySSM and the RFC authors in headers and docs [skip ci] Signed-off-by: BlivionIaG --- csrc/rocm/gdn_leapquant_layout.cuh | 2 + csrc/rocm/gdn_leapquant_rdna2.cu | 3 + docs/rdna2/leapquant-gdn-2026-11-13.md | 83 ++++++++++++------- .../layers/mamba/ops/leapquant_gdn.py | 5 ++ 4 files changed, 62 insertions(+), 31 deletions(-) diff --git a/csrc/rocm/gdn_leapquant_layout.cuh b/csrc/rocm/gdn_leapquant_layout.cuh index 8b8520f0eed1..7ed3e8b3ee36 100644 --- a/csrc/rocm/gdn_leapquant_layout.cuh +++ b/csrc/rocm/gdn_leapquant_layout.cuh @@ -1,3 +1,5 @@ +// Algorithm: LeapQuant (Pan et al., arXiv 2609.38166); record ring after +// ReplaySSM (vllm-project/vllm#62004). Original code written for this fork. // LeapQuant GDN blob layout shared by gdn_leapquant_rdna2.cu and the // MonoKernel GDN (mono_gdn_rdna2.cu). Must match LeapQuantLayout.offsets in // vllm/model_executor/layers/mamba/ops/leapquant_gdn.py. diff --git a/csrc/rocm/gdn_leapquant_rdna2.cu b/csrc/rocm/gdn_leapquant_rdna2.cu index 7f788347667e..665818ec2f00 100644 --- a/csrc/rocm/gdn_leapquant_rdna2.cu +++ b/csrc/rocm/gdn_leapquant_rdna2.cu @@ -1,3 +1,6 @@ +// Algorithm: LeapQuant (Pan et al., arXiv 2609.38166). Record ring, cursor and +// per-window flush follow ReplaySSM (Dao AI Lab and NVIDIA; vllm-project/vllm +// #48018, #62004). These HIP kernels are original code written for this fork. // LeapQuant INT8 recurrent-state storage for GatedDeltaNet on AMD RDNA2 // (gfx1030). arXiv 2609.38166; layout and FP32 oracle: // vllm/model_executor/layers/mamba/ops/leapquant_gdn.py. diff --git a/docs/rdna2/leapquant-gdn-2026-11-13.md b/docs/rdna2/leapquant-gdn-2026-11-13.md index 8a992d060c09..f772f9214d72 100644 --- a/docs/rdna2/leapquant-gdn-2026-11-13.md +++ b/docs/rdna2/leapquant-gdn-2026-11-13.md @@ -5,6 +5,27 @@ requested in [vllm#64725](https://github.com/vllm-project/vllm/issues/64725)) to the RDNA2 Gated DeltaNet path. Opt-in with `VLLM_RDNA_GDN_LEAPQUANT=1` (gfx10x only; the flag raises elsewhere). Default off: the change is lossy. +## Credits + +- **Algorithm:** LeapQuant, Yi Pan, Haocheng Xi, Kan Zhu, Xingyang Li, Yibo Wu, + Mayank Mishra, Hongtao Zhang, William X. Zheng, Baris Kasikci, Song Han, + Kurt Keutzer, Rishabh Iyer and Ion Stoica + ([arXiv 2609.38166](https://arxiv.org/abs/2609.38166)). +- **Record ring, cursor and per-window flush:** the ReplaySSM design by Dao AI Lab + and NVIDIA ([blog](https://dao-lab.ai/blog/2026/replayssm/), + [vllm#48018](https://github.com/vllm-project/vllm/pull/48018), + [vllm#62004](https://github.com/vllm-project/vllm/pull/62004) by Johnny-Liou). + ReplaySSM for Mamba2 reaches this tree through the upstream merge. Its GDN + decode kernel (#62004, still open upstream) is not in this tree. +- **Proposal and design context:** the LeapQuant RFC + [vllm#64725](https://github.com/vllm-project/vllm/issues/64725) by + lihaoyang-amd. It reports a prototype with one fold launch across layers and + the per-window FP16 checkpoint baseline that LeapQuant must beat on memory. + The RFC has no source code, and none of it is used here. +- **Code:** the HIP kernels, layout header, FP32 oracle, scheduler changes and + tests in this PR are original work for this fork. Existing files keep their + upstream authors in git history. + ## What it stores The FP32 state of one value head (`[V, K] = [128, 128]`, 64 KiB) becomes one @@ -35,39 +56,39 @@ instead of once per token. The FP32 oracle is ## How it fits the serving path (MTP2, cudagraphs, prefix caching) -* **Slots are record counts.** The FP32 layout keeps one state per speculative +- **Slots are record counts.** The FP32 layout keeps one state per speculative token (`spec_state_indices[n, t]`). The blob lives at the home block `spec_state_indices[n, 0]`; "slot `t`" is "commit `t + 1` pending records". A decode/verify step appends its records and sets `pending`; `gdn_lq_normalize_rdna2` (first op of the next step) commits `num_accepted` of them (rejected drafts are dropped by never committing) and folds once `cursor >= window`. MTP verify needs no per-token state copies. -* **Readers of slot 0** (chunked prefill, non-spec decodes) commit +- **Readers of slot 0** (chunked prefill, non-spec decodes) commit `min(1, pending)`, the same state the FP32 path reads. -* **Prefill** materializes the blob to an FP32 `[N, HV, V, K]` initial state, +- **Prefill** materializes the blob to an FP32 `[N, HV, V, K]` initial state, runs the unchanged HIP prefill chain (`VLLM_GDN_HIP_PREFILL=1`) and stores the final state back (`gdn_lq_store_rdna2`). Inside a chunk nothing changes; LeapQuant error enters at chunk boundaries (MAXBAT 2048). -* **Prefix caching (`mamba-align`, V2 runner):** the align precopy/postprocess +- **Prefix caching (`mamba-align`, V2 runner):** the align precopy/postprocess HIP kernels (`v1_mamba_rdna2.cu`, `copy_mamba_state_block`) copy the whole blob from the home block and write `cursor + min(bias + 1, pending)` into the copy's header, which is exactly the FP32 "copy slot `bias`" rule. The V1 runner's memcpy specs and the Triton precopy fallback raise instead of silently copying wrong bytes. -* **MonoKernel (`EXTRA_ARGS=--monokernel`):** pure decode and pure verify +- **MonoKernel (`EXTRA_ARGS=--monokernel`):** pure decode and pure verify batches run `gdn_lq_normalize_rdna2` and then `mono_gdn_lq_rdna2`, the `mono_gdn_rdna2` fused conv + delta rule + gated norm launch with the LeapQuant load/append in place of the FP32 state load/store (each 32-row tile rebuilds its rows from the blob; records are written per tile). Batches with prefill take the layer's path as with FP32 state. -* **HIP only.** Every LeapQuant read and write is one of the kernels above; +- **HIP only.** Every LeapQuant read and write is one of the kernels above; the FP32-state Triton decode warmup is skipped under the flag (it never runs). Paths the flag does not change (conv update, gated norm outside the MonoKernel) are the recipe's, identical with the flag off. -* **Cudagraphs:** every op is a fixed-shape launch on the current stream that +- **Cudagraphs:** every op is a fixed-shape launch on the current stream that reads its metadata from device tensors, so FULL_AND_PIECEWISE captures it like the FP32 kernels. -* **Refused combinations** (raise at init): fused CUDA spec decode, +- **Refused combinations** (raise at init): fused CUDA spec decode, AITER/FlashInfer GDN, align mode on the V1 runner, head dims other than 128. ## Use @@ -269,13 +290,13 @@ difference's cause. The user rejected a watermark/cadence workaround. The opt-in LeapQuant scheduler now changes admission and work ordering instead: -* **Reserve bounded growth before admission.** The full-sequence check uses +- **Reserve bounded growth before admission.** The full-sequence check uses prompt length + requested maximum output + an in-flight verification tail, clamped to the model context limit. The tail includes every configured concurrent batch, not just one final verify. Draft KV lookahead is separate. Already admitted requests' unallocated growth is subtracted from free blocks. These are allocator-derived promises, not eagerly allocated KV pages. -* **Account for physical recurrent peaks.** A future single-step allocation +- **Account for physical recurrent peaks.** A future single-step allocation alone undercounts Mamba align mode: initial state + two draft slots is three pages. A synchronous boundary transition needs four; two unresolved prefill chunks can keep five live pages. The peak bound includes @@ -283,11 +304,11 @@ scheduler now changes admission and work ordering instead: limited by the logical state columns. The coordinator takes the larger of its normal prediction and this peak, subtracting held/adopted pages and accounting for evictable prefix hits. -* **Ready decoders first under FCFS.** Their target and draft rows consume +- **Ready decoders first under FCFS.** Their target and draft rows consume the input budget before running prefills; prefills use what remains. Explicit priority policy is unchanged. No sleep, watermark increase, fixed sequence cap, cadence change, or reduction in speculative width. -* **No reservation ledger.** Growth is derived from live group tables. +- **No reservation ledger.** Growth is derived from live group tables. Allocated pages stop being promises; early EOS and cancellation release the request's remaining promise. Shared cached prefixes are not charged twice. @@ -422,12 +443,12 @@ MonoKernel on, same columns: | 32k/2k c=4 | +0.1 % | -0.0 % | +0.5 % | +0.3 % (regr.) | +3.5 % | | 32k/2k c=8 | +2.7 % (regr.) | -9.5 % (regr.) | +0.9 % (regr.) | +0.1 % (regr.) | -3.9 % | -* Output aggregate follows MTP acceptance (1.68-2.60 per boot), not the +- Output aggregate follows MTP acceptance (1.68-2.60 per boot), not the flag. -* **c=1 ITL p99 +6 %** (23.6 -> 25.0 ms) is the fold: all 36 layers +- **c=1 ITL p99 +6 %** (23.6 -> 25.0 ms) is the fold: all 36 layers refold on the same step every ~16 committed tokens, one layer after the other. -* 16k/1k c=8 TTFT p50 is bimodal per boot (MonoKernel-on base boots: 11.7 s +- 16k/1k c=8 TTFT p50 is bimodal per boot (MonoKernel-on base boots: 11.7 s and 7.0 s); prefill aggregate does not move. ### Efficiency pass: one normalize launch per KV-cache group @@ -438,7 +459,7 @@ Per decode step on one V620 (36 layers, HIP-graph replay, on folds (40 us per fold workgroup, one layer after the other) and ~55 us on replaying the window's records. -* **Grouped normalize.** The GDN layers of one KV-cache group share a block +- **Grouped normalize.** The GDN layers of one KV-cache group share a block table and one attention metadata, so the group's first LeapQuant layer normalizes all of its layers' blobs in one launch (`layer_offsets`, grid.y = layer) and the others skip. The groups are built at KV-cache bind @@ -449,9 +470,9 @@ replaying the window's records. decode) stay per layer. Same bytes as per-layer normalize (`test_grouped_normalize_matches_per_layer`, `test_normalize_groups_follow_kv_cache_groups`). -* **Replay loads.** The record replay loops unroll by 2 so two records' +- **Replay loads.** The record replay loops unroll by 2 so two records' loads issue together (unroll 4 was slower at n=8). Same arithmetic order. -* Tried and dropped: LDS instead of shuffle reductions in the fold (40 us -> +- Tried and dropped: LDS instead of shuffle reductions in the fold (40 us -> 39 us at n=1, 96 us -> 110 us at n=8). Power iterations 2 -> 0 saves 20-40 us per step but changes the compensators; not taken, quality not measured. @@ -478,19 +499,19 @@ state slots per group. The leader committed the other groups' records at its own group's slots, which belong to other requests in those groups. Their own slots were never committed. -* Symptom: decode fell into repetition loops, which MTP accepts easily. +- Symptom: decode fell into repetition loops, which MTP accepts easily. The MonoKernel-on ABBA receipt showed fake speed "wins" (c=1 output +37 %, ITL p50 -4 to -8 % at c=4/8). Outputs whose last 600 chars compress below 0.12 (zlib), out of 32 at c=8: base boots 2-7, flag-on boots 10-21. On the first build, the flag-on boots matched base (1-6). -* The gated checks missed it. Prompt KL and teacher-forced PPL do not run +- The gated checks missed it. Prompt KL and teacher-forced PPL do not run decode (head KL 0.01178/0.01172 vs floor CI 0.01413, PPL +0.47/+0.15 %). The concurrency probe hinted at it: 1 FAIL and 10 soft divergences, vs 0-5 soft on every other boot. -* Isolation: the tree3 and tree2 kernels are bitwise identical over 200 +- Isolation: the tree3 and tree2 kernels are bitwise identical over 200 MTP steps (decode, normalize, MonoKernel). Disabling only the Python grouping restores base behaviour. -* Fix: `d879da8c93` builds one group per KV-cache group. +- Fix: `d879da8c93` builds one group per KV-cache group. `test_normalize_groups_follow_kv_cache_groups` uses two groups with different slots per request. @@ -511,21 +532,21 @@ columns and rule as above: | 32k/2k c=4 | +1.5 % (regr.) | +0.0 % | -1.0 % | -0.1 % | +1.4 % | | 32k/2k c=8 | +1.7 % | +2.1 % (win) | +0.1 % | -0.1 % (win) | +4.6 % (win) | -* **The c=1 ITL p99 fold spike is gone** (+6 % in the first build, now +- **The c=1 ITL p99 fold spike is gone** (+6 % in the first build, now -0.6 / -0.3 / +0.3 %): each KV-cache group's folds run in one launch. -* ITL p50 moves -0.1 to -1.4 %, and -5.4 % at 1k/512 c=8 (45.2 vs 47.8 ms). -* At 1k/512 c=1 (4 requests), acceptance length from each benchmark JSON +- ITL p50 moves -0.1 to -1.4 %, and -5.4 % at 1k/512 c=8 (45.2 vs 47.8 ms). +- At 1k/512 c=1 (4 requests), acceptance length from each benchmark JSON is A1=2.034, B1=1.774, B2=1.622, A2=2.269 tokens per step. Both flag-on boots have lower acceptance and output throughput despite similar ITL. The -19.3 % throughput result remains a regression to investigate. Other cells move in both directions; pooling their acceptance logs cannot establish parity for this cell. -* 1k/512 c=4 ITL p99 +258 % is one boot (B2 380.9 ms vs 60.9-62.6 ms on the +- 1k/512 c=4 ITL p99 +258 % is one boot (B2 380.9 ms vs 60.9-62.6 ms on the others). The first build had the same stall on a base boot (-72 %). -* At 32k/2k c=8 every arm logs `Running: 5-6 reqs, Waiting: 2-3 reqs` +- At 32k/2k c=8 every arm logs `Running: 5-6 reqs, Waiting: 2-3 reqs` (TTFT p50 ~85 s). This is consistent with KV admission pressure [INFERENCE]; the 512 experiment will check admission and throughput. -* Gates: quality PASS (head KL 0.01158 / 0.01178 vs floor 0.01376, top-1 +- Gates: quality PASS (head KL 0.01158 / 0.01178 vs floor 0.01376, top-1 0.967 vs 0.964, PPL +0.055 / -0.077 %), concurrency probe PASS with 0 failing on both flag-on boots, stall PASS. Repetition loops (`loopscan.py`, out of 32 per c=8 cell): base 0-5, flag on 1-5. @@ -548,12 +569,12 @@ Speed is mixed (5 winning cells, 4 with at least one regression): | 32k/2k c=4 | -0.9 % | +0.0 % | -1.2 % | +0.0 % | +1.4 % | | 32k/2k c=8 | +3.3 % | -0.6 % | -2.0 % | -0.1 % | -7.2 % | -* Both head boots pass KL against the floor (0.01186/0.01205 vs 0.01490), +- Both head boots pass KL against the floor (0.01186/0.01205 vs 0.01490), PPL (+0.121/+0.350 %, limit 2 %) and concurrency (0 failing). -* At 1k c=1, acceptance length is A1=1.814, B1=1.865, B2=1.909, +- At 1k c=1, acceptance length is A1=1.814, B1=1.865, B2=1.909, A2=1.787: the MonoKernel-on acceptance regression does not reproduce in this MonoKernel-off receipt. This does not establish the cause. -* Loop scan: c=8 base 2-5/32, head 1-4/32. At 16k c=1 and 32k c=1, +- Loop scan: c=8 base 2-5/32, head 1-4/32. At 16k c=1 and 32k c=1, B1 flags 2/4 while the other boots flag 0/4; retain these as quality concerns for the larger decode study rather than treating gate PASS as complete decode-quality evidence. diff --git a/vllm/model_executor/layers/mamba/ops/leapquant_gdn.py b/vllm/model_executor/layers/mamba/ops/leapquant_gdn.py index 96d5819addd5..095fea33c3ea 100644 --- a/vllm/model_executor/layers/mamba/ops/leapquant_gdn.py +++ b/vllm/model_executor/layers/mamba/ops/leapquant_gdn.py @@ -1,5 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# Algorithm: LeapQuant, Y. Pan, H. Xi, K. Zhu, X. Li, Y. Wu, M. Mishra, H. Zhang, +# W. X. Zheng, B. Kasikci, S. Han, K. Keutzer, R. Iyer, I. Stoica (arXiv 2609.38166). +# The record ring, cursor and per-window flush follow ReplaySSM (Dao AI Lab and +# NVIDIA; vllm-project/vllm#48018, #62004). The FP32 oracle below and the RDNA2 +# HIP kernels are original code written for this fork. """LeapQuant recurrent-state storage for Gated DeltaNet (arXiv 2609.38166). The FP32 GDN state ``S`` of one value head (``[V, K]``, the row-major layout of From a60a54d6d2b566682040399cd752374d9a88ffa1 Mon Sep 17 00:00:00 2001 From: BlivionIaG Date: Sat, 14 Nov 2026 22:22:42 +0100 Subject: [PATCH 17/18] bench(rdna2): LeapQuant versus FP32 four-boot matrix, 1k/16k/32k at c=1 and c=8 [skip ci] Signed-off-by: BlivionIaG --- .../matrix-report.json | 1412 +++++++++++++++++ .../matrix-tables.md | 30 + 2 files changed, 1442 insertions(+) create mode 100644 bench_results/2026-11-14_leapquant-matrix/matrix-report.json create mode 100644 bench_results/2026-11-14_leapquant-matrix/matrix-tables.md diff --git a/bench_results/2026-11-14_leapquant-matrix/matrix-report.json b/bench_results/2026-11-14_leapquant-matrix/matrix-report.json new file mode 100644 index 000000000000..9e90010b597e --- /dev/null +++ b/bench_results/2026-11-14_leapquant-matrix/matrix-report.json @@ -0,0 +1,1412 @@ +{ + "protocol": { + "order": [ + "A1", + "C1", + "C2", + "A2" + ], + "boot_repeats_per_mode": 2, + "requests_per_cell_per_boot": 16, + "baseline": "FP32 state/block1024, flashnext-mtp2 + MonoKernel, budget0", + "leapquant": "window32/rank4/block512, flashnext-leapquant-mtp2, adaptive450ms/share0/overhead1", + "shared": "TP4 V620 b4 HIP2-5 d9/dc/df/e2; MTP2, MonoKernel, FULL_AND_PIECEWISE", + "sampling": "T1 top_p.95 top_k20 min_p0 presence0 repetition1; random/completions dataset seed12345; no API seed; ignore EOS", + "warmup": "One separate 64-output request at each workload length before its measured group; prefix cache reset before every cell", + "aggregation": "Rates/latencies mean of two boot summaries; percentile columns are means of per-boot percentiles. Acceptance pools counts. PP/request=input/meanTTFT; PP wave=c*input/p99TTFT, not separately timed GPU PP; TG/request=1000/meanTPOT; TG aggregate=outputs/walltime including prefill.", + "quality_scope": "PPL/prefix/concurrency/after-load probes plus random-output loop flags; not a new powered task-accuracy or decode-KL study" + }, + "throughput_requests": 384, + "boots": { + "A1": { + "run": "/home/chenco_adm/w4a8_runs/leapquant/matrix-A1-20261114-191152", + "peak_junction_c": 83.0, + "ppl": 4.83, + "verification_summary": "[19:12:02] boot TREE=/home/chenco_adm/w4a8_runs/leapquant/tree10 RECIPE=flashnext-mtp2 MODEL=/home/chenco_adm/.cache/huggingface/hub/models--wtdcode--Qwen3.8-Flash-Next-AWQ-W4A16/snapshots/0939125b929543a783ce700c90e36dd1a575c00c GPUS=2,3,4,5 PORT=18787 overrides=VLLM_RDNA_GDN_LEAPQUANT=0 BLOCK_SIZE=1024 VLLM_SERVER_DEV_MODE=1 VLLM_RDNA_DECODE_STALL_MS=0 VLLM_RDNA_DECODE_SHARE=0 VLLM_DECODE_STALL_MAX_OVERHEAD=1.0 EXTRA_ARGS=--monokernel bench_extra=\n[19:12:02] waiting for boot lock /home/chenco_adm/w4a8_runs/boot.lock\n[19:16:42] READY in 280s served=flash-next\n[19:16:49] PROBE PASS\nPASS prompt='The capital of France is'\n reply=' Paris. Paris is a city of great historical and cultural significance, known for its iconic landmarks such as the Eiffel Tower, the Louvre Museum, and Notre-Dame Cathedral. It is also a major center for fashion, gastronomy,'\nPASS prompt='1+1='\n reply='2, 2+2=4, 3+3=6, 4+4=8, 5+5=10, 6+6=12, 7+7=14, '\nPASS prompt='The capital of Germany is'\n reply=\" Berlin, which is also the country's largest city and one of its 16 federal states. With a population of approximately 3.5 million, Berlin is the second most populous city in the European Union. It serves as the headquarters\"\nRESULT PASS (3/3)\n[19:16:56] PPL PROBE PASS\n PPL 4.83 (mean NLL 1.576 over 261 tokens) PASS\n ok 'The history of the printing press begins' -> ' with the invention of movable type in China during the Song Dynasty (960-1279 AD). Bi Sheng, a Chinese artisan, is credited with creating the first movable type printing system around 1040 AD. This innovation allowed for the mass production of texts, significantly impacting the spread of'\n ok 'Here is a short recipe for vegetable soup:' -> '\\n\\n**Ingredients**\\n- 1 onion, chopped \\n- 2 carrots, sliced \\n- 2 celery stalks, chopped \\n- 1 potato, diced \\n- 1 can (14 oz) diced tomatoes \\n- 4 cups vegetable broth \\n- 1 tsp dried thyme \\n- Salt'\n ok 'In mathematics, a prime number is' -> ' a natural number greater than 1 that has no positive divisors other than 1 and itself. This definition establishes the foundational criterion for identifying primes. To determine whether a given number is prime, one must verify that it is not divisible by any integer between 2 and the square root of the number, inclusive. This'\n RESULT PASS\n[19:17:03] PREFIX PASS\n cold TTFT 0.86 s, warm TTFT 0.40 s, speedup 2.2x -> PASS\n prefix cache: queries +3854, hits +1024, warm hit rate 53.1% (hit granularity 2048 tokens)\n cold text: '\\n\\n\\nThe user wants me to summarize a long text that appears to be a sequence of GPU/computing-related terms (like '\n warm text: '\\n\\n\\nThe user wants me to summarize a long text that appears to be a sequence of GPU/computing-related terms (like '\n[19:17:17] warmup rc=0 in 14s (sizes default); slowest:\n decode batch 3: 1.76 s\n decode batch 4: 1.39 s\n decode batch 6: 0.82 s\n[19:17:49] CONCURRENCY PASS\n WARN churn (8, 0) prompt 0: near-tie token 4/16: margin solo 0.016 got 0.266, continuation mean lp -0.62 vs solo -0.89\n WARN churn (10, 6) prompt 6: near-tie token 15/24: margin solo 0.094 got 0.047, continuation mean lp -0.11 vs solo -0.12\n WARN churn (13, 7) prompt 7: near-tie token 3/40: margin solo 0.109 got 0.000, continuation mean lp -0.47 vs solo -0.26\n WARN churn (14, 2) prompt 2: near-tie token 43/48: margin solo 0.234 got 0.078, continuation mean lp -0.19 vs solo -0.15\n WARN long 0 prompt 0: near-tie token 47/64: margin solo 0.047 got 0.078, continuation mean lp -0.23 vs solo -0.25\n WARN long 2 prompt 2: near-tie token 46/64: margin solo 0.047 got 0.031, continuation mean lp -0.14 vs solo -0.14\n WARN long 3 prompt 3: near-tie token 42/64: margin solo 0.234 got 0.078, continuation mean lp -0.27 vs solo -0.22\n RESULT PASS: 0 failing, 20 near-tie, 0 soft, 0 solo-noise, 28 identical of 48 checks (27.1s; tie <= 0.2, fail > 1.0 nats, lp drop > 1.5; run-to-run noise 1.86 nats, a flip fails if both runs are sure)\n[19:17:49] load hook: /home/chenco_adm/w4a8_runs/leapquant/tree10/.tmp/leapquant_matrix.sh\n[20:05:37] LOAD HOOK rc=0 PASS\n [19:49:23] START warmup-long-c4 n=1\n [19:49:52] FINISH warmup-long-c4\n [19:49:57] START bench-32768x2048-c4 n=16\n [19:56:11] FINISH bench-32768x2048-c4\n [19:56:11] WINDOW-END long-c4\n [19:58:12] WINDOW long-c8\n [19:58:17] START warmup-long-c8 n=1\n [19:58:46] FINISH warmup-long-c8\n [19:58:51] START bench-32768x2048-c8 n=16\n [20:05:37] FINISH bench-32768x2048-c8\n [20:05:37] WINDOW-END long-c8\n MATRIX HOOK PASS\n[20:05:43] PROBE-AFTER-LOAD PASS\n[20:05:43] evicted_ms since READY: Worker_TP0_EP0=+0 Worker_TP1_EP1=+0 Worker_TP2_EP2=+0 Worker_TP3_EP3=+0 -> ok\n[20:05:43] markers:\n 4 Initialized AMD PLE embedding language_model.model.layers.1.ple.ple_embedding.ngram_embedding: quantization_config=CompressedTensorsConfig, quantizat\n 2 rdna_ar: hierarchical all-reduce (VLLM_RDNA_AR_HIER=auto) does not apply, using t\n 1 WNA16 MoE backend.\n 1 Using CompressedTensorsWNA16RDNA2MoEMethod (native RDNA2 W4A16 HIP kernel)\n 1 rdna_ar: one-shot all-reduce active (handle 1, rank 3/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 1, rank 2/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 1, rank 1/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 1, rank 0/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 0, rank 3/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 0, rank 2/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 0, rank 1/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 0, rank 0/4, devices [0, 1, \n 1 qwen_gdn_full_forward', 'vllm::gdn_attention_core_x\n 1 qwen_gdn_full_forward (opaque to inductor; projecti\n 1 Model loading took 20.56 GiB memory and 87.249807 seconds\n 1 Model loading took 20.56 GiB memory and 87.104928 seconds\n 1 Model loading took 20.56 GiB memory and 86.314715 seconds\n 1 Model loading took 20.56 GiB memory and 85.971809 seconds\n 1 GPU KV cache size: 266,945 tokens, Maximum concurrency fo\n[20:05:43] fault markers in serve.log: 0\n[20:05:43] new amdgpu dmesg faults: 0\nn/a\n[20:05:43] done\n", + "preemptions_total": 10.0, + "preemptions_during_matrix": 10.0, + "loop_flags": 17 + }, + "C1": { + "run": "/home/chenco_adm/w4a8_runs/leapquant/matrix-C1-20261114-200608", + "peak_junction_c": 83.0, + "ppl": 4.81, + "verification_summary": "[20:06:18] boot TREE=/home/chenco_adm/w4a8_runs/leapquant/tree10 RECIPE=flashnext-leapquant-mtp2 MODEL=/home/chenco_adm/.cache/huggingface/hub/models--wtdcode--Qwen3.8-Flash-Next-AWQ-W4A16/snapshots/0939125b929543a783ce700c90e36dd1a575c00c GPUS=2,3,4,5 PORT=18787 overrides=VLLM_RDNA_GDN_LEAPQUANT=1 BLOCK_SIZE=512 VLLM_SERVER_DEV_MODE=1 VLLM_RDNA_DECODE_STALL_MS=450 VLLM_RDNA_DECODE_SHARE=0 VLLM_DECODE_STALL_MAX_OVERHEAD=1.0 EXTRA_ARGS=--monokernel bench_extra=\n[20:06:18] waiting for boot lock /home/chenco_adm/w4a8_runs/boot.lock\n[20:10:58] READY in 280s served=flash-next\n[20:11:05] PROBE PASS\nPASS prompt='The capital of France is'\n reply=' Paris. Paris is the capital of France.\\n\\n### 1.2.2.2 \u57fa\u4e8e\u89c4\u5219\u7684\u8bed\u6cd5\\n\\n\u57fa\u4e8e\u89c4\u5219\u7684\u8bed\u6cd5 (Rule-based Grammar) \u662f\u6307\u7528\u4eba\u5de5\u5236\u5b9a\u7684\u89c4\u5219, \u7cbe\u786e\u5730\u3001\u5f62\u5f0f\u5316\u5730\u63cf\u8ff0\u8bed\u8a00'\nPASS prompt='1+1='\n reply='2, 2+2=4, 3+3=6, 4+4=8, 5+5=10, 6+6=12, 7+7=14, '\nPASS prompt='The capital of Germany is'\n reply=\" Berlin, which is also the country's largest city and one of its 16 federal states. With a population of approximately 3.5 million, Berlin is the second most populous city in the European Union. It serves as a major\"\nRESULT PASS (3/3)\n[20:11:12] PPL PROBE PASS\n PPL 4.81 (mean NLL 1.570 over 261 tokens) PASS\n ok 'The history of the printing press begins' -> ' with the invention of movable type in China during the Song Dynasty (960-1279 AD). Bi Sheng, a Chinese artisan, is credited with creating the first movable type printing system around 1040 AD. This innovation allowed for the mass production of texts, significantly impacting the spread of'\n ok 'Here is a short recipe for vegetable soup:' -> '\\n\\n1. Heat 2 tablespoons of olive oil in a large pot over medium heat.\\n2. Add 1 chopped onion and 3 minced garlic cloves, and saut\u00e9 until softened.\\n3. Add 2 chopped carrots, 2 chopped celery stalks, and 1 chopped bell pepper, and cook'\n ok 'In mathematics, a prime number is' -> ' a natural number greater than 1 that has no positive divisors other than 1 and itself. This definition establishes the foundational criterion for identifying primes. To determine whether a given number is prime, one must verify that it is not divisible by any integer between 2 and the square root of the number, inclusive. This'\n RESULT PASS\n[20:11:19] PREFIX PASS\n cold TTFT 0.90 s, warm TTFT 0.28 s, speedup 3.2x -> PASS\n prefix cache: queries +3854, hits +1536, warm hit rate 79.7% (hit granularity 2048 tokens)\n cold text: '\\n\\n\\nThe user wants me to summarize a long text that appears to be a sequence of GPU/computing-related terms (like '\n warm text: '\\n\\n\\nThe user wants me to summarize a long text that appears to be a sequence of GPU/computing-related terms (like '\n[20:11:34] warmup rc=0 in 15s (sizes default); slowest:\n decode batch 2: 1.88 s\n decode batch 4: 1.37 s\n decode batch 6: 1.06 s\n[20:12:05] CONCURRENCY PASS\n WARN churn (2, 6) prompt 6: near-tie token 15/24: margin solo 0.016 got 0.000, continuation mean lp -0.11 vs solo -0.12\n WARN churn (8, 0) prompt 0: near-tie token 12/16: margin solo 0.023 got 0.000, continuation mean lp -0.99 vs solo -0.95\n WARN churn (10, 6) prompt 6: near-tie token 15/24: margin solo 0.016 got 0.047, continuation mean lp -0.11 vs solo -0.12\n WARN churn (11, 1) prompt 1: near-tie token 18/64: margin solo 0.000 got 0.094, continuation mean lp -0.24 vs solo -0.30\n WARN long 0 prompt 0: near-tie token 11/64: margin solo 0.156 got 0.219, continuation mean lp -0.23 vs solo -0.22\n WARN long 1 prompt 1: near-tie token 48/64: margin solo 0.234 got 0.125, continuation mean lp -0.18 vs solo -0.21\n WARN long 3 prompt 3: near-tie token 42/64: margin solo 0.078 got 0.078, continuation mean lp -0.28 vs solo -0.22\n RESULT PASS: 0 failing, 16 near-tie, 1 soft, 0 solo-noise, 31 identical of 48 checks (27.0s; tie <= 0.2, fail > 1.0 nats, lp drop > 1.5; run-to-run noise 1.99 nats, a flip fails if both runs are sure)\n[20:12:05] load hook: /home/chenco_adm/w4a8_runs/leapquant/tree10/.tmp/leapquant_matrix.sh\n[20:58:40] LOAD HOOK rc=0 PASS\n [20:44:17] FINISH warmup-long-c4\n [20:44:22] START bench-32768x2048-c4 n=16\n [20:50:27] FINISH bench-32768x2048-c4\n [20:50:27] WINDOW-END long-c4\n [20:52:28] WINDOW long-c8\n [20:52:33] START warmup-long-c8 n=1\n [20:53:02] FINISH warmup-long-c8\n [20:53:07] START bench-32768x2048-c8 n=16\n [20:58:40] FINISH bench-32768x2048-c8\n [20:58:40] WINDOW-END long-c8\n PREEMPTION GATE PASS: zero preemptions across this boot\n MATRIX HOOK PASS\n[20:58:47] PROBE-AFTER-LOAD PASS\n[20:58:47] evicted_ms since READY: Worker_TP0_EP0=+0 Worker_TP1_EP1=+0 Worker_TP2_EP2=+0 Worker_TP3_EP3=+0 -> ok\n[20:58:47] markers:\n 4 Initialized AMD PLE embedding language_model.model.layers.1.ple.ple_embedding.ngram_embedding: quantization_config=CompressedTensorsConfig, quantizat\n 2 rdna_ar: hierarchical all-reduce (VLLM_RDNA_AR_HIER=auto) does not apply, using t\n 1 WNA16 MoE backend.\n 1 Using CompressedTensorsWNA16RDNA2MoEMethod (native RDNA2 W4A16 HIP kernel)\n 1 rdna_ar: one-shot all-reduce active (handle 1, rank 3/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 1, rank 2/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 1, rank 1/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 1, rank 0/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 0, rank 3/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 0, rank 2/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 0, rank 1/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 0, rank 0/4, devices [0, 1, \n 1 qwen_gdn_full_forward', 'vllm::gdn_attention_core_x\n 1 qwen_gdn_full_forward (opaque to inductor; projecti\n 1 Model loading took 20.28 GiB memory and 87.214551 seconds\n 1 Model loading took 20.28 GiB memory and 86.710545 seconds\n 1 Model loading took 20.28 GiB memory and 86.398442 seconds\n 1 Model loading took 20.28 GiB memory and 86.337664 seconds\n 1 GPU KV cache size: 273,456 tokens, Maximum concurrency fo\n[20:58:47] fault markers in serve.log: 0\n[20:58:47] new amdgpu dmesg faults: 0\nn/a\n[20:58:47] done\n", + "preemptions_total": 0.0, + "preemptions_during_matrix": 0.0, + "loop_flags": 16 + }, + "C2": { + "run": "/home/chenco_adm/w4a8_runs/leapquant/matrix-C2-20261114-205912", + "peak_junction_c": 83.0, + "ppl": 4.85, + "verification_summary": "[20:59:22] boot TREE=/home/chenco_adm/w4a8_runs/leapquant/tree10 RECIPE=flashnext-leapquant-mtp2 MODEL=/home/chenco_adm/.cache/huggingface/hub/models--wtdcode--Qwen3.8-Flash-Next-AWQ-W4A16/snapshots/0939125b929543a783ce700c90e36dd1a575c00c GPUS=2,3,4,5 PORT=18787 overrides=VLLM_RDNA_GDN_LEAPQUANT=1 BLOCK_SIZE=512 VLLM_SERVER_DEV_MODE=1 VLLM_RDNA_DECODE_STALL_MS=450 VLLM_RDNA_DECODE_SHARE=0 VLLM_DECODE_STALL_MAX_OVERHEAD=1.0 EXTRA_ARGS=--monokernel bench_extra=\n[20:59:22] waiting for boot lock /home/chenco_adm/w4a8_runs/boot.lock\n[21:04:03] READY in 281s served=flash-next\n[21:04:09] PROBE PASS\nPASS prompt='The capital of France is'\n reply=' Paris. Paris is the capital of France.\\n\\n### 1.2.2.2 \u57fa\u4e8e\u89c4\u5219\u7684\u8bed\u6cd5\\n\\n\u57fa\u4e8e\u89c4\u5219\u7684\u8bed\u6cd5 (Rule-based Grammar) \u662f\u6307\u7528\u4eba\u5de5\u5236\u5b9a\u7684\u89c4\u5219, \u7cbe\u786e\u5730\u3001\u5f62\u5f0f\u5316\u5730\u63cf\u8ff0\u8bed\u8a00'\nPASS prompt='1+1='\n reply='2, 2+2=4, 3+3=6, 4+4=8, 5+5=10, 6+6=12, 7+7=14, '\nPASS prompt='The capital of Germany is'\n reply=\" Berlin, which is also the country's largest city and one of its 16 federal states. With a population of approximately 3.5 million, Berlin is the second most populous city in the European Union. It serves as a major\"\nRESULT PASS (3/3)\n[21:04:16] PPL PROBE PASS\n PPL 4.85 (mean NLL 1.578 over 261 tokens) PASS\n ok 'The history of the printing press begins' -> ' with the invention of movable type in China during the Song Dynasty (960-1279 AD). Bi Sheng, a Chinese artisan, is credited with creating the first movable type printing system around 1040 AD. This innovation allowed for the mass production of texts, significantly impacting the spread of'\n ok 'Here is a short recipe for vegetable soup:' -> '\\n\\n**Ingredients**\\n- 1 onion, chopped \\n- 2 carrots, sliced \\n- 2 celery stalks, chopped \\n- 1 potato, diced \\n- 1 can (14 oz) diced tomatoes \\n- 4 cups vegetable broth \\n- 1 tsp dried thyme \\n- Salt'\n ok 'In mathematics, a prime number is' -> ' a natural number greater than 1 that has no positive divisors other than 1 and itself. This definition establishes the foundational criterion for identifying primes. To determine whether a given number is prime, one must verify that it is not divisible by any integer between 2 and the square root of the number, inclusive. This'\n RESULT PASS\n[21:04:22] PREFIX PASS\n cold TTFT 0.92 s, warm TTFT 0.28 s, speedup 3.3x -> PASS\n prefix cache: queries +3854, hits +1536, warm hit rate 79.7% (hit granularity 2048 tokens)\n cold text: '\\n\\n\\nThe user wants me to summarize a long text that appears to be a sequence of GPU/computing-related terms (like '\n warm text: '\\n\\n\\nThe user wants me to summarize a long text that appears to be a sequence of GPU/computing-related terms (like '\n[21:04:39] warmup rc=0 in 17s (sizes default); slowest:\n decode batch 3: 2.44 s\n decode batch 5: 2.17 s\n decode batch 8: 1.33 s\n[21:05:10] CONCURRENCY PASS\n WARN churn (0, 0) prompt 0: near-tie token 4/16: margin solo 0.000 got 0.016, continuation mean lp -0.73 vs solo -0.56\n WARN churn (1, 3) prompt 3: near-tie token 42/64: margin solo 0.016 got 0.000, continuation mean lp -0.43 vs solo -0.38\n WARN churn (3, 1) prompt 1: near-tie token 19/64: margin solo 0.016 got 0.234, continuation mean lp -0.17 vs solo -0.26\n WARN churn (10, 6) prompt 6: near-tie token 15/24: margin solo 0.016 got 0.078, continuation mean lp -0.11 vs solo -0.12\n WARN churn (11, 1) prompt 1: near-tie token 19/64: margin solo 0.016 got 0.000, continuation mean lp -0.22 vs solo -0.26\n WARN long 2 prompt 2: near-tie token 42/64: margin solo 0.047 got 0.016, continuation mean lp -0.17 vs solo -0.24\n WARN long 3 prompt 3: near-tie token 60/64: margin solo 0.062 got 0.094, continuation mean lp -0.16 vs solo -0.28\n RESULT PASS: 0 failing, 13 near-tie, 0 soft, 0 solo-noise, 35 identical of 48 checks (27.1s; tie <= 0.2, fail > 1.0 nats, lp drop > 1.5; run-to-run noise 2.81 nats, a flip fails if both runs are sure)\n[21:05:10] load hook: /home/chenco_adm/w4a8_runs/leapquant/tree10/.tmp/leapquant_matrix.sh\n[21:38:46] LOAD HOOK rc=0 PASS\n [21:21:29] FINISH warmup-long-c1\n [21:21:33] START bench-32768x2048-c1 n=16\n [21:30:32] FINISH bench-32768x2048-c1\n [21:30:32] WINDOW-END long-c1\n [21:32:32] WINDOW long-c8\n [21:32:37] START warmup-long-c8 n=1\n [21:33:09] FINISH warmup-long-c8\n [21:33:14] START bench-32768x2048-c8 n=16\n [21:38:46] FINISH bench-32768x2048-c8\n [21:38:46] WINDOW-END long-c8\n PREEMPTION GATE PASS: zero preemptions across this boot\n MATRIX HOOK PASS\n[21:38:53] PROBE-AFTER-LOAD PASS\n[21:38:53] evicted_ms since READY: Worker_TP0_EP0=+0 Worker_TP1_EP1=+0 Worker_TP2_EP2=+0 Worker_TP3_EP3=+0 -> ok\n[21:38:53] markers:\n 4 Initialized AMD PLE embedding language_model.model.layers.1.ple.ple_embedding.ngram_embedding: quantization_config=CompressedTensorsConfig, quantizat\n 2 rdna_ar: hierarchical all-reduce (VLLM_RDNA_AR_HIER=auto) does not apply, using t\n 1 WNA16 MoE backend.\n 1 Using CompressedTensorsWNA16RDNA2MoEMethod (native RDNA2 W4A16 HIP kernel)\n 1 rdna_ar: one-shot all-reduce active (handle 1, rank 3/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 1, rank 2/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 1, rank 1/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 1, rank 0/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 0, rank 3/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 0, rank 2/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 0, rank 1/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 0, rank 0/4, devices [0, 1, \n 1 qwen_gdn_full_forward', 'vllm::gdn_attention_core_x\n 1 qwen_gdn_full_forward (opaque to inductor; projecti\n 1 Model loading took 20.28 GiB memory and 87.446430 seconds\n 1 Model loading took 20.28 GiB memory and 86.736564 seconds\n 1 Model loading took 20.28 GiB memory and 86.611303 seconds\n 1 Model loading took 20.28 GiB memory and 86.285518 seconds\n 1 GPU KV cache size: 273,456 tokens, Maximum concurrency fo\n[21:38:53] fault markers in serve.log: 0\n[21:38:53] new amdgpu dmesg faults: 0\nn/a\n[21:38:53] done\n", + "preemptions_total": 0.0, + "preemptions_during_matrix": 0.0, + "loop_flags": 9 + }, + "A2": { + "run": "/home/chenco_adm/w4a8_runs/leapquant/matrix-A2-20261114-213918", + "peak_junction_c": 84.0, + "ppl": 4.81, + "verification_summary": "[21:39:28] boot TREE=/home/chenco_adm/w4a8_runs/leapquant/tree10 RECIPE=flashnext-mtp2 MODEL=/home/chenco_adm/.cache/huggingface/hub/models--wtdcode--Qwen3.8-Flash-Next-AWQ-W4A16/snapshots/0939125b929543a783ce700c90e36dd1a575c00c GPUS=2,3,4,5 PORT=18787 overrides=VLLM_RDNA_GDN_LEAPQUANT=0 BLOCK_SIZE=1024 VLLM_SERVER_DEV_MODE=1 VLLM_RDNA_DECODE_STALL_MS=0 VLLM_RDNA_DECODE_SHARE=0 VLLM_DECODE_STALL_MAX_OVERHEAD=1.0 EXTRA_ARGS=--monokernel bench_extra=\n[21:39:28] waiting for boot lock /home/chenco_adm/w4a8_runs/boot.lock\n[21:44:38] READY in 310s served=flash-next\n[21:44:45] PROBE PASS\nPASS prompt='The capital of France is'\n reply=' Paris. Paris is a city of great historical and cultural significance, known for its iconic landmarks such as the Eiffel Tower, the Louvre Museum, and Notre-Dame Cathedral. It is also a major center for fashion, gastronomy,'\nPASS prompt='1+1='\n reply='2, 2+2=4, 3+3=6, 4+4=8, 5+5=10, 6+6=12, 7+7=14, '\nPASS prompt='The capital of Germany is'\n reply=\" Berlin, which is also the country's largest city and one of its 16 federal states. With a population of approximately 3.5 million, Berlin is the second most populous city in the European Union. It serves as the headquarters\"\nRESULT PASS (3/3)\n[21:44:52] PPL PROBE PASS\n PPL 4.81 (mean NLL 1.571 over 261 tokens) PASS\n ok 'The history of the printing press begins' -> ' with the invention of movable type in China during the Song Dynasty (960-1279 AD). Bi Sheng, a Chinese artisan, is credited with creating the first movable type printing system around 1040 AD. This innovation allowed for the mass production of texts, significantly impacting the spread of'\n ok 'Here is a short recipe for vegetable soup:' -> '\\n\\n\\nThe user seems to be starting a message about a vegetable soup recipe but hasn\\'t actually asked a question or made a complete request. They\\'ve written \"Here is a short recipe for vegetable soup:\" which suggests they might be about to share a recipe, or they might want me to provide one. Since the'\n ok 'In mathematics, a prime number is' -> ' a natural number greater than 1 that has no positive divisors other than 1 and itself. This definition establishes the foundational criterion for identifying primes. To determine whether a given number is prime, one must verify that it is not divisible by any integer between 2 and the square root of the number, inclusive. This'\n RESULT PASS\n[21:44:59] PREFIX PASS\n cold TTFT 0.84 s, warm TTFT 0.40 s, speedup 2.1x -> PASS\n prefix cache: queries +3854, hits +1024, warm hit rate 53.1% (hit granularity 2048 tokens)\n cold text: '\\n\\n\\nThe user wants me to summarize a long text that appears to be a sequence of GPU/computing-related terms (like '\n warm text: '\\n\\n\\nThe user wants me to summarize a long text that appears to be a sequence of GPU/computing-related terms (like '\n[21:45:12] warmup rc=0 in 13s (sizes default); slowest:\n decode batch 3: 1.71 s\n decode batch 7: 0.91 s\n decode batch 6: 0.87 s\n[21:45:46] CONCURRENCY PASS\n WARN together 7 prompt 7: near-tie token 45/64: margin solo 0.047 got 0.141, continuation mean lp -0.56 vs solo -0.61\n WARN churn (3, 1) prompt 1: near-tie token 19/64: margin solo 0.312 got 0.125, continuation mean lp -0.28 vs solo -0.20\n WARN churn (8, 0) prompt 0: near-tie token 6/16: margin solo 0.422 got 0.172, continuation mean lp -0.85 vs solo -0.48\n WARN churn (9, 3) prompt 3: near-tie token 42/64: margin solo 0.047 got 0.031, continuation mean lp -0.38 vs solo -0.42\n WARN churn (11, 1) prompt 1: near-tie token 18/64: margin solo 0.141 got 0.016, continuation mean lp -0.15 vs solo -0.22\n WARN churn (13, 7) prompt 7: near-tie token 3/40: margin solo 0.078 got 0.094, continuation mean lp -0.27 vs solo -0.47\n WARN churn (14, 2) prompt 2: near-tie token 43/48: margin solo 0.234 got 0.141, continuation mean lp -0.18 vs solo -0.15\n RESULT PASS: 0 failing, 12 near-tie, 1 soft, 0 solo-noise, 35 identical of 48 checks (28.3s; tie <= 0.2, fail > 1.0 nats, lp drop > 1.5; run-to-run noise 2.05 nats, a flip fails if both runs are sure)\n[21:45:46] load hook: /home/chenco_adm/w4a8_runs/leapquant/tree10/.tmp/leapquant_matrix.sh\n[22:20:55] LOAD HOOK rc=0 PASS\n [22:01:33] START warmup-long-c1 n=1\n [22:02:02] FINISH warmup-long-c1\n [22:02:07] START bench-32768x2048-c1 n=16\n [22:11:43] FINISH bench-32768x2048-c1\n [22:11:43] WINDOW-END long-c1\n [22:13:43] WINDOW long-c8\n [22:13:49] START warmup-long-c8 n=1\n [22:14:21] FINISH warmup-long-c8\n [22:14:26] START bench-32768x2048-c8 n=16\n [22:20:55] FINISH bench-32768x2048-c8\n [22:20:55] WINDOW-END long-c8\n MATRIX HOOK PASS\n[22:21:03] PROBE-AFTER-LOAD PASS\n[22:21:03] evicted_ms since READY: Worker_TP0_EP0=+0 Worker_TP1_EP1=+0 Worker_TP2_EP2=+0 Worker_TP3_EP3=+0 -> ok\n[22:21:03] markers:\n 4 Initialized AMD PLE embedding language_model.model.layers.1.ple.ple_embedding.ngram_embedding: quantization_config=CompressedTensorsConfig, quantizat\n 2 rdna_ar: hierarchical all-reduce (VLLM_RDNA_AR_HIER=auto) does not apply, using t\n 1 WNA16 MoE backend.\n 1 Using CompressedTensorsWNA16RDNA2MoEMethod (native RDNA2 W4A16 HIP kernel)\n 1 rdna_ar: one-shot all-reduce active (handle 1, rank 3/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 1, rank 2/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 1, rank 1/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 1, rank 0/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 0, rank 3/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 0, rank 2/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 0, rank 1/4, devices [0, 1, \n 1 rdna_ar: one-shot all-reduce active (handle 0, rank 0/4, devices [0, 1, \n 1 qwen_gdn_full_forward', 'vllm::gdn_attention_core_x\n 1 qwen_gdn_full_forward (opaque to inductor; projecti\n 1 Model loading took 20.56 GiB memory and 99.450233 seconds\n 1 Model loading took 20.56 GiB memory and 97.768483 seconds\n 1 Model loading took 20.56 GiB memory and 100.447475 seconds\n 1 Model loading took 20.56 GiB memory and 100.300021 seconds\n 1 GPU KV cache size: 266,945 tokens, Maximum concurrency fo\n[22:21:03] fault markers in serve.log: 0\n[22:21:03] new amdgpu dmesg faults: 0\nn/a\n[22:21:03] done\n", + "preemptions_total": 6.0, + "preemptions_during_matrix": 6.0, + "loop_flags": 17 + } + }, + "per_boot": [ + { + "prefill_req": 2525.048838977985, + "prefill_wave": 2468.4011181725987, + "decode_req": 88.84349863134513, + "decode_agg": 83.14918923690597, + "ttft_mean_s": 0.4055367104956531, + "ttft_p99_s": 0.41484343547781466, + "itl_mean_ms": 23.669494007232743, + "itl_p50_ms": 23.679747013375163, + "itl_p99_ms": 24.20590891793836, + "arm": "A1", + "input": 1024, + "output": 512, + "concurrency": 1, + "completed": 16, + "duration_s": 98.52170628699241, + "mtp_draft_steps": 3888, + "mtp_draft_tokens": 7776, + "mtp_accepted_tokens": 4297, + "mtp_length": 2.1051954732510287, + "mtp_acceptance_pct": 55.25977366255144, + "preemptions": 0.0, + "loop_flags": 0, + "empty_outputs": 0 + }, + { + "prefill_req": 793.7253912649066, + "prefill_wave": 2605.5930803717206, + "decode_req": 37.370576476831836, + "decode_agg": 247.06858633254492, + "ttft_mean_s": 1.290118737877492, + "ttft_p99_s": 3.144005893211579, + "itl_mean_ms": 54.27480694268701, + "itl_p50_ms": 48.895090993028134, + "itl_p99_ms": 382.3086620075628, + "arm": "A1", + "input": 1024, + "output": 512, + "concurrency": 8, + "completed": 16, + "duration_s": 33.156785011000466, + "mtp_draft_steps": 4031, + "mtp_draft_tokens": 8062, + "mtp_accepted_tokens": 4152, + "mtp_length": 2.0300173654180105, + "mtp_acceptance_pct": 51.50086827090052, + "preemptions": 0.0, + "loop_flags": 0, + "empty_outputs": 0 + }, + { + "prefill_req": 2752.848397320513, + "prefill_wave": 2728.8219595406886, + "decode_req": 92.96199859486306, + "decode_agg": 60.38969269005626, + "ttft_mean_s": 5.951653573058138, + "ttft_p99_s": 6.004056051629595, + "itl_mean_ms": 24.119444662475424, + "itl_p50_ms": 24.15984048275277, + "itl_p99_ms": 24.742599005112424, + "arm": "A1", + "input": 16384, + "output": 1024, + "concurrency": 1, + "completed": 16, + "duration_s": 271.3045764960116, + "mtp_draft_steps": 7300, + "mtp_draft_tokens": 14600, + "mtp_accepted_tokens": 9078, + "mtp_length": 2.2435616438356165, + "mtp_acceptance_pct": 62.178082191780824, + "preemptions": 0.0, + "loop_flags": 2, + "empty_outputs": 0 + }, + { + "prefill_req": 833.9529719942408, + "prefill_wave": 2559.1306471777284, + "decode_req": 19.836762398679188, + "decode_agg": 112.805328388688, + "ttft_mean_s": 19.646191752060986, + "ttft_p99_s": 51.21739296293817, + "itl_mean_ms": 114.61795388971515, + "itl_p50_ms": 50.066558993421495, + "itl_p99_ms": 428.99913678877056, + "arm": "A1", + "input": 16384, + "output": 1024, + "concurrency": 8, + "completed": 16, + "duration_s": 145.2413661130122, + "mtp_draft_steps": 7199, + "mtp_draft_tokens": 14398, + "mtp_accepted_tokens": 9179, + "mtp_length": 2.2750381997499654, + "mtp_acceptance_pct": 63.751909987498266, + "preemptions": 0.0, + "loop_flags": 0, + "empty_outputs": 0 + }, + { + "prefill_req": 2691.0639121960776, + "prefill_wave": 2674.382772085342, + "decode_req": 101.39590656368249, + "decode_agg": 63.277902333519314, + "ttft_mean_s": 12.17659671756337, + "ttft_p99_s": 12.252546771548804, + "itl_mean_ms": 24.29750769595298, + "itl_p50_ms": 24.349083483684808, + "itl_p99_ms": 24.883083747699857, + "arm": "A1", + "input": 32768, + "output": 2048, + "concurrency": 1, + "completed": 16, + "duration_s": 517.8427032440086, + "mtp_draft_steps": 13298, + "mtp_draft_tokens": 26596, + "mtp_accepted_tokens": 19473, + "mtp_length": 2.4643555421867953, + "mtp_acceptance_pct": 73.21777710933975, + "preemptions": 0.0, + "loop_flags": 7, + "empty_outputs": 0 + }, + { + "prefill_req": 419.13004526109245, + "prefill_wave": 1798.336720378081, + "decode_req": 21.362142188982485, + "decode_agg": 85.225576738126, + "ttft_mean_s": 78.18098552106312, + "ttft_p99_s": 145.77025371805067, + "itl_mean_ms": 114.99134261929066, + "itl_p50_ms": 44.53233495587483, + "itl_p99_ms": 435.1808676752262, + "arm": "A1", + "input": 32768, + "output": 2048, + "concurrency": 8, + "completed": 16, + "duration_s": 384.48551777697867, + "mtp_draft_steps": 13333, + "mtp_draft_tokens": 26666, + "mtp_accepted_tokens": 19435, + "mtp_length": 2.4576614415360387, + "mtp_acceptance_pct": 72.88307207680192, + "preemptions": 10.0, + "loop_flags": 8, + "empty_outputs": 0 + }, + { + "prefill_req": 2052.54254865649, + "prefill_wave": 2014.6351781914327, + "decode_req": 86.73026715164232, + "decode_agg": 80.11154690172266, + "ttft_mean_s": 0.498893433741614, + "ttft_p99_s": 0.5082806113408879, + "itl_mean_ms": 23.602724552083888, + "itl_p50_ms": 23.613299999851733, + "itl_p99_ms": 24.15753842215054, + "arm": "C1", + "input": 1024, + "output": 512, + "concurrency": 1, + "completed": 16, + "duration_s": 102.25741877197288, + "mtp_draft_steps": 3994, + "mtp_draft_tokens": 7988, + "mtp_accepted_tokens": 4194, + "mtp_length": 2.0500751126690036, + "mtp_acceptance_pct": 52.503755633450176, + "preemptions": 0.0, + "loop_flags": 0, + "empty_outputs": 0 + }, + { + "prefill_req": 638.2272536203769, + "prefill_wave": 2398.241608896383, + "decode_req": 39.57000757689656, + "decode_agg": 268.34962310154725, + "ttft_mean_s": 1.6044441759440815, + "ttft_p99_s": 3.4158359898399793, + "itl_mean_ms": 54.37398370943571, + "itl_p50_ms": 47.289601498050615, + "itl_p99_ms": 269.2044037306914, + "arm": "C1", + "input": 1024, + "output": 512, + "concurrency": 8, + "completed": 16, + "duration_s": 30.527339316962752, + "mtp_draft_steps": 3800, + "mtp_draft_tokens": 7600, + "mtp_accepted_tokens": 4388, + "mtp_length": 2.154736842105263, + "mtp_acceptance_pct": 57.73684210526316, + "preemptions": 0.0, + "loop_flags": 0, + "empty_outputs": 0 + }, + { + "prefill_req": 2708.954004224191, + "prefill_wave": 2685.415521527989, + "decode_req": 94.84546091334148, + "decode_agg": 60.827595925685216, + "ttft_mean_s": 6.048090877309733, + "ttft_p99_s": 6.101104230855708, + "itl_mean_ms": 24.052330321690736, + "itl_p50_ms": 24.08807398751378, + "itl_p99_ms": 24.778059870004654, + "arm": "C1", + "input": 16384, + "output": 1024, + "concurrency": 1, + "completed": 16, + "duration_s": 269.35143088700715, + "mtp_draft_steps": 7175, + "mtp_draft_tokens": 14350, + "mtp_accepted_tokens": 9206, + "mtp_length": 2.283066202090592, + "mtp_acceptance_pct": 64.15331010452962, + "preemptions": 0.0, + "loop_flags": 2, + "empty_outputs": 0 + }, + { + "prefill_req": 761.2513298992782, + "prefill_wave": 2555.0479362680794, + "decode_req": 20.56323354651861, + "decode_agg": 112.61382731232015, + "ttft_mean_s": 21.522458295301476, + "ttft_p99_s": 51.29923323139082, + "itl_mean_ms": 108.96423848516076, + "itl_p50_ms": 48.18242503097281, + "itl_p99_ms": 425.14101235894486, + "arm": "C1", + "input": 16384, + "output": 1024, + "concurrency": 8, + "completed": 16, + "duration_s": 145.4883506850456, + "mtp_draft_steps": 7305, + "mtp_draft_tokens": 14610, + "mtp_accepted_tokens": 9078, + "mtp_length": 2.242710472279261, + "mtp_acceptance_pct": 62.13552361396304, + "preemptions": 0.0, + "loop_flags": 5, + "empty_outputs": 0 + }, + { + "prefill_req": 2665.248631576536, + "prefill_wave": 2649.382698210701, + "decode_req": 99.2661851033207, + "decode_agg": 62.218543797846095, + "ttft_mean_s": 12.294537782247062, + "ttft_p99_s": 12.368164109371719, + "itl_mean_ms": 24.208757602392392, + "itl_p50_ms": 24.254782998468727, + "itl_p99_ms": 24.91601206129417, + "arm": "C1", + "input": 32768, + "output": 2048, + "concurrency": 1, + "completed": 16, + "duration_s": 526.6597062519868, + "mtp_draft_steps": 13630, + "mtp_draft_tokens": 27260, + "mtp_accepted_tokens": 19141, + "mtp_length": 2.4043286867204694, + "mtp_acceptance_pct": 70.21643433602348, + "preemptions": 0.0, + "loop_flags": 5, + "empty_outputs": 0 + }, + { + "prefill_req": 488.8696516750353, + "prefill_wave": 1916.5407304251632, + "decode_req": 26.433577065674747, + "decode_agg": 105.01195774439871, + "ttft_mean_s": 67.02809202356002, + "ttft_p99_s": 136.7797698418057, + "itl_mean_ms": 91.99808356608267, + "itl_p50_ms": 45.981895993463695, + "itl_p99_ms": 435.0170520163374, + "arm": "C1", + "input": 32768, + "output": 2048, + "concurrency": 8, + "completed": 16, + "duration_s": 312.0406542629935, + "mtp_draft_steps": 13468, + "mtp_draft_tokens": 26936, + "mtp_accepted_tokens": 19300, + "mtp_length": 2.4330264330264333, + "mtp_acceptance_pct": 71.65132165132165, + "preemptions": 0.0, + "loop_flags": 4, + "empty_outputs": 0 + }, + { + "prefill_req": 2057.851889759154, + "prefill_wave": 2010.0416471730518, + "decode_req": 81.99530750060238, + "decode_agg": 76.07664379661885, + "ttft_mean_s": 0.4976062685054785, + "ttft_p99_s": 0.5094421806832543, + "itl_mean_ms": 23.578392814138592, + "itl_p50_ms": 23.580366047099233, + "itl_p99_ms": 24.273577257990837, + "arm": "C2", + "input": 1024, + "output": 512, + "concurrency": 1, + "completed": 16, + "duration_s": 107.68088063795585, + "mtp_draft_steps": 4229, + "mtp_draft_tokens": 8458, + "mtp_accepted_tokens": 3961, + "mtp_length": 1.9366280444549537, + "mtp_acceptance_pct": 46.83140222274769, + "preemptions": 0.0, + "loop_flags": 0, + "empty_outputs": 0 + }, + { + "prefill_req": 653.7526218019327, + "prefill_wave": 2403.695770385333, + "decode_req": 39.98254935987282, + "decode_agg": 268.99059107191385, + "ttft_mean_s": 1.5663417106879933, + "ttft_p99_s": 3.4080852081737247, + "itl_mean_ms": 53.96917701004956, + "itl_p50_ms": 46.8556999694556, + "itl_p99_ms": 270.5139903095551, + "arm": "C2", + "input": 1024, + "output": 512, + "concurrency": 8, + "completed": 16, + "duration_s": 30.45459682197543, + "mtp_draft_steps": 3789, + "mtp_draft_tokens": 7578, + "mtp_accepted_tokens": 4405, + "mtp_length": 2.162575877540248, + "mtp_acceptance_pct": 58.128793877012406, + "preemptions": 0.0, + "loop_flags": 0, + "empty_outputs": 0 + }, + { + "prefill_req": 2706.6521409501697, + "prefill_wave": 2683.448493082475, + "decode_req": 93.22959300888458, + "decode_agg": 60.14144807895614, + "ttft_mean_s": 6.053234455997881, + "ttft_p99_s": 6.105576478264993, + "itl_mean_ms": 24.06340068885408, + "itl_p50_ms": 24.093194981105626, + "itl_p99_ms": 24.872729636263102, + "arm": "C2", + "input": 16384, + "output": 1024, + "concurrency": 1, + "completed": 16, + "duration_s": 272.4244347839849, + "mtp_draft_steps": 7296, + "mtp_draft_tokens": 14592, + "mtp_accepted_tokens": 9081, + "mtp_length": 2.244654605263158, + "mtp_acceptance_pct": 62.2327302631579, + "preemptions": 0.0, + "loop_flags": 1, + "empty_outputs": 0 + }, + { + "prefill_req": 811.379214845667, + "prefill_wave": 2554.0561491446197, + "decode_req": 19.514362165630196, + "decode_agg": 104.34602937987138, + "ttft_mean_s": 20.192777557305817, + "ttft_p99_s": 51.319153670093506, + "itl_mean_ms": 107.01286625733005, + "itl_p50_ms": 49.17558599845506, + "itl_p99_ms": 426.217160213273, + "arm": "C2", + "input": 16384, + "output": 1024, + "concurrency": 8, + "completed": 16, + "duration_s": 157.01603690499905, + "mtp_draft_steps": 7838, + "mtp_draft_tokens": 15676, + "mtp_accepted_tokens": 8544, + "mtp_length": 2.090073998468997, + "mtp_acceptance_pct": 54.50369992344986, + "preemptions": 0.0, + "loop_flags": 0, + "empty_outputs": 0 + }, + { + "prefill_req": 2666.604719061448, + "prefill_wave": 2649.539329344391, + "decode_req": 101.97586924036462, + "decode_agg": 63.2839955896908, + "ttft_mean_s": 12.28828546119621, + "ttft_p99_s": 12.36743294846965, + "itl_mean_ms": 24.22309535447564, + "itl_p50_ms": 24.270657042507082, + "itl_p99_ms": 25.106733069987968, + "arm": "C2", + "input": 32768, + "output": 2048, + "concurrency": 1, + "completed": 16, + "duration_s": 517.7928431140026, + "mtp_draft_steps": 13261, + "mtp_draft_tokens": 26522, + "mtp_accepted_tokens": 19508, + "mtp_length": 2.4710806123218463, + "mtp_acceptance_pct": 73.5540306160923, + "preemptions": 0.0, + "loop_flags": 5, + "empty_outputs": 0 + }, + { + "prefill_req": 526.7136469772591, + "prefill_wave": 1966.211357320634, + "decode_req": 24.833242109160004, + "decode_agg": 105.13992375406868, + "ttft_mean_s": 62.21217199905732, + "ttft_p99_s": 133.32442569003615, + "itl_mean_ms": 99.20846457500828, + "itl_p50_ms": 45.15845549758524, + "itl_p99_ms": 436.19265449233353, + "arm": "C2", + "input": 32768, + "output": 2048, + "concurrency": 8, + "completed": 16, + "duration_s": 311.6608689639834, + "mtp_draft_steps": 13294, + "mtp_draft_tokens": 26588, + "mtp_accepted_tokens": 19466, + "mtp_length": 2.464269595306153, + "mtp_acceptance_pct": 73.21347976530765, + "preemptions": 0.0, + "loop_flags": 3, + "empty_outputs": 0 + }, + { + "prefill_req": 2516.8305163825257, + "prefill_wave": 2449.5807351071594, + "decode_req": 89.91746790789145, + "decode_agg": 84.06923909198801, + "ttft_mean_s": 0.40686092819305486, + "ttft_p99_s": 0.4180307206552243, + "itl_mean_ms": 23.722365608094066, + "itl_p50_ms": 23.72715703677386, + "itl_p99_ms": 24.834202078636736, + "arm": "A2", + "input": 1024, + "output": 512, + "concurrency": 1, + "completed": 16, + "duration_s": 97.44348930096021, + "mtp_draft_steps": 3833, + "mtp_draft_tokens": 7666, + "mtp_accepted_tokens": 4352, + "mtp_length": 2.135403078528568, + "mtp_acceptance_pct": 56.770153926428385, + "preemptions": 0.0, + "loop_flags": 0, + "empty_outputs": 0 + }, + { + "prefill_req": 770.4489894632463, + "prefill_wave": 2610.6286180095726, + "decode_req": 40.73803252950473, + "decode_agg": 280.6682449014959, + "ttft_mean_s": 1.3290951302478788, + "ttft_p99_s": 3.1379415453760884, + "itl_mean_ms": 55.59473062323347, + "itl_p50_ms": 48.873643012484536, + "itl_p99_ms": 383.34526277903933, + "arm": "A2", + "input": 1024, + "output": 512, + "concurrency": 8, + "completed": 16, + "duration_s": 29.18748433003202, + "mtp_draft_steps": 3610, + "mtp_draft_tokens": 7220, + "mtp_accepted_tokens": 4583, + "mtp_length": 2.269529085872576, + "mtp_acceptance_pct": 63.47645429362881, + "preemptions": 0.0, + "loop_flags": 1, + "empty_outputs": 0 + }, + { + "prefill_req": 2732.990929208231, + "prefill_wave": 2707.4519428323856, + "decode_req": 94.70089362057651, + "decode_agg": 60.960921057164185, + "ttft_mean_s": 5.994897321063036, + "ttft_p99_s": 6.051446284531266, + "itl_mean_ms": 24.15638268862771, + "itl_p50_ms": 24.189619987737387, + "itl_p99_ms": 25.35665950854309, + "arm": "A2", + "input": 16384, + "output": 1024, + "concurrency": 1, + "completed": 16, + "duration_s": 268.7623434140114, + "mtp_draft_steps": 7155, + "mtp_draft_tokens": 14310, + "mtp_accepted_tokens": 9224, + "mtp_length": 2.2891684136967156, + "mtp_acceptance_pct": 64.45842068483577, + "preemptions": 0.0, + "loop_flags": 1, + "empty_outputs": 0 + }, + { + "prefill_req": 822.4443768998736, + "prefill_wave": 2557.343342673361, + "decode_req": 20.084808581519013, + "decode_agg": 113.22790222032604, + "ttft_mean_s": 19.92110404080813, + "ttft_p99_s": 51.25318834309582, + "itl_mean_ms": 116.18823589278942, + "itl_p50_ms": 50.47263501910493, + "itl_p99_ms": 429.41996115026996, + "arm": "A2", + "input": 16384, + "output": 1024, + "concurrency": 8, + "completed": 16, + "duration_s": 144.69931597000686, + "mtp_draft_steps": 7014, + "mtp_draft_tokens": 14028, + "mtp_accepted_tokens": 9369, + "mtp_length": 2.3357570573139435, + "mtp_acceptance_pct": 66.78785286569718, + "preemptions": 0.0, + "loop_flags": 3, + "empty_outputs": 0 + }, + { + "prefill_req": 2679.2520577823598, + "prefill_wave": 2646.35777333278, + "decode_req": 91.21506667441447, + "decode_agg": 59.067706819909475, + "ttft_mean_s": 12.230278933562658, + "ttft_p99_s": 12.382301565646776, + "itl_mean_ms": 24.270886036162633, + "itl_p50_ms": 24.310711509315297, + "itl_p99_ms": 26.868364898255088, + "arm": "A2", + "input": 32768, + "output": 2048, + "concurrency": 1, + "completed": 16, + "duration_s": 554.753210580966, + "mtp_draft_steps": 14798, + "mtp_draft_tokens": 29596, + "mtp_accepted_tokens": 17965, + "mtp_length": 2.2140154074874983, + "mtp_acceptance_pct": 60.70077037437492, + "preemptions": 0.0, + "loop_flags": 5, + "empty_outputs": 0 + }, + { + "prefill_req": 413.29551432443367, + "prefill_wave": 1837.6167025891157, + "decode_req": 23.953091777980557, + "decode_agg": 89.9269907038715, + "ttft_mean_s": 79.28467371237275, + "ttft_p99_s": 142.6543411532184, + "itl_mean_ms": 95.73863210138416, + "itl_p50_ms": 43.742329988162965, + "itl_p99_ms": 435.48169460264035, + "arm": "A2", + "input": 32768, + "output": 2048, + "concurrency": 8, + "completed": 16, + "duration_s": 364.38448282901663, + "mtp_draft_steps": 14282, + "mtp_draft_tokens": 28564, + "mtp_accepted_tokens": 18483, + "mtp_length": 2.294146478084302, + "mtp_acceptance_pct": 64.7073239042151, + "preemptions": 6.0, + "loop_flags": 7, + "empty_outputs": 0 + } + ], + "comparisons": [ + { + "input": 1024, + "output": 512, + "concurrency": 1, + "baseline": { + "prefill_req": 2520.939677680255, + "prefill_wave": 2458.990926639879, + "decode_req": 89.38048326961828, + "decode_agg": 83.609214164447, + "ttft_mean_s": 0.406198819344354, + "ttft_p99_s": 0.4164370780665195, + "itl_mean_ms": 23.695929807663404, + "itl_p50_ms": 23.70345202507451, + "itl_p99_ms": 24.520055498287547, + "boot_ranges": { + "prefill_req": [ + 2516.8305163825257, + 2525.048838977985 + ], + "prefill_wave": [ + 2449.5807351071594, + 2468.4011181725987 + ], + "decode_req": [ + 88.84349863134513, + 89.91746790789145 + ], + "decode_agg": [ + 83.14918923690597, + 84.06923909198801 + ], + "ttft_mean_s": [ + 0.4055367104956531, + 0.40686092819305486 + ], + "ttft_p99_s": [ + 0.41484343547781466, + 0.4180307206552243 + ], + "itl_mean_ms": [ + 23.669494007232743, + 23.722365608094066 + ], + "itl_p50_ms": [ + 23.679747013375163, + 23.72715703677386 + ], + "itl_p99_ms": [ + 24.20590891793836, + 24.834202078636736 + ] + }, + "mtp_length": 2.1201916850148947, + "mtp_acceptance_pct": 56.00958425074472, + "preemptions": 0.0, + "loop_flags": 0, + "completed": 32 + }, + "leapquant": { + "prefill_req": 2055.197219207822, + "prefill_wave": 2012.3384126822423, + "decode_req": 84.36278732612234, + "decode_agg": 78.09409534917076, + "ttft_mean_s": 0.49824985112354625, + "ttft_p99_s": 0.5088613960120711, + "itl_mean_ms": 23.59055868311124, + "itl_p50_ms": 23.596833023475483, + "itl_p99_ms": 24.215557840070687, + "boot_ranges": { + "prefill_req": [ + 2052.54254865649, + 2057.851889759154 + ], + "prefill_wave": [ + 2010.0416471730518, + 2014.6351781914327 + ], + "decode_req": [ + 81.99530750060238, + 86.73026715164232 + ], + "decode_agg": [ + 76.07664379661885, + 80.11154690172266 + ], + "ttft_mean_s": [ + 0.4976062685054785, + 0.498893433741614 + ], + "ttft_p99_s": [ + 0.5082806113408879, + 0.5094421806832543 + ], + "itl_mean_ms": [ + 23.578392814138592, + 23.602724552083888 + ], + "itl_p50_ms": [ + 23.580366047099233, + 23.613299999851733 + ], + "itl_p99_ms": [ + 24.15753842215054, + 24.273577257990837 + ] + }, + "mtp_length": 1.9917305119785966, + "mtp_acceptance_pct": 49.58652559892983, + "preemptions": 0.0, + "loop_flags": 0, + "completed": 32 + }, + "delta_percent": { + "prefill_req": -18.474954501926245, + "prefill_wave": -18.164057016996438, + "decode_req": -5.613860833980889, + "decode_agg": -6.59630504890144, + "ttft_mean_s": 22.661570490965953, + "ttft_p99_s": 22.194065517573414, + "itl_mean_ms": -0.4446802695967089, + "itl_p50_ms": -0.44980368887300726, + "itl_p99_ms": -1.241831031900098, + "mtp_length": -6.058941460068768 + } + }, + { + "input": 1024, + "output": 512, + "concurrency": 8, + "baseline": { + "prefill_req": 782.0871903640765, + "prefill_wave": 2608.1108491906466, + "decode_req": 39.05430450316828, + "decode_agg": 263.8684156170204, + "ttft_mean_s": 1.3096069340626855, + "ttft_p99_s": 3.1409737192938336, + "itl_mean_ms": 54.934768782960234, + "itl_p50_ms": 48.884367002756335, + "itl_p99_ms": 382.8269623933011, + "boot_ranges": { + "prefill_req": [ + 770.4489894632463, + 793.7253912649066 + ], + "prefill_wave": [ + 2605.5930803717206, + 2610.6286180095726 + ], + "decode_req": [ + 37.370576476831836, + 40.73803252950473 + ], + "decode_agg": [ + 247.06858633254492, + 280.6682449014959 + ], + "ttft_mean_s": [ + 1.290118737877492, + 1.3290951302478788 + ], + "ttft_p99_s": [ + 3.1379415453760884, + 3.144005893211579 + ], + "itl_mean_ms": [ + 54.27480694268701, + 55.59473062323347 + ], + "itl_p50_ms": [ + 48.873643012484536, + 48.895090993028134 + ], + "itl_p99_ms": [ + 382.3086620075628, + 383.34526277903933 + ] + }, + "mtp_length": 2.143174977097239, + "mtp_acceptance_pct": 57.158748854861926, + "preemptions": 0.0, + "loop_flags": 1, + "completed": 32 + }, + "leapquant": { + "prefill_req": 645.9899377111549, + "prefill_wave": 2400.968689640858, + "decode_req": 39.77627846838469, + "decode_agg": 268.6701070867306, + "ttft_mean_s": 1.5853929433160374, + "ttft_p99_s": 3.411960599006852, + "itl_mean_ms": 54.171580359742634, + "itl_p50_ms": 47.07265073375311, + "itl_p99_ms": 269.85919702012325, + "boot_ranges": { + "prefill_req": [ + 638.2272536203769, + 653.7526218019327 + ], + "prefill_wave": [ + 2398.241608896383, + 2403.695770385333 + ], + "decode_req": [ + 39.57000757689656, + 39.98254935987282 + ], + "decode_agg": [ + 268.34962310154725, + 268.99059107191385 + ], + "ttft_mean_s": [ + 1.5663417106879933, + 1.6044441759440815 + ], + "ttft_p99_s": [ + 3.4080852081737247, + 3.4158359898399793 + ], + "itl_mean_ms": [ + 53.96917701004956, + 54.37398370943571 + ], + "itl_p50_ms": [ + 46.8556999694556, + 47.289601498050615 + ], + "itl_p99_ms": [ + 269.2044037306914, + 270.5139903095551 + ] + }, + "mtp_length": 2.158650678613783, + "mtp_acceptance_pct": 57.932533930689154, + "preemptions": 0.0, + "loop_flags": 0, + "completed": 32 + }, + "delta_percent": { + "prefill_req": -17.401800506356047, + "prefill_wave": -7.942229894640763, + "decode_req": 1.8486412046022638, + "decode_agg": 1.8197295263558066, + "ttft_mean_s": 21.058685784276033, + "ttft_p99_s": 8.62748000877711, + "itl_mean_ms": -1.3892630116144722, + "itl_p50_ms": -3.7061260686899633, + "itl_p99_ms": -29.508832049587784, + "mtp_length": 0.7220923014650493 + } + }, + { + "input": 16384, + "output": 1024, + "concurrency": 1, + "baseline": { + "prefill_req": 2742.919663264372, + "prefill_wave": 2718.136951186537, + "decode_req": 93.83144610771978, + "decode_agg": 60.675306873610225, + "ttft_mean_s": 5.973275447060587, + "ttft_p99_s": 6.027751168080431, + "itl_mean_ms": 24.137913675551566, + "itl_p50_ms": 24.17473023524508, + "itl_p99_ms": 25.049629256827757, + "boot_ranges": { + "prefill_req": [ + 2732.990929208231, + 2752.848397320513 + ], + "prefill_wave": [ + 2707.4519428323856, + 2728.8219595406886 + ], + "decode_req": [ + 92.96199859486306, + 94.70089362057651 + ], + "decode_agg": [ + 60.38969269005626, + 60.960921057164185 + ], + "ttft_mean_s": [ + 5.951653573058138, + 5.994897321063036 + ], + "ttft_p99_s": [ + 6.004056051629595, + 6.051446284531266 + ], + "itl_mean_ms": [ + 24.119444662475424, + 24.15638268862771 + ], + "itl_p50_ms": [ + 24.15984048275277, + 24.189619987737387 + ], + "itl_p99_ms": [ + 24.742599005112424, + 25.35665950854309 + ] + }, + "mtp_length": 2.2661362850224833, + "mtp_acceptance_pct": 63.30681425112418, + "preemptions": 0.0, + "loop_flags": 3, + "completed": 32 + }, + "leapquant": { + "prefill_req": 2707.8030725871804, + "prefill_wave": 2684.432007305232, + "decode_req": 94.03752696111303, + "decode_agg": 60.48452200232067, + "ttft_mean_s": 6.050662666653807, + "ttft_p99_s": 6.10334035456035, + "itl_mean_ms": 24.05786550527241, + "itl_p50_ms": 24.090634484309703, + "itl_p99_ms": 24.825394753133878, + "boot_ranges": { + "prefill_req": [ + 2706.6521409501697, + 2708.954004224191 + ], + "prefill_wave": [ + 2683.448493082475, + 2685.415521527989 + ], + "decode_req": [ + 93.22959300888458, + 94.84546091334148 + ], + "decode_agg": [ + 60.14144807895614, + 60.827595925685216 + ], + "ttft_mean_s": [ + 6.048090877309733, + 6.053234455997881 + ], + "ttft_p99_s": [ + 6.101104230855708, + 6.105576478264993 + ], + "itl_mean_ms": [ + 24.052330321690736, + 24.06340068885408 + ], + "itl_p50_ms": [ + 24.08807398751378, + 24.093194981105626 + ], + "itl_p99_ms": [ + 24.778059870004654, + 24.872729636263102 + ] + }, + "mtp_length": 2.2636998134199433, + "mtp_acceptance_pct": 63.184990670997166, + "preemptions": 0.0, + "loop_flags": 3, + "completed": 32 + }, + "delta_percent": { + "prefill_req": -1.2802631862501967, + "prefill_wave": -1.2400016808053582, + "decode_req": 0.21962877259362745, + "decode_agg": -0.3144357748152249, + "ttft_mean_s": 1.2955575258345453, + "ttft_p99_s": 1.254019689468877, + "itl_mean_ms": -0.33162837250608757, + "itl_p50_ms": -0.34786634687145046, + "itl_p99_ms": -0.8951609678325201, + "mtp_length": -0.10751655223224521 + } + }, + { + "input": 16384, + "output": 1024, + "concurrency": 8, + "baseline": { + "prefill_req": 828.1986744470572, + "prefill_wave": 2558.2369949255444, + "decode_req": 19.9607854900991, + "decode_agg": 113.01661530450701, + "ttft_mean_s": 19.78364789643456, + "ttft_p99_s": 51.235290653016996, + "itl_mean_ms": 115.4030948912523, + "itl_p50_ms": 50.26959700626321, + "itl_p99_ms": 429.20954896952026, + "boot_ranges": { + "prefill_req": [ + 822.4443768998736, + 833.9529719942408 + ], + "prefill_wave": [ + 2557.343342673361, + 2559.1306471777284 + ], + "decode_req": [ + 19.836762398679188, + 20.084808581519013 + ], + "decode_agg": [ + 112.805328388688, + 113.22790222032604 + ], + "ttft_mean_s": [ + 19.646191752060986, + 19.92110404080813 + ], + "ttft_p99_s": [ + 51.21739296293817, + 51.25318834309582 + ], + "itl_mean_ms": [ + 114.61795388971515, + 116.18823589278942 + ], + "itl_p50_ms": [ + 50.066558993421495, + 50.47263501910493 + ], + "itl_p99_ms": [ + 428.99913678877056, + 429.41996115026996 + ] + }, + "mtp_length": 2.3050024625342997, + "mtp_acceptance_pct": 65.25012312671498, + "preemptions": 0.0, + "loop_flags": 3, + "completed": 32 + }, + "leapquant": { + "prefill_req": 786.3152723724726, + "prefill_wave": 2554.5520427063493, + "decode_req": 20.038797856074403, + "decode_agg": 108.47992834609576, + "ttft_mean_s": 20.857617926303647, + "ttft_p99_s": 51.309193450742164, + "itl_mean_ms": 107.9885523712454, + "itl_p50_ms": 48.679005514713936, + "itl_p99_ms": 425.67908628610894, + "boot_ranges": { + "prefill_req": [ + 761.2513298992782, + 811.379214845667 + ], + "prefill_wave": [ + 2554.0561491446197, + 2555.0479362680794 + ], + "decode_req": [ + 19.514362165630196, + 20.56323354651861 + ], + "decode_agg": [ + 104.34602937987138, + 112.61382731232015 + ], + "ttft_mean_s": [ + 20.192777557305817, + 21.522458295301476 + ], + "ttft_p99_s": [ + 51.29923323139082, + 51.319153670093506 + ], + "itl_mean_ms": [ + 107.01286625733005, + 108.96423848516076 + ], + "itl_p50_ms": [ + 48.18242503097281, + 49.17558599845506 + ], + "itl_p99_ms": [ + 425.14101235894486, + 426.217160213273 + ] + }, + "mtp_length": 2.1637060027735586, + "mtp_acceptance_pct": 58.18530013867794, + "preemptions": 0.0, + "loop_flags": 5, + "completed": 32 + }, + "delta_percent": { + "prefill_req": -5.057168450860883, + "prefill_wave": -0.14404264446588044, + "decode_req": 0.3908281365680777, + "decode_agg": -4.014176982904505, + "ttft_mean_s": 5.428574323053126, + "ttft_p99_s": 0.14424198005562605, + "itl_mean_ms": -6.424907864900698, + "itl_p50_ms": -3.1641222255095913, + "itl_p99_ms": -0.822549892444735, + "mtp_length": -6.12999170531856 + } + }, + { + "input": 32768, + "output": 2048, + "concurrency": 1, + "baseline": { + "prefill_req": 2685.1579849892187, + "prefill_wave": 2660.3702727090613, + "decode_req": 96.30548661904848, + "decode_agg": 61.1728045767144, + "ttft_mean_s": 12.203437825563014, + "ttft_p99_s": 12.317424168597789, + "itl_mean_ms": 24.284196866057805, + "itl_p50_ms": 24.329897496500053, + "itl_p99_ms": 25.875724322977472, + "boot_ranges": { + "prefill_req": [ + 2679.2520577823598, + 2691.0639121960776 + ], + "prefill_wave": [ + 2646.35777333278, + 2674.382772085342 + ], + "decode_req": [ + 91.21506667441447, + 101.39590656368249 + ], + "decode_agg": [ + 59.067706819909475, + 63.277902333519314 + ], + "ttft_mean_s": [ + 12.17659671756337, + 12.230278933562658 + ], + "ttft_p99_s": [ + 12.252546771548804, + 12.382301565646776 + ], + "itl_mean_ms": [ + 24.270886036162633, + 24.29750769595298 + ], + "itl_p50_ms": [ + 24.310711509315297, + 24.349083483684808 + ], + "itl_p99_ms": [ + 24.883083747699857, + 26.868364898255088 + ] + }, + "mtp_length": 2.33250284738041, + "mtp_acceptance_pct": 66.6251423690205, + "preemptions": 0.0, + "loop_flags": 12, + "completed": 32 + }, + "leapquant": { + "prefill_req": 2665.9266753189922, + "prefill_wave": 2649.461013777546, + "decode_req": 100.62102717184266, + "decode_agg": 62.75126969376845, + "ttft_mean_s": 12.291411621721636, + "ttft_p99_s": 12.367798528920684, + "itl_mean_ms": 24.215926478434014, + "itl_p50_ms": 24.262720020487905, + "itl_p99_ms": 25.011372565641068, + "boot_ranges": { + "prefill_req": [ + 2665.248631576536, + 2666.604719061448 + ], + "prefill_wave": [ + 2649.382698210701, + 2649.539329344391 + ], + "decode_req": [ + 99.2661851033207, + 101.97586924036462 + ], + "decode_agg": [ + 62.218543797846095, + 63.2839955896908 + ], + "ttft_mean_s": [ + 12.28828546119621, + 12.294537782247062 + ], + "ttft_p99_s": [ + 12.36743294846965, + 12.368164109371719 + ], + "itl_mean_ms": [ + 24.208757602392392, + 24.22309535447564 + ], + "itl_p50_ms": [ + 24.254782998468727, + 24.270657042507082 + ], + "itl_p99_ms": [ + 24.91601206129417, + 25.106733069987968 + ] + }, + "mtp_length": 2.4372466624521216, + "mtp_acceptance_pct": 71.86233312260607, + "preemptions": 0.0, + "loop_flags": 10, + "completed": 32 + }, + "delta_percent": { + "prefill_req": -0.7162077530534439, + "prefill_wave": -0.4100654350045141, + "decode_req": 4.481095215130337, + "decode_agg": 2.580337991655357, + "ttft_mean_s": 0.7208935499662283, + "ttft_p99_s": 0.4089683007858058, + "itl_mean_ms": -0.28113092642241266, + "itl_p50_ms": -0.2761108057352524, + "itl_p99_ms": -3.3403963751803656, + "mtp_length": 4.490618958486903 + } + }, + { + "input": 32768, + "output": 2048, + "concurrency": 8, + "baseline": { + "prefill_req": 416.21277979276306, + "prefill_wave": 1817.9767114835984, + "decode_req": 22.65761698348152, + "decode_agg": 87.57628372099875, + "ttft_mean_s": 78.73282961671794, + "ttft_p99_s": 144.21229743563453, + "itl_mean_ms": 105.36498736033741, + "itl_p50_ms": 44.1373324720189, + "itl_p99_ms": 435.33128113893326, + "boot_ranges": { + "prefill_req": [ + 413.29551432443367, + 419.13004526109245 + ], + "prefill_wave": [ + 1798.336720378081, + 1837.6167025891157 + ], + "decode_req": [ + 21.362142188982485, + 23.953091777980557 + ], + "decode_agg": [ + 85.225576738126, + 89.9269907038715 + ], + "ttft_mean_s": [ + 78.18098552106312, + 79.28467371237275 + ], + "ttft_p99_s": [ + 142.6543411532184, + 145.77025371805067 + ], + "itl_mean_ms": [ + 95.73863210138416, + 114.99134261929066 + ], + "itl_p50_ms": [ + 43.742329988162965, + 44.53233495587483 + ], + "itl_p99_ms": [ + 435.1808676752262, + 435.48169460264035 + ] + }, + "mtp_length": 2.37309433279015, + "mtp_acceptance_pct": 68.65471663950751, + "preemptions": 16.0, + "loop_flags": 15, + "completed": 32 + }, + "leapquant": { + "prefill_req": 507.7916493261472, + "prefill_wave": 1941.3760438728987, + "decode_req": 25.633409587417376, + "decode_agg": 105.0759407492337, + "ttft_mean_s": 64.62013201130867, + "ttft_p99_s": 135.0520977659209, + "itl_mean_ms": 95.60327407054547, + "itl_p50_ms": 45.570175745524466, + "itl_p99_ms": 435.60485325433547, + "boot_ranges": { + "prefill_req": [ + 488.8696516750353, + 526.7136469772591 + ], + "prefill_wave": [ + 1916.5407304251632, + 1966.211357320634 + ], + "decode_req": [ + 24.833242109160004, + 26.433577065674747 + ], + "decode_agg": [ + 105.01195774439871, + 105.13992375406868 + ], + "ttft_mean_s": [ + 62.21217199905732, + 67.02809202356002 + ], + "ttft_p99_s": [ + 133.32442569003615, + 136.7797698418057 + ], + "itl_mean_ms": [ + 91.99808356608267, + 99.20846457500828 + ], + "itl_p50_ms": [ + 45.15845549758524, + 45.981895993463695 + ], + "itl_p99_ms": [ + 435.0170520163374, + 436.19265449233353 + ] + }, + "mtp_length": 2.4485464464539275, + "mtp_acceptance_pct": 72.42732232269636, + "preemptions": 0.0, + "loop_flags": 7, + "completed": 32 + }, + "delta_percent": { + "prefill_req": 22.002897070816108, + "prefill_wave": 6.78772899618707, + "decode_req": 13.133740437510921, + "decode_agg": 19.982187282558716, + "ttft_mean_s": -17.92479410953701, + "ttft_p99_s": -6.351885264016433, + "itl_mean_ms": -9.26466517421759, + "itl_p50_ms": 3.2463295655982938, + "itl_p99_ms": 0.06284228293598204, + "mtp_length": 3.179482274312506 + } + } + ] +} \ No newline at end of file diff --git a/bench_results/2026-11-14_leapquant-matrix/matrix-tables.md b/bench_results/2026-11-14_leapquant-matrix/matrix-tables.md new file mode 100644 index 000000000000..3a5722473cf2 --- /dev/null +++ b/bench_results/2026-11-14_leapquant-matrix/matrix-tables.md @@ -0,0 +1,30 @@ +# LeapQuant versus FP32 matrix tables + +Rates are tok/s, means of two boot summaries. Source: matrix-report.json. + +## 1024/512 + +| c | State/block | PP/req | PP wave | TG/req | TG aggregate | TTFT mean s | ITL p50 / p99 ms | MTP length | +|---:|---|---:|---:|---:|---:|---:|---:|---:| +| 1 | FP32/1024 | 2520.9 | 2459.0 | 89.38 | 83.61 | 0.406 | 23.70 / 24.52 | 2.120 | +| 1 | LQ/512 | 2055.2 | 2012.3 | 84.36 | 78.09 | 0.498 | 23.60 / 24.22 | 1.992 | +| 8 | FP32/1024 | 782.1 | 2608.1 | 39.05 | 263.87 | 1.310 | 48.88 / 382.83 | 2.143 | +| 8 | LQ/512 | 646.0 | 2401.0 | 39.78 | 268.67 | 1.585 | 47.07 / 269.86 | 2.159 | + +## 16384/1024 + +| c | State/block | PP/req | PP wave | TG/req | TG aggregate | TTFT mean s | ITL p50 / p99 ms | MTP length | +|---:|---|---:|---:|---:|---:|---:|---:|---:| +| 1 | FP32/1024 | 2742.9 | 2718.1 | 93.83 | 60.68 | 5.973 | 24.17 / 25.05 | 2.266 | +| 1 | LQ/512 | 2707.8 | 2684.4 | 94.04 | 60.48 | 6.051 | 24.09 / 24.83 | 2.264 | +| 8 | FP32/1024 | 828.2 | 2558.2 | 19.96 | 113.02 | 19.784 | 50.27 / 429.21 | 2.305 | +| 8 | LQ/512 | 786.3 | 2554.6 | 20.04 | 108.48 | 20.858 | 48.68 / 425.68 | 2.164 | + +## 32768/2048 + +| c | State/block | PP/req | PP wave | TG/req | TG aggregate | TTFT mean s | ITL p50 / p99 ms | MTP length | +|---:|---|---:|---:|---:|---:|---:|---:|---:| +| 1 | FP32/1024 | 2685.2 | 2660.4 | 96.31 | 61.17 | 12.203 | 24.33 / 25.88 | 2.333 | +| 1 | LQ/512 | 2665.9 | 2649.5 | 100.62 | 62.75 | 12.291 | 24.26 / 25.01 | 2.437 | +| 8 | FP32/1024 | 416.2 | 1818.0 | 22.66 | 87.58 | 78.733 | 44.14 / 435.33 | 2.373 | +| 8 | LQ/512 | 507.8 | 1941.4 | 25.63 | 105.08 | 64.620 | 45.57 / 435.60 | 2.449 | From 89413e0a674443cbdadb9fdf41917761f8dce3e9 Mon Sep 17 00:00:00 2001 From: BlivionIaG Date: Sat, 14 Nov 2026 23:12:10 +0100 Subject: [PATCH 18/18] docs(rdna2): LeapQuant four-boot matrix result and accepted 1k c=1 cost [skip ci] Signed-off-by: BlivionIaG --- docs/rdna2/leapquant-gdn-2026-11-13.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/rdna2/leapquant-gdn-2026-11-13.md b/docs/rdna2/leapquant-gdn-2026-11-13.md index f772f9214d72..f611733f534a 100644 --- a/docs/rdna2/leapquant-gdn-2026-11-13.md +++ b/docs/rdna2/leapquant-gdn-2026-11-13.md @@ -709,3 +709,22 @@ baseline ones are the recipe's (`_causal_conv1d_update/_fwd` because the recipe sets `VLLM_CAUSAL_CONV1D_RDNA2_*=0`, `_mamba_align_block_table_kernel`, `_qsa_mqa_paged_kernel`, the PLE kernels, and under the MonoKernel `_fused_qk_rmsnorm_rope_gate_kernel` and `_num_nans_kernel`). + +## Four-boot matrix versus FP32 (2026-11-14) + +Receipts: `bench_results/2026-11-14_leapquant-matrix/`. FP32 (flashnext-mtp2, +block 1024) against `flashnext-leapquant-mtp2` (block 512), two boots each, +TP4 V620, MTP2, MonoKernel, FULL_AND_PIECEWISE, 16 requests per cell. + +- Aggregate output tok/s, LeapQuant versus FP32: 1k/512 c=1 -6.6%, c=8 +1.8%; + 16k/1k c=1 -0.3%, c=8 -4.0%; 32k/2k c=1 +2.6%, c=8 +20.0%. +- 32k/2k c=8: the FP32 boots preempted (10 and 6 times), LeapQuant never did. + The run changes block size and admission together; no FP32 + new-scheduler + control exists, so the gain is not attributed to LeapQuant alone. +- 1k/512 c=1: -6.6% (76.08 and 80.11 versus 83.15 and 84.07). ITL p50 is equal; + MTP length is lower (1.937 and 2.050 versus 2.135 and 2.105) and 1k TTFT is + +21% to +23%. Cause not established ([INFERENCE]: the lossy state lowers + draft agreement). **Accepted by the owner (2026-11-14) as the cost of the + recipe.** +- No candidate preemption, KFD eviction, fault or thermal abort. PPL 4.81 and + 4.85 versus 4.83 and 4.81.