diff --git a/CLAUDE.md b/CLAUDE.md index 61ef2361af..1c3bffb18c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -214,9 +214,9 @@ AST types (TypeDecl, Expression, Function, Structure, Enumeration, Variable, Mak Quick rules: - AST nodes have **unique ownership** — don't insert the same pointer into two parents; use `clone_type` / `clone_expression` / `clone_function` / `clone_variable` / `clone_structure` to duplicate. -- AST pointers use plain `=` and pass-by-value. **No `var inscope`, no `<-`** for them. `var inscope` is only for the residual smart_ptr types. +- AST pointers use plain `=` and pass-by-value. **No `var inscope`, no `<-`** for them. `var inscope` is for OWNED locals that should finalize at scope exit (smart_ptr types, plain arrays/structs) — never for gc_node AST pointers. - Tools that build AST at runtime (outside the compile pipeline) must wrap their scope in `ast_gc_guard() { ... }` from `daslib/ast`, or the leak detector reports `GC APP LEAK` at exit. -- daslang has garbage collection, but plain `var arr : array` does NOT finalize on scope exit. Either declare with `var inscope` (smart_ptr only), call `delete` explicitly, or move out via `<-`. Per-frame leaks in hot paths usually trace to a local `var arr` never deleted. +- daslang has garbage collection, but plain `var arr : array` does NOT finalize on scope exit. Either declare with `var inscope` (finalize-at-scope-exit — serves plain arrays/structs as well as the smart_ptr types; Boris-confirmed + compile-verified 2026-07-26), call `delete` explicitly, or move out via `<-`. Per-frame leaks in hot paths usually trace to a local `var arr` never deleted. Full migration table (when reading older docs that say `var inscope` or `<-` for AST types): **`skills/gc_migration.md`**. diff --git a/modules/dasLLAMA/.das_module b/modules/dasLLAMA/.das_module index 21cce1d58f..077ed5758d 100644 --- a/modules/dasLLAMA/.das_module +++ b/modules/dasLLAMA/.das_module @@ -5,7 +5,7 @@ require daslib/fio def initialize(project_path : string) { let dasllama_paths = [ "dasllama_env", "dasllama_par", "dasllama_parity", "dasllama_tune", "dasllama_math", "dasllama_math_default", "dasllama_math_aarch64_neon", "dasllama_math_gen", "dasllama_math_metal", "dasllama_math_vulkan", "dasllama_math_accelerate", "dasllama_kernel_access", "dasllama_vulkan_lens", "dasllama_metal_lens", "dasllama_metal_common", "dasllama_metal_kernels", "dasllama_metal_prefill", "dasllama_metal_llama", "dasllama_gemm_gen", "dasllama_gemm_register", "dasllama_gemm_schema", "dasllama_quant", "dasllama_gguf", - "dasllama_unicode", "dasllama_tokenizer", "dasllama_bpe", "dasllama_common", "dasllama_prefix", "dasllama_audio", "dasllama_audio_io", "dasllama_whisper", "dasllama_qwen3a", "dasllama_parakeet", "dasllama_canary", "dasllama_gemma4a", "dasllama_vad", "dasllama_asr", + "dasllama_unicode", "dasllama_tokenizer", "dasllama_bpe", "dasllama_common", "dasllama_layout", "dasllama_prefix", "dasllama_audio", "dasllama_audio_io", "dasllama_whisper", "dasllama_qwen3a", "dasllama_parakeet", "dasllama_canary", "dasllama_gemma4a", "dasllama_vad", "dasllama_asr", "dasllama_arch_llama", "dasllama_arch_qwen2", "dasllama_arch_qwen3", "dasllama_arch_phi3", "dasllama_arch_gemma2", "dasllama_arch_gemma3", "dasllama_arch_gemma4", "dasllama_arch_qwen2moe", "dasllama_arch_qwen3moe", "dasllama_arch_qwen35", "dasllama_arch_gptoss", "dasllama_arch_mistral3", "dasllama_arch_glm4moe", "dasllama_image", "dasllama_transformer", "dasllama_chat", "dasllama" ] diff --git a/modules/dasLLAMA/ENVIRONMENT.md b/modules/dasLLAMA/ENVIRONMENT.md index 1c1d7dee20..3baefe49dd 100644 --- a/modules/dasLLAMA/ENVIRONMENT.md +++ b/modules/dasLLAMA/ENVIRONMENT.md @@ -27,6 +27,8 @@ Read by the inference engine itself, so these affect any program that loads a mo | `DASLLAMA_NOISY` | flag | off | Print engine diagnostics (tier selection, upload plan, arm/decline reasons). | | `DASLLAMA_CALLER_PRIO` | number | unset | A/B rail for the dispatch caller's thread priority, -2..2; unset claims the top notch. | | `DASLLAMA_TRUTH_REFRESH` | flag | off | Regenerate the stored parity truth files instead of comparing against them. | +| `DASLLAMA_CONV_PROF` | flag | off | Bucket gguf -> image conversion time by kind over the weight walk; one clock pair per tensor. | +| `DASLLAMA_ALLOW_INTERP_LOAD` | flag | off | Permit a big gguf load without -jit; the transforms run interpreted, so expect minutes per GB. | | `DASLLAMA_GPU` | flag | off | One switch for the measured-best GPU rail set; any DASLLAMA_GPU_* knob still overrides individually. | | `DASLLAMA_GPU_MOE_LAYERS` | number | -1 (auto) | How many MoE expert layers to hold resident on the GPU; -1 lets the upload walk place the split. | | `DASLLAMA_GPU_MOE_STREAM` | number | -1 (auto) | How many MoE layers to stream rather than hold resident; -1 is auto. | @@ -90,6 +92,10 @@ Vulkan GPU backend. Present only where the dasVulkan package is installed. |---|---|---|---| | `DASLLAMA_COOPMAT` | number | auto | Cooperative-matrix mode; the flash-attention twin needs it even when the GEMM runs sdot4. | | `DASLLAMA_MM_SMALL` | number | 32 | Small-batch tier: 32 = sdot4 (default, beats both coopmat tiles below the crossover), 64 = coopmat M, 128 = always-L. | +| `DASLLAMA_MM_SMALLD` | number | 64 | Small-d cutoff routing narrow roles (k/v) to the small tier; widening measured worse, so this is an instrument. | +| `DASLLAMA_VK_FUSE` | flag | on | Fused decode tail (add+rms+requant, qk-norm+rope); 0 pins the split dispatches for a same-build A/B. | +| `DASLLAMA_VK_XFERQ` | flag | on | Stream expert uploads on the dedicated transfer queue, overlapped via a timeline semaphore; 0 keeps the single-queue rail. | +| `DASLLAMA_VK_MEMPRIO` | flag | on | Tag allocations high-priority (VK_EXT_memory_priority) so the driver demotes desktop memory, not ours. | | `DASLLAMA_VK_FA` | flag | on | Vulkan flash-attention kernel; 0 falls back to the chunked path. | | `DASLLAMA_VK_REBAR` | flag | on | Use a ReBAR device-local host-visible heap when one larger than 1GB is present. | | `DASLLAMA_VK_HAZARD_PARANOID` | flag | off | Barrier at every dispatch (correctness bisect). | @@ -144,6 +150,8 @@ Apple Accelerate / AMX float lane. `DASLLAMA_ACCEL` arms the whole group. | `DASMETAL_LAB_PASSES` | number | 4 | Passes per round in the Metal MoE lab. | | `DASMETAL_LAB_ATTN_NSGS` | number | 16 | Simdgroups per threadgroup in the Metal attention lab. | | `DASMETAL_LAB_DUMP_MSL` | flag | off | Dump generated MSL from the Metal labs instead of running it. | +| `PROBE_PATH` | path | _wcliff.bin | Scratch file the write-cliff probe writes, cwd-relative by default; point it at the drive under test. | +| `PROBE_GB` | number | 50 | Gigabytes the write-cliff probe writes before reporting. | ## Profiling and baselines diff --git a/modules/dasLLAMA/benchmarks/decode_step_trace.das b/modules/dasLLAMA/benchmarks/decode_step_trace.das index bc146d9bab..bbf115d329 100644 --- a/modules/dasLLAMA/benchmarks/decode_step_trace.das +++ b/modules/dasLLAMA/benchmarks/decode_step_trace.das @@ -4,6 +4,7 @@ options _jit_fast_math = true // ggml-parity FP laxity — the CPU prefill/sam require dasllama/dasllama_transformer // nolint:STYLE029 — umbrella fires each arch [init] registration; requiring dasllama_common directly would drop them require dasllama/dasllama_math // setup_dasllama_jobque_ + pin_kernel_backend + parallel_argmax +require dasllama/dasllama_layout // convert_model_to_metal_blob — the blob-flavor transform require ?das_metal dasllama/dasllama_metal_llama // nolint:STYLE030 — GPU driver half, Apple builds only require daslib/jobque_boost require daslib/clargs diff --git a/modules/dasLLAMA/benchmarks/lcpp_bench.das b/modules/dasLLAMA/benchmarks/lcpp_bench.das index 4a63bd33a5..111836de95 100644 --- a/modules/dasLLAMA/benchmarks/lcpp_bench.das +++ b/modules/dasLLAMA/benchmarks/lcpp_bench.das @@ -5,6 +5,8 @@ options _jit_fast_math = true // ggml-parity FP laxity — matches how llama.c require dasllama/dasllama_transformer // nolint:STYLE029 — umbrella fires each arch [init] registration; requiring dasllama_common directly would drop them require dasllama/dasllama_math // setup_dasllama_jobque_ + apply_box_profile_runtime (engine-level, not re-exported) require dasllama/dasllama_image // load_model_cached — the .dlim rail, without the full facade's compile weight +require dasllama/dasllama_tokenizer // the tg-real row needs REAL tokens (see Lever.real) +require daslib/module_path // get_this_module_dir — the default prompt corpus beside this bench require ?das_accelerate dasllama/dasllama_math_accelerate // nolint:STYLE030 — the --accel leg; Apple-only C++ module require ../performance/profile_common.das // profile_threads + affinity_on + bench records — the standing per-box methodology require daslib/jobque_boost @@ -93,6 +95,16 @@ struct Args { @clarg_doc = "A/B the GPU MoE prefill arm in-process: every row runs twice, device-side prefill OFF then ON. Needs resident expert layers (DASLLAMA_GPU_MOE_LAYERS); decode has no npos gate so tg reads as a wash. Mutually exclusive with the other A/B levers" prefill_ab : bool + @clarg_name = "mtp-ab" + @clarg_doc = "A/B MTP/NextN self-speculation in-process, OFF then ON (needs a NextN head in the GGUF). Runs the tg-real row, NOT the llama-bench tg row — acceptance is meaningless on synthetic ids. Reports the draft accept rate; mutually exclusive with the other A/B levers" + mtp_ab : bool + + @clarg_doc = "Prompt corpus for the tg-real row (segments split on lines of two percent signs; default: data/wikitext2_prompts.txt beside this bench)" + prompts : string = "" + + @clarg_doc = "After each tg-real arm's timed reps, run one UNTIMED window with forward_profile buckets — the per-section time split of that arm. With --mtp-ab this is what attributes draft vs verify vs classifier" + prof : bool + @clarg_short = "?" @clarg_doc = "Show this help and exit" help : bool @@ -106,6 +118,79 @@ def private build_prompt(n : int64; seed : int64) : array { return <- [for (i in range64(n)); synth_id(i, seed)] } +// One in-process A/B lever: both arms run under a SINGLE model load, so cross-run drift cancels and +// the pair is directly comparable. Arms ride the test name (" dense=off" / " dense=on"), which keeps +// both inside one BenchRun and leaves the (box, engine, backend, flavor) upsert identity intact. +// Adding a lever — including one pulled over from an archived bench — is one row in levers_of(). +struct private Lever { + flag : string // the CLI spelling, for diagnostics: ---ab + tag : string // the row suffix key + route : function<(on : bool) : void> + env : string // env that must arm the uploads first; "" when the lever needs none + real : bool // this lever is only meaningful under REAL greedy continuation, so it runs the + // tg-real row instead of the llama-bench pp/tg rows (synthetic ids would + // measure something the lever cannot affect and read as a valid result) +} + +def private levers_of(cfg : Args) : array { + var out : array + if (cfg.dense_ab) { + out |> emplace(Lever(flag = "dense", tag = "dense", route = @@set_moe_gpu_dense_route, + env = "DASLLAMA_GPU_DENSE")) + } + if (cfg.comb_ab) { + out |> emplace(Lever(flag = "comb", tag = "comb", route = @@set_gpu_combine_route, env = "")) + } + if (cfg.dn_ab) { + out |> emplace(Lever(flag = "dn", tag = "dn", route = @@set_moe_gpu_dn_route, + env = "DASLLAMA_GPU_DN")) + } + if (cfg.attn_ab) { + out |> emplace(Lever(flag = "attn", tag = "attn", route = @@set_moe_gpu_attn_route, + env = "DASLLAMA_GPU_ATTN")) + } + if (cfg.prefill_ab) { + out |> emplace(Lever(flag = "prefill", tag = "gpuprefill", route = @@set_moe_gpu_prefill_route, + env = "DASLLAMA_GPU_MOE_LAYERS")) + } + if (cfg.mtp_ab) { + // set_mtp_spec must be live BEFORE the prefill — it warms the draft head's KV + out |> emplace(Lever(flag = "mtp", tag = "mtp", route = @@set_mtp_spec, env = "", real = true)) + } + return <- out +} + +// Greedy pick over the last logits — the tg-real row's sampler. Runs INSIDE the timed window, but +// both A/B arms pay it identically so it cancels for judging. +def private argmax_logit(s : Session; vocab : int64) : int64 { + var best = 0l + var bv = s.logits[0] + for (v in range64(vocab)) { + if (s.logits[v] > bv) { + bv = s.logits[v] + best = v + } + } + return best +} + +// Prompt corpus: segments separated by lines of two percent signs, '#' segments skipped. +def private load_corpus(path : string) : array { + var texts : array + fopen(path, "rb") $(f) { + if (f == null) { + panic("lcpp_bench: can't open prompt corpus {path}") + } + for (p in split(fread(f), "%%")) { + let tp = strip(p) + if (!empty(tp) && !starts_with(tp, "#")) { + texts |> push(clone_string(tp)) + } + } + } + return <- texts +} + // progress line — stdout in txt mode, stderr when stdout carries the md table / json records def private say(fmt : OutFmt; text : string) { if (fmt == OutFmt.txt) { @@ -164,10 +249,15 @@ def main() { let plen = max(int64(cfg.plen), 0l) let ngen = max(int64(cfg.ngen), 0l) let reps = max(cfg.reps, 1) - if ((cfg.dense_ab ? 1 : 0) + (cfg.comb_ab ? 1 : 0) + (cfg.dn_ab ? 1 : 0) + (cfg.attn_ab ? 1 : 0) + (cfg.prefill_ab ? 1 : 0) > 1) { - print("warning: --dense-ab / --comb-ab / --dn-ab / --attn-ab / --prefill-ab are mutually exclusive — running the first only\n") + var levers <- levers_of(cfg) + if (length(levers) > 1) { + var names <- [for (l in levers); "--{l.flag}-ab"] + print("warning: A/B levers are mutually exclusive ({join(names, " ")}) — running --{levers[0].flag}-ab only\n") + delete names + levers |> resize(1) } - let ab = cfg.dense_ab || cfg.comb_ab || cfg.dn_ab || cfg.attn_ab || cfg.prefill_ab + let ab = !empty(levers) + let real_row = ab && levers[0].real var fmt = cfg.output if (ab && fmt != OutFmt.txt) { print("warning: -o {cfg.output} with an A/B lever — records cover the plain protocol only, falling back to txt\n") @@ -205,31 +295,58 @@ def main() { if (gpu) { set_metal_mode(MetalMode.required) // before load_model — pins the portable backend, activates the metal overrides } - if (cfg.dense_ab && !has_env_variable("DASLLAMA_GPU_DENSE")) { - print("warning: --dense-ab without DASLLAMA_GPU_DENSE=1 — no dense marks, both arms identical\n") - } - if (cfg.dn_ab && !has_env_variable("DASLLAMA_GPU_DN")) { - print("warning: --dn-ab without DASLLAMA_GPU_DN=1 — no deltanet marks, both arms identical\n") + // an unarmed lever silently measures the SAME thing twice — the one A/B failure that looks + // like a real result, so it warns rather than being left to the reader + if (ab && !empty(levers[0].env) && !has_env_variable(levers[0].env) + && !(cfg.prefill_ab && has_env_variable("DASLLAMA_GPU_MOE_STREAM"))) { + print("warning: --{levers[0].flag}-ab without {levers[0].env}=1 — nothing armed, both arms identical\n") } - if (cfg.attn_ab && !has_env_variable("DASLLAMA_GPU_ATTN")) { - print("warning: --attn-ab without DASLLAMA_GPU_ATTN=1 — no attention marks, both arms identical\n") - } - if (cfg.prefill_ab && !has_env_variable("DASLLAMA_GPU_MOE_LAYERS") && !has_env_variable("DASLLAMA_GPU_MOE_STREAM")) { - print("warning: --prefill-ab without DASLLAMA_GPU_MOE_LAYERS/_STREAM — no resident expert layers, both arms identical\n") - } - say(fmt, "loading {cfg.model}\n") - var results : array - var modelArch = "" - var activeB = 0.0lf - var execFmt = "" + // -m accepts a comma-separated list: one process benches every model in turn, so a sweep + // pays the compile/startup cost once; a single -m path behaves exactly as before + // trim: `-m "a.gguf, b.gguf"` is the natural spelling, and an untrimmed path fails to open + var model_list <- [for (m in split(cfg.model, ",")); strip(m)] + var mdRows : array> + var jsonModels : array with_job_que() { setup_dasllama_jobque_() + for (mpath in model_list) { + continue if (empty(mpath)) + say(fmt, "loading {mpath}\n") + var results : array + var modelArch = "" + var activeB = 0.0lf + var execFmt = "" // load INSIDE the que, on the image rail: the .dlim maps in ms, and the GPU tier's // gather/upload parallelizes over the lanes (que-less it runs single-threaded — at // STREAM=35 that gather was the dominant per-run cost) - var t <- load_model_cached(cfg.model, cfg.quant) - t.config.seq_len = min(t.config.seq_len, max(plen, ngen) + 8l) // cap the KV allocation + var t <- load_model_cached(mpath, cfg.quant) + // the tg-real row continues REAL prompts, so its KV must cover the longest one + the window + var inscope toks : array> + if (real_row) { + var tk <- load_tokenizer_auto(mpath) + var texts <- load_corpus(empty(cfg.prompts) + ? path_join(get_this_module_dir(), "data/wikitext2_prompts.txt") : cfg.prompts) + var maxlen = 0l + for (tx in texts) { + var ids <- encode(tk, tx, false) + maxlen = max(maxlen, long_length(ids)) + toks |> emplace(ids) + } + delete texts + delete tk + if (empty(toks)) { + panic("lcpp_bench: prompt corpus tokenized to nothing — --{levers[0].flag}-ab needs real prompts") + } + t.config.seq_len = min(t.config.seq_len, maxlen + ngen + 8l) + } else { + t.config.seq_len = min(t.config.seq_len, max(plen, ngen) + 8l) // cap the KV allocation + } let c = t.config + // a model with no NextN head silently no-ops set_mtp_spec — both arms would be identical + // and the row would read as a valid 1.00x result. Fail closed instead. + if (cfg.mtp_ab && c.n_layer_nextn <= 0l) { + panic("lcpp_bench: --mtp-ab needs a NextN draft head, and {mpath} has none") + } say(fmt, "config: dim={c.dim} layers={c.n_layers} vocab={c.vocab_size} | backend {active_kernel_backend()} (batch {active_batch_backend()}) | t{threads} affinity {pinned ? "hard" : "env/off"} | pp{plen} tg{ngen} x {reps} reps\n") modelArch = "{t.arch}" activeB = active_params_b(t) @@ -237,21 +354,88 @@ def main() { var s <- make_run_state(c) for (arm in range(ab ? 2 : 1)) { var dtag = "" - if (cfg.dense_ab) { - dtag = arm == 0 ? " dense=off" : " dense=on" - set_moe_gpu_dense_route(arm != 0) - } elif (cfg.comb_ab) { - dtag = arm == 0 ? " comb=off" : " comb=on" - set_gpu_combine_route(arm != 0) - } elif (cfg.dn_ab) { - dtag = arm == 0 ? " dn=off" : " dn=on" - set_moe_gpu_dn_route(arm != 0) - } elif (cfg.attn_ab) { - dtag = arm == 0 ? " attn=off" : " attn=on" - set_moe_gpu_attn_route(arm != 0) - } elif (cfg.prefill_ab) { - dtag = arm == 0 ? " gpuprefill=off" : " gpuprefill=on" - set_moe_gpu_prefill_route(arm != 0) + if (ab) { + dtag = " {levers[0].tag}={arm == 0 ? "off" : "on"}" + invoke(levers[0].route, arm != 0) + } + if (real_row) { + // REAL greedy continuation: fresh session per prompt (no KV/recurrent bleed), + // prefill untimed as llama-bench excludes the prompt, argmax inside the timed loop + // (both arms pay it, so it cancels). NOT comparable to the synthetic tg row. + let mtp_on = cfg.mtp_ab && arm != 0 + var drafted = 0l + var accepted = 0l + var tps : array + for (_rep in range(reps)) { + var total_us = 0l + var rep_gen = 0l + for (pi in range(length(toks))) { + var rs <- make_run_state(c) + let pl = long_length(toks[pi]) + forward_prefill(t, rs, toks[pi], pl, 0l) + let t0 = ref_time_ticks() + if (mtp_on) { + rs.n_past = pl // forward_prefill leaves n_past alone; mtp_spec_eval drives it + var tok = argmax_logit(rs, c.vocab_size) + var gen = 0l + while (gen < ngen) { + var acc = 0l // the id is unused — only the 1-vs-2 advance is timed + gen += mtp_spec_eval(t, rs, tok, acc) ? 2l : 1l + tok = argmax_logit(rs, c.vocab_size) + } + rep_gen += gen // a trailing accept may overshoot by 1 — count what ran + } else { + var pos = pl + for (_g in range64(ngen)) { + forward(t, rs, argmax_logit(rs, c.vocab_size), pos) + pos++ + } + rep_gen += ngen + } + total_us += int64(get_time_usec(t0)) + drafted += rs.mtp_drafted // session-lifetime counters — harvest before the free + accepted += rs.mtp_accepted + delete rs + } + tps |> push(double(rep_gen) * 1.0e6lf / double(total_us)) + } + let st = row_stat("tg-real{ngen}{dtag}", tps) + say(fmt, "{st.test}: {st.mean:.2f} ± {st.sd:.2f} tok/s ({reps} reps, {length(toks)} prompts)\n") + if (mtp_on) { + let rate = drafted > 0l ? double(accepted) * 100.0lf / double(drafted) : 0.0lf + say(fmt, " mtp: {accepted}/{drafted} drafts accepted ({rate:.1f}%)\n") + } + results |> push(st) + delete tps + if (cfg.prof) { + // untimed bucket window for THIS arm — the prefill lands BEFORE the reset, so + // the report is pure decode (or pure draft+verify on the on arm). Running it + // per arm is the point: the off/on bucket delta is what attributes the draft. + var ps <- make_run_state(c) + let pl0 = long_length(toks[0]) + forward_prefill(t, ps, toks[0], pl0, 0l) + forward_profile_reset() + if (mtp_on) { + ps.n_past = pl0 + var ptok = argmax_logit(ps, c.vocab_size) + var pgen = 0l + while (pgen < ngen) { + var pacc = 0l + pgen += mtp_spec_eval(t, ps, ptok, pacc) ? 2l : 1l + ptok = argmax_logit(ps, c.vocab_size) + } + } else { + var ppos = pl0 + for (_g in range64(ngen)) { + forward(t, ps, argmax_logit(ps, c.vocab_size), ppos) + ppos++ + } + } + say(fmt, "--- forward_profile: {st.test} ---\n") + forward_profile_report() + delete ps + } + continue } if (plen > 0l) { let warm <- build_prompt(plen, 999l) @@ -294,46 +478,44 @@ def main() { } delete s delete t - } - // the reference runs AFTER our model is freed — one model process at a time, per the protocol - var refRows : array - var refCmd = "" - if (!empty(cfg.ref)) { - let aff = (!gpu && affinity_on()) ? lcpp_affinity_args(threads) : "" - refCmd = "\"{cfg.ref}\" -m \"{cfg.model}\" -ngl {cfg.ngl} -t {threads}{aff} -p {cfg.plen} -n {cfg.ngen} -r {reps} -o json" - say(fmt, "ref: {refCmd}\n") - if (!run_llama_bench(refCmd, refRows)) { - say(fmt, "warning: llama-bench run/parse failed — no ref rows\n") + // the reference runs AFTER our model is freed — one model in memory at a time + var refRows : array + var refCmd = "" + if (!empty(cfg.ref)) { + let aff = (!gpu && affinity_on()) ? lcpp_affinity_args(threads) : "" + refCmd = "\"{cfg.ref}\" -m \"{mpath}\" -ngl {cfg.ngl} -t {threads}{aff} -p {cfg.plen} -n {cfg.ngen} -r {reps} -o json" + say(fmt, "ref: {refCmd}\n") + if (!run_llama_bench(refCmd, refRows)) { + say(fmt, "warning: llama-bench run/parse failed — no ref rows\n") + } + for (rr in refRows) { + let tag = rr.n_gen == 0 ? "pp{rr.n_prompt}" : "tg{rr.n_gen}" + say(fmt, "ref {tag}: {rr.avg_ts:.2f} tok/s\n") + } } - for (rr in refRows) { - let tag = rr.n_gen == 0 ? "pp{rr.n_prompt}" : "tg{rr.n_gen}" - say(fmt, "ref {tag}: {rr.avg_ts:.2f} tok/s\n") + if (fmt == OutFmt.txt) { + delete results + delete refRows + continue } - } - if (fmt == OutFmt.txt) { - return - } - let backend = gpu ? "metal" : "cpu" - let gguf = base_name(cfg.model) - if (fmt == OutFmt.md) { - let headers <- ["engine", "model", "backend", "ngl", "threads", "test", "t/s"] - let aligns <- [ColAlign.left, ColAlign.left, ColAlign.left, ColAlign.right, ColAlign.right, ColAlign.left, ColAlign.right] - var rows : array> - rows |> reserve(length(results) + length(refRows)) - for (st in results) { - var row <- ["das", gguf, backend, "{cfg.ngl}", "{threads}", st.test, "{st.mean:.2f} ± {st.sd:.2f}"] - rows |> emplace(row) - } - for (rr in refRows) { - let tag = rr.n_gen == 0 ? "pp{rr.n_prompt}" : "tg{rr.n_gen}" - let sd = rr.stddev_ts - var row <- ["llama.cpp", gguf, backend, "{cfg.ngl}", "{threads}", tag, "{rr.avg_ts:.2f} ± {sd:.2f}"] - rows |> emplace(row) + let backend = gpu ? "metal" : "cpu" + let gguf = base_name(mpath) + if (fmt == OutFmt.md) { + for (st in results) { + var row <- ["das", gguf, backend, "{cfg.ngl}", "{threads}", st.test, "{st.mean:.2f} ± {st.sd:.2f}"] + mdRows |> emplace(row) + } + for (rr in refRows) { + let tag = rr.n_gen == 0 ? "pp{rr.n_prompt}" : "tg{rr.n_gen}" + let sd = rr.stddev_ts + var row <- ["llama.cpp", gguf, backend, "{cfg.ngl}", "{threads}", tag, "{rr.avg_ts:.2f} ± {sd:.2f}"] + mdRows |> emplace(row) + } + delete results + delete refRows + continue } - print(render_md_table(headers, aligns, rows)) - return - } - // -o json: the public bench records — one BenchModel, das run + optional llama.cpp sibling + // -o json: the public bench records — one BenchModel per gguf, das run + optional llama.cpp sibling let hw = gather_hardware() let date = format_time(get_clock(), "%Y-%m-%d") let box = box_name() @@ -342,11 +524,10 @@ def main() { arch = modelArch, quant = quant_str(cfg.quant), active_b = activeB, - size_bytes = int64(stat(cfg.model).size)) + size_bytes = int64(stat(mpath).size)) if (!empty(refRows)) { m.params_b = double(refRows[0].model_n_params) / 1.0e9lf } - let argstr = join(get_cli_arguments(), " ") var dr = BenchRun( engine = "das", backend = backend, @@ -354,7 +535,8 @@ def main() { box = box, threads = threads, date = date, - cmd = "bin/daslang -jit modules/dasLLAMA/benchmarks/lcpp_bench.das -- {argstr}", + cmd = bench_cmd_line(), + env = bench_env_line(), sha = run_capture("git rev-parse --short HEAD", true), version = das_version(), tune = tune_summary(), @@ -380,9 +562,21 @@ def main() { } m.runs |> emplace(rr) } - var models : array - models |> emplace(m) - let text = sprint_json(models, true) + jsonModels |> emplace(m) + delete results + delete refRows + } // model loop + } // with_job_que + if (fmt == OutFmt.md) { + let headers <- ["engine", "model", "backend", "ngl", "threads", "test", "t/s"] + let aligns <- [ColAlign.left, ColAlign.left, ColAlign.left, ColAlign.right, ColAlign.right, ColAlign.left, ColAlign.right] + print(render_md_table(headers, aligns, mdRows)) + return + } + if (fmt != OutFmt.json) { + return + } + let text = sprint_json(jsonModels, true) if (!empty(cfg.json_path)) { var wrote = false fopen(cfg.json_path, "wb") $(f) { diff --git a/modules/dasLLAMA/benchmarks/write_cliff_probe.das b/modules/dasLLAMA/benchmarks/write_cliff_probe.das new file mode 100644 index 0000000000..996e6146bb --- /dev/null +++ b/modules/dasLLAMA/benchmarks/write_cliff_probe.das @@ -0,0 +1,70 @@ +options gen2 + +// Isolated sustained-write probe: fill ONE buffer, write it N times, report the rolling rate. +// Nothing else — no model, no JIT, no tune, no transcode. The point is to watch a consumer SSD's +// SLC cache exhaust as a time series (the "cliff"), instead of inferring it from a conversion +// total. The .dlim writer is the caller that hits this: 75 GB at 275 MB/s where a 2.5 GB image +// writes at ~1 GB/s. +// bin/daslang modules/dasLLAMA/benchmarks/write_cliff_probe.das +// PROBE_GB=50 PROBE_PATH=/mnt/nvme/_wcliff.bin bin/daslang .../write_cliff_probe.das + +require daslib/fio +require math +require dasllama/dasllama_env + +let CHUNK_MB = 16l +let WINDOW_CHUNKS = 32l // one report line per 512 MB + +[export] +def main() { + let gb = max(env_int64("PROBE_GB", 50l), 1l) + // cwd-relative so the default works on every host; point it at the drive under test + let path = env_str("PROBE_PATH", "_wcliff.bin") + let chunk_bytes = CHUNK_MB * 1024l * 1024l + let chunks = max(gb * 1024l / CHUNK_MB, 1l) + var buf : array + buf |> resize(int(chunk_bytes)) + // a varying pattern, not zeros — zero pages can be swallowed by controller compression or + // sparse-file handling and would read as infinite bandwidth + for (i in range64(chunk_bytes)) { + buf[i] = uint8((i * 31l + 17l) & 255l) + } + print("write-cliff probe: {gb} GB as {chunks} x {CHUNK_MB} MB -> {path}\n") + print(" done | window MB/s | avg MB/s\n") + var t_close = 0l + var slowest = 0.0lf + var fastest = 0.0lf + fopen(path, "wb") $(f) { + if (f == null) { + panic("write-cliff probe: cannot open {path} for writing") + } + let t_start = ref_time_ticks() + var t_win = t_start + var written = 0l + for (c in range64(chunks)) { + fwrite(f, buf) + written += chunk_bytes + if ((c + 1l) % WINDOW_CHUNKS == 0l) { + let win_s = double(get_time_nsec(t_win)) / 1.0e9lf + let tot_s = double(get_time_nsec(t_start)) / 1.0e9lf + let win_mbs = double(WINDOW_CHUNKS * CHUNK_MB) / win_s + let avg_mbs = double(written / 1048576l) / tot_s + slowest = slowest == 0.0lf ? win_mbs : min(slowest, win_mbs) + fastest = max(fastest, win_mbs) + print(" {written / 1073741824l:5} GB | {win_mbs:10.0f} | {avg_mbs:8.0f}\n") + t_win = ref_time_ticks() + } + } + // fflush pushes stdio's buffer to the OS; the REAL flush to media happens in fclose at + // block exit, and on a big write that stall is part of the cost — so it is timed too + fflush(f) + t_close = ref_time_ticks() + } + print("close/flush: {get_time_nsec(t_close) / 1000000l} ms\n") + print("window rate: fastest {fastest:.0f} MB/s, slowest {slowest:.0f} MB/s") + print(fastest > 0.0lf ? " — cliff {fastest / max(slowest, 1.0lf):.1f}x\n" : "\n") + delete buf + if (!remove(path)) { + print("warning: could not remove {path} — delete it by hand\n") + } +} diff --git a/modules/dasLLAMA/dasllama/dasllama_asr.das b/modules/dasLLAMA/dasllama/dasllama_asr.das index 9a4070ac39..22e79ae520 100644 --- a/modules/dasLLAMA/dasllama/dasllama_asr.das +++ b/modules/dasLLAMA/dasllama/dasllama_asr.das @@ -142,7 +142,7 @@ def load_asr_model(path : string; mmproj : string) : AsrModel { fmap(ef) $(var bytes : array#) { emagic = rd_u32(bytes, 0l) if (emagic == 0x46554747u) { // "GGUF" — sniff the audio projector type - let mm <- parse_gguf_meta(bytes) + var inscope mm <- parse_gguf_meta(bytes) if (gguf_has(mm, "clip.audio.projector_type")) { eproj = gguf_str(mm, bytes, "clip.audio.projector_type") } elif (gguf_has(mm, "clip.projector_type")) { diff --git a/modules/dasLLAMA/dasllama/dasllama_audio.das b/modules/dasLLAMA/dasllama/dasllama_audio.das index 31d79b0129..d577f4a5f2 100644 --- a/modules/dasLLAMA/dasllama/dasllama_audio.das +++ b/modules/dasLLAMA/dasllama/dasllama_audio.das @@ -1005,7 +1005,7 @@ def private load_audio_tower_(path : string; q8 : bool = false) : AudioTower { panic("dasLLAMA: cannot open mmproj gguf '{path}'") } fmap(f) $(var bytes : array#) { - let m <- parse_gguf_meta(bytes) + var inscope m <- parse_gguf_meta(bytes) let proj = gguf_str(m, bytes, "clip.projector_type") if (proj == "qwen2a" || proj == "qwen2.5o") { // omni's audio side is qwen2a verbatim t.proj_kind = AudioProjKind.qwen2a diff --git a/modules/dasLLAMA/dasllama/dasllama_bpe.das b/modules/dasLLAMA/dasllama/dasllama_bpe.das index 97714e7f36..33bd4c73a0 100644 --- a/modules/dasLLAMA/dasllama/dasllama_bpe.das +++ b/modules/dasLLAMA/dasllama/dasllama_bpe.das @@ -36,9 +36,9 @@ def load_bpe_tokenizer_gguf(path : string) : BpeTokenizer { panic("dasLLAMA: cannot open gguf '{path}'") } fmap(f) $(var bytes : array#) { - let m <- parse_gguf_meta(bytes) - let toks <- gguf_str_array(m, bytes, "tokenizer.ggml.tokens") - let merges <- gguf_str_array(m, bytes, "tokenizer.ggml.merges") + var inscope m <- parse_gguf_meta(bytes) + var inscope toks <- gguf_str_array(m, bytes, "tokenizer.ggml.tokens") + var inscope merges <- gguf_str_array(m, bytes, "tokenizer.ggml.merges") let n = length(toks) tk.vocab |> reserve(n) for (i in range(n)) { diff --git a/modules/dasLLAMA/dasllama/dasllama_common.das b/modules/dasLLAMA/dasllama/dasllama_common.das index 39b7cb25c3..25eb878bbb 100644 --- a/modules/dasLLAMA/dasllama/dasllama_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_common.das @@ -43,8 +43,8 @@ enum QuantMode { } //! Per-weight storage format under a q8-mode load with native planes: q8 = qblob/qscales; -//! k4/k5/k6/q40 = superblock plane pairs (a file mixes formats PER TENSOR — tags ride per layer); -//! q51 = native Q5_1, per-32-BLOCK planes + Q8_0-form activations (rows % 32, expert stacks only). +//! k4/k5/k6/q40 = superblock plane pairs (mixed PER TENSOR); q51 = native Q5_1 per-32 planes; +//! q8n = q8 as gguf-native 34B interleaved blocks (ONE plane, the cm2 GPU-tier tag, never disk). enum KqFmt : uint8 { q8 k4 @@ -52,6 +52,7 @@ enum KqFmt : uint8 { k6 q40 q51 + q8n } // The superblock-lattice formats (Q8_K-form activations, % 256 rows, repack + stamped kq @@ -319,6 +320,232 @@ def metal_model_servable(t : Model) : bool { return g_metal_servable_set && invoke(g_metal_servable, t) } +// ===== disk->compute layout hooks (registered by dasllama_layout at [init]) ===== + +// The layout transforms live in dasllama_layout (which requires this module back for Model/KqFmt); +// the loader reaches them through these hooks. Unset hooks PANIC at first use — a program that +// loads models must require dasllama/dasllama_layout (umbrella + transformer facade both do). + +typedef ModelRepackFn = function<(var t : Model) : void> +typedef GatherStackFn = function<(t : Model; woff : int64; n, rows, slice_rows : int64; + mr, kgroup, wbias : int64; + var wq : array; var ws : array) : void> +typedef GatherStackKqFn = function<(t : Model; fmt : KqFmt; woff : int64; n, rows, slice_rows : int64; + repacked : bool; mr : int64; + var wq : array; var ws : array) : void> +typedef GatherStackQ8nFn = function<(t : Model; woff : int64; n, rows, slice_rows : int64; + mr, kgroup, wbias : int64; var wq : array) : void> + +var private g_layout_repack_q8 : ModelRepackFn +var private g_layout_repack_kq : ModelRepackFn +var private g_layout_repack_mx4 : ModelRepackFn +var private g_layout_repack_q51 : ModelRepackFn +var private g_layout_gather : GatherStackFn +var private g_layout_gather_kq : GatherStackKqFn +var private g_layout_gather_q8n : GatherStackQ8nFn +var private g_layout_hooks_set = false + +def register_model_layout(rq8, rkq, rmx4, rq51 : ModelRepackFn; gs : GatherStackFn; gskq : GatherStackKqFn; + gsq8n : GatherStackQ8nFn) { + g_layout_repack_q8 = rq8 + g_layout_repack_kq = rkq + g_layout_repack_mx4 = rmx4 + g_layout_repack_q51 = rq51 + g_layout_gather = gs + g_layout_gather_kq = gskq + g_layout_gather_q8n = gsq8n + g_layout_hooks_set = true +} + +def private layout_hooks_check { + if (!g_layout_hooks_set) { + panic("dasLLAMA: layout hooks unset — the program must require dasllama/dasllama_layout") + } +} + +def private repack_q8_weights(var t : Model) { + layout_hooks_check() + invoke(g_layout_repack_q8, t) +} + +def private repack_kq_weights(var t : Model) { + layout_hooks_check() + invoke(g_layout_repack_kq, t) +} + +def private repack_mx4_stacks(var t : Model) { + layout_hooks_check() + invoke(g_layout_repack_mx4, t) +} + +def private repack_q51_stacks(var t : Model) { + layout_hooks_check() + invoke(g_layout_repack_q51, t) +} + +// ===== the vulkan-flavor image bake (driven by dasllama_image) ===== +// Collect: a normal load's GPU walk appends every gather's output to the bake blob; slice: a +// flavor load's walk skips the transforms and copies the mapped blob back, plan-checked. + +//! One baked gather of the vulkan flavor's plan — walk order, offsets into Model.vkblob. +struct VkPlanEntry { + fmt : int + woff : int64 + n : int64 + rows : int64 + wq_off : int64 + wq_bytes : int64 + ws_off : int64 + ws_bytes : int64 +} + +enum private VkBakeMode { + off + collect + slice +} + +var private g_vk_bake = VkBakeMode.off +var private g_vk_bake_blob : array +var private g_vk_bake_plan : array +var private g_vk_bake_cursor = 0l + +//! Arm collection for the next model load's GPU walk (the vulkan-flavor bake). +def vulkan_bake_begin { + g_vk_bake = VkBakeMode.collect + g_vk_bake_blob |> clear() + g_vk_bake_plan |> clear() +} + +//! Move the collected planes into the model and disarm; false = nothing collected (no GPU walk ran). +def vulkan_bake_take(var t : Model) : bool { + g_vk_bake = VkBakeMode.off + if (empty(g_vk_bake_plan)) { + delete g_vk_bake_blob + return false + } + t.vkblob <- g_vk_bake_blob + t.vkplan <- g_vk_bake_plan + return true +} + +//! Arm slicing for a mapped vulkan-flavor load: the walk reads t.vkblob instead of gathering. +def vulkan_bake_slice_begin { + g_vk_bake = VkBakeMode.slice + g_vk_bake_cursor = 0l +} + +//! Disarm after a successful flavor load; a partially-consumed plan means the walk diverged +//! (the slicer already declined to live gathers — correct, just slower) and is worth a rebake. +def vulkan_bake_slice_end(t : Model) { + if (g_vk_bake != VkBakeMode.off && g_vk_bake_cursor != long_length(t.vkplan)) { + to_log(LOG_WARNING, "dasLLAMA vulkan image: the GPU walk consumed {g_vk_bake_cursor} of {length(t.vkplan)} baked gathers — the image is stale for this run's shape, consider rebaking\n") + } + g_vk_bake = VkBakeMode.off +} + +//! Disarm without verification (a flavor-load attempt that never ran the walk). +def vulkan_bake_disarm { + g_vk_bake = VkBakeMode.off +} + +def private vk_bake_append(a : array) : int64 { + // 4KB-aligned starts (import windows); long_ forms — the blob crosses the int32 array rail + let off = (long_length(g_vk_bake_blob) + 4095l) / 4096l * 4096l + let abytes = long_length(a) + let need = off + abytes + // growth capped at 2GB steps past 4GB — doubling a multi-GB blob wastes up to half its size + if (long_capacity(g_vk_bake_blob) < need) { + let cap = long_capacity(g_vk_bake_blob) + let grown = cap < 4_294_967_296l ? cap * 2l : cap + 2_147_483_648l + g_vk_bake_blob |> reserve(max(need, grown)) + } + g_vk_bake_blob |> resize(need) + if (abytes > 0l) { + unsafe { + memcpy(addr(g_vk_bake_blob[off]), addr(a[0]), abytes) + } + } + return off +} + +def private vk_bake_collect(fmt : int; woff, n, rows : int64; wq, ws : array) { + if (g_vk_bake != VkBakeMode.collect) { + return + } + var e = VkPlanEntry(fmt = fmt, woff = woff, n = n, rows = rows, + wq_bytes = long_length(wq), ws_bytes = long_length(ws)) + e.wq_off = vk_bake_append(wq) + e.ws_off = vk_bake_append(ws) + g_vk_bake_plan |> push(e) +} + +def private vk_bake_slice(t : Model; fmt : int; woff, n, rows : int64; + var wq : array; var ws : array) : bool { + if (g_vk_bake != VkBakeMode.slice) { + return false + } + // plan divergence DECLINES to live gathers mid-walk (seamless: matched entries were byte-identical) + if (g_vk_bake_cursor >= long_length(t.vkplan)) { + to_log(LOG_WARNING, "dasLLAMA vulkan image: the GPU walk overran the baked plan at gather {g_vk_bake_cursor} — regathering live from here\n") + g_vk_bake = VkBakeMode.off + return false + } + let e = t.vkplan[int(g_vk_bake_cursor)] + if (e.fmt != fmt || e.woff != woff || e.n != n || e.rows != rows) { + to_log(LOG_WARNING, "dasLLAMA vulkan image: baked gather {g_vk_bake_cursor} is (fmt {e.fmt} woff {e.woff} {e.n}x{e.rows}), the walk wants (fmt {fmt} woff {woff} {n}x{rows}) — regathering live from here\n") + g_vk_bake = VkBakeMode.off + return false + } + g_vk_bake_cursor++ + // int-indexed destinations: a >2GB plane would resize truncated and then memcpy past it + assert(e.wq_bytes <= 2_147_483_647l && e.ws_bytes <= 2_147_483_647l, + "vulkan image: a baked plane exceeds the int-indexed array cap") + wq |> resize(int(e.wq_bytes)) + ws |> resize(int(e.ws_bytes)) + unsafe { + // int64 blob indexing — wq_off crosses INT32_MAX on multi-GB bakes + if (e.wq_bytes > 0l) { + memcpy(addr(wq[0]), addr(t.vkblob[e.wq_off]), e.wq_bytes) + } + if (e.ws_bytes > 0l) { + memcpy(addr(ws[0]), addr(t.vkblob[e.ws_off]), e.ws_bytes) + } + } + return true +} + +def private moe_gpu_gather_stack(t : Model; woff : int64; n, rows, slice_rows : int64; mr, kgroup, wbias : int64; + var wq : array; var ws : array) { + layout_hooks_check() + if (vk_bake_slice(t, 0, woff, n, rows, wq, ws)) { + return + } + invoke(g_layout_gather, t, woff, n, rows, slice_rows, mr, kgroup, wbias, wq, ws) + vk_bake_collect(0, woff, n, rows, wq, ws) +} + +def private moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice_rows : int64; + repacked : bool; mr : int64; var wq : array; var ws : array) { + layout_hooks_check() + if (vk_bake_slice(t, int(fmt), woff, n, rows, wq, ws)) { + return + } + invoke(g_layout_gather_kq, t, fmt, woff, n, rows, slice_rows, repacked, mr, wq, ws) + vk_bake_collect(int(fmt), woff, n, rows, wq, ws) +} + +def private moe_gpu_gather_stack_q8n(t : Model; woff : int64; n, rows, slice_rows : int64; + mr, kgroup, wbias : int64; var wq : array) { + layout_hooks_check() + var no_ws : array + if (vk_bake_slice(t, 6, woff, n, rows, wq, no_ws)) { + return + } + invoke(g_layout_gather_q8n, t, woff, n, rows, slice_rows, mr, kgroup, wbias, wq) + vk_bake_collect(6, woff, n, rows, wq, no_ws) +} + def register_batch_decode_override(name : string; fn : BatchDecodeOverrideFn) { g_batch_decode_overrides[name] = fn } @@ -695,6 +922,11 @@ struct Model { mtp_enorm_off : int64 = -1 // fblob: nextn.enorm (token-embedding RMS weight), dim mtp_hnorm_off : int64 = -1 // fblob: nextn.hnorm (trunk-hidden RMS weight), dim mtp_headnorm_off : int64 = -1 // fblob: nextn.shared_head_norm, dim (-1 = output_norm) + // vulkan-flavor bake (dasllama_image): the GPU walk's gathered device-layout planes + the + // walk-order plan. Empty on planar/metal flavors (empty planes never save); borrowed + // views on a flavor load — the generic finalizer walk covers both. + vkblob : array + vkplan : array // prepared-model image backing (dasllama_image): when set, every locked array field above // is a borrowed VIEW into this mapping (zero-copy) and the finalizer below unmaps it. // Both fields are skipped by the image save/load field walk. @@ -862,7 +1094,7 @@ def private kq_take(var cur : KqCursors; f : KqFmt; n : int64) : int64 { } // Format tag for layer l of a per-kind tag array; empty array (non-kq load) = q8. -def private fmt_at(a : array; l : int64) : KqFmt => empty(a) ? KqFmt.q8 : a[l] +def fmt_at(a : array; l : int64) : KqFmt => empty(a) ? KqFmt.q8 : a[l] //! Tied K-quant classifier: the embedding table lives in a native K-quant plane at wcls_off — //! no fp32 table, no qblob copy (the cls_q8 analog for kquant_native loads). @@ -1239,45 +1471,126 @@ def load_checkpoint(path : string) : Model { // Read one big 2D weight tensor into the active representation: transcode straight from disk when // the on-disk type already matches the target precision, otherwise dequant to a reused fp32 scratch // and quantize into the shared blob. `fmt` != q8 routes into that native K-quant plane pair. + +// ===== conversion-time accounting ===== +// load_big's branches ARE the conversion kinds, so bucketing here answers "where does gguf -> image +// time go". DASLLAMA_CONV_PROF arms it: one clock pair per tensor, a dead branch when off. + +struct private ConvBucket { + nsec : int64 + elems : int64 + calls : int64 +} + +struct private ConvRow { + name : string + nsec : int64 + elems : int64 + calls : int64 +} + +var private g_conv_prof = false +var private g_conv : table + +//! Arm per-conversion-kind timing over the gguf -> image weight walk (env DASLLAMA_CONV_PROF=1 +//! arms it at load). Buckets reset on each arm; dump them with [[conv_profile_report]]. +def set_conv_profile(on : bool) { + g_conv_prof = on + g_conv |> clear() +} + +//! Current state of the [[set_conv_profile]] gate. +def get_conv_profile() : bool => g_conv_prof + +def private conv_note(kind : string; t0, n : int64) { + if (!g_conv_prof) { + return + } + let ns = get_time_nsec(t0) + var b = g_conv[kind] + b.nsec += ns + b.elems += n + b.calls++ + g_conv[kind] = b +} + +//! Dump the conversion-kind breakdown gathered since the last [[set_conv_profile]] arm, slowest +//! first: wall ms, share of the accounted total, weights moved, and the arm's throughput. +def conv_profile_report() { + if (!g_conv_prof || empty(g_conv)) { + return + } + var rows : array + var total = 0l + for (k, v in keys(g_conv), values(g_conv)) { + rows |> emplace(ConvRow(name = clone_string(k), nsec = v.nsec, elems = v.elems, calls = v.calls)) + total += v.nsec + } + rows |> sort() $(a, b) => a.nsec > b.nsec + to_log(LOG_INFO, "dasLLAMA conv: {length(rows)} kinds, {double(total) / 1.0e6lf:.0f} ms accounted\n") + for (r in rows) { + let ms = double(r.nsec) / 1.0e6lf + let pct = total > 0l ? 100.0lf * double(r.nsec) / double(total) : 0.0lf + let mws = r.nsec > 0l ? double(r.elems) * 1000.0lf / double(r.nsec) : 0.0lf + to_log(LOG_INFO, "dasLLAMA conv: {r.name}: {ms:.0f} ms ({pct:.1f}%), " + + "{double(r.elems) / 1.0e6lf:.0f} Mw over {r.calls} tensors, {mws:.0f} Mw/s\n") + } + delete rows +} + def private load_big(m : GGUFMeta; bytes : array | #; name : string; var t : Model; woff, n : int64; var scratch : array; src_off : int64 = 0l; fmt : KqFmt = KqFmt.q8) { + let t0 = g_conv_prof ? ref_time_ticks() : 0l + var kind = "f32 read (no quant)" if (fmt == KqFmt.k4) { gguf_transcode_q4k(m, bytes, name, t.k4q, t.k4s, woff, n, src_off) + kind = "k4 transcode (Q4_K)" } elif (fmt == KqFmt.k5) { if (gguf_tensor_type(m, name) == GGML_TYPE_Q5_K) { gguf_transcode_q5k(m, bytes, name, t.k5q, t.k5s, woff, n, src_off) + kind = "k5 transcode (Q5_K)" } else { // Q5_0 source: no exact embedding exists, so dequant and re-encode (lossy) scratch |> resize(n) gguf_read_tensor_f32(m, bytes, name, scratch, 0l, n, src_off) quantize_k5_plane(scratch, n, t.k5q, t.k5s, woff) + kind = "k5 RE-ENCODE (Q5_0 dequant+quant)" } } elif (fmt == KqFmt.k6) { gguf_transcode_q6k(m, bytes, name, t.k6q, t.k6s, woff, n, src_off) + kind = "k6 transcode (Q6_K)" } elif (fmt == KqFmt.q40) { gguf_transcode_q40(m, bytes, name, t.q40q, t.q40s, woff, n, src_off) + kind = "q40 transcode (Q4_0)" } elif (fmt == KqFmt.q51) { gguf_transcode_q51(m, bytes, name, t.q51q, t.q51s, woff, n, src_off) + kind = "q51 transcode (Q5_1)" } elif (t.quant == QuantMode.q8) { let gt = gguf_tensor_type(m, name) if (gt == GGML_TYPE_Q8_0) { gguf_transcode_q8_0(m, bytes, name, t.qblob, t.qscales, woff, woff / 32l, n, src_off) + kind = "q8 transcode (Q8_0 verbatim)" } elif (gt == GGML_TYPE_Q5_0) { gguf_transcode_q5_0_to_q8(m, bytes, name, t.qblob, t.qscales, woff, woff / 32l, n, src_off) + kind = "q8 transcode (Q5_0 exact)" } else { scratch |> resize(n) gguf_read_tensor_f32(m, bytes, name, scratch, 0l, n, src_off) quantize_q8_0_into(scratch, n, t.qblob, t.qscales, woff, woff / 32l) + kind = "q8 REQUANT (dequant+quant)" } } elif (t.quant == QuantMode.q4_0) { if (gguf_tensor_type(m, name) == GGML_TYPE_Q4_0) { gguf_transcode_q4_0(m, bytes, name, t.q4blob, t.q4scales, woff / 2l, woff / 32l, n, src_off) + kind = "q4_0 transcode (Q4_0 verbatim)" } else { scratch |> resize(n) gguf_read_tensor_f32(m, bytes, name, scratch, 0l, n, src_off) quantize_q4_0_into(scratch, n, t.q4blob, t.q4scales, woff / 2l, woff / 32l) + kind = "q4_0 REQUANT (dequant+quant)" } } else { gguf_read_tensor_f32(m, bytes, name, t.wblob, woff, n, src_off) } + conv_note(kind, t0, n) } // qwen3next-layout deltanet big weights: same planes as the qwen35 arm, but the file's v-indexed @@ -1664,7 +1977,7 @@ def private moe_layer_kq(t : Model; l : int64) : bool { } // Are layer l's deltanet projection planes all q8? (Metal and the GPU tiers bind/upload dn q8 only.) -def private dn_layer_q8(t : Model; l : int64) : bool { +def dn_layer_q8(t : Model; l : int64) : bool { return fmt_at(t.dnqkv_fmt, l) == KqFmt.q8 && fmt_at(t.dngate_fmt, l) == KqFmt.q8 && fmt_at(t.dnout_fmt, l) == KqFmt.q8 } @@ -1709,236 +2022,6 @@ def private wscale_convert_f16(var t : Model) { // ===== GPU MoE tier upload (post-prepare, loader-agnostic) ===== -// Gather one PREPARED Q8 weight stack into the GPU tier's two row-major planes — int8 quants + -// f16 halfword scales. slice_rows is the repack unit (each expert slice interleaves independently). -// Reads whatever the active backend prepared (grp interleave or plain row-major). -def moe_gpu_gather_stack(t : Model; woff : int64; n, rows, slice_rows : int64; mr, kgroup, wbias : int64; - var wq : array; var ws : array) { - let nb = n / 32l - assert(rows * n <= 2147483647l, "GPU MoE stack exceeds the int32 gather-buffer rail") - wq |> resize(int(rows * n)) - ws |> resize(int(rows * nb * 2l)) - assert(wbias == 0l || wbias == 128l, "GPU MoE gather: unknown q8 plane bias") - let nslices = int(rows / slice_rows) - unsafe { - let wp = addr(t.qblob[woff]) - var s16p : uint16 const? = null - var s32p : float const? = null - if (t.wscale_f16) { - s16p = addr(t.qscales16[woff / 32l]) - } else { - s32p = addr(t.qscales[woff / 32l]) - } - var wqp = addr(wq[0]) - var wsp = addr(ws[0]) - let njobs = is_job_que_available() ? min(nslices, 4 * (get_total_hw_jobs() + 1)) : 1 - maybe_parallel_for(0, nslices, njobs) $(sb, se) { - unsafe { - let ng = slice_rows / mr - for (s in range(sb, se)) { - let swoff = int64(s) * slice_rows * n - let ssoff = int64(s) * slice_rows * nb - for (rr in range64(slice_rows)) { - let grouped = rr < ng * mr - let g = rr / mr - let r = rr % mr - for (bi in range64(nb)) { - // scale: group-interleaved [g][bi][r] or the row-major tail; the f32 - // fallback takes the same clamped rounding wscale_convert_f16 applies - let si = grouped ? ssoff + (g * nb + bi) * mr + r : ssoff + rr * nb + bi - wsp[ssoff + rr * nb + bi] = (s16p != null ? s16p[si] - : uint16(f32_to_f16(clamp(s32p[si], -65504.0, 65504.0)))) - var dst = wqp + swoff + rr * n + bi * 32l - if (grouped) { - let gbase = swoff + g * mr * n + r * kgroup - var kg = (bi * 32l) / kgroup - var o = 0l - for (_kk in range64(32l / kgroup)) { - let src = gbase + kg * kgroup * mr - for (j in range64(kgroup)) { - dst[o + j] = uint8(int(wp[src + j]) ^ int(wbias)) - } - kg++ - o += kgroup - } - } else { - let src = swoff + rr * n + bi * 32l - for (j in range64(32l)) { - dst[j] = uint8(wp[src + j]) - } - } - } - } - } - } - } - } -} - -// Disk-order K-quant plane accessors for the kq gather's un-repacked/tail rows (source of -// truth: the private twins beside repack_k4/k5/k6_grp in dasllama_math_gen.das — replicated -// because math_gen mounts conditionally behind the dasLLVM guard). - -// low 4 bits of weight k off a disk-order k4/k5 nibble row (128 bytes per superblock) -def private kq_disk_nib45(rowq : uint8 const?; k : int64) : uint { - let js = k / 64l - let rem = k % 64l - let b = uint(unsafe(rowq[js * 32l + rem % 32l])) - return rem < 32l ? b & 15u : b >> 4u -} - -// the 5th bit of weight k off a disk-order k5 qh row (32 bytes per superblock) -def private kq_disk_hbit5(rowh : uint8 const?; k : int64) : uint { - let js = k / 64l - let rem = k % 64l - return (uint(unsafe(rowh[rem % 32l])) >> uint(2l * js + (rem < 32l ? 0l : 1l))) & 1u -} - -// low 4 bits of weight k off a disk-order k6 ql row (128 bytes per superblock) -def private kq_disk_nib6(rowq : uint8 const?; k : int64) : uint { - let h = k / 128l - let g = (k % 128l) / 32l - let b = uint(unsafe(rowq[h * 64l + (g % 2l) * 32l + k % 32l])) - return g < 2l ? b & 15u : b >> 4u -} - -// Gather one PREPARED K-quant weight stack into the GPU tier's device planes — quant payloads -// row-major per superblock in the k/k+16 nibble pairing, plus DECODED 20B scale rows. Reads -// whatever the load prepared (grp interleave or disk-order); device bytes are identical either way. -def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice_rows : int64; - repacked : bool; mr : int64; var wq : array; var ws : array) { - let nsb = n / 256l - let qsb = fmt == KqFmt.k5 ? 160l : (fmt == KqFmt.k6 ? 192l : 128l) // k4 and q40 share the 128B nibble region - let dssb = fmt == KqFmt.k6 ? 18l : (fmt == KqFmt.q40 ? 16l : 20l) // disk/plane scale stride; device rows are 20B for all - assert(n % 256l == 0l, "GPU MoE kq gather: row length must be a superblock multiple") - assert(rows * nsb * qsb <= 2147483647l, "GPU MoE kq stack exceeds the int32 gather-buffer rail") - wq |> resize(int(rows * nsb * qsb)) - ws |> resize(int(rows * nsb * 20l)) - let nslices = int(rows / slice_rows) - let qrow = nsb * qsb - let srow = nsb * dssb - let ng = repacked ? slice_rows / mr : 0l - unsafe { - let sb0 = woff / 256l - let qp = (fmt == KqFmt.k4 ? addr(t.k4q[sb0 * 128l]) - : (fmt == KqFmt.k5 ? addr(t.k5q[sb0 * 160l]) - : (fmt == KqFmt.k6 ? addr(t.k6q[sb0 * 192l]) : addr(t.q40q[sb0 * 128l])))) - let sp = (fmt == KqFmt.k4 ? addr(t.k4s[sb0 * 20l]) - : (fmt == KqFmt.k5 ? addr(t.k5s[sb0 * 20l]) - : (fmt == KqFmt.k6 ? addr(t.k6s[sb0 * 18l]) : addr(t.q40s[sb0 * 16l])))) - var wqp = addr(wq[0]) - var wsp = addr(ws[0]) - let njobs = is_job_que_available() ? min(nslices, 4 * (get_total_hw_jobs() + 1)) : 1 - maybe_parallel_for(0, nslices, njobs) $(ssb, sse) { - unsafe { - for (s in range(ssb, sse)) { - let sliceQ = int64(s) * slice_rows * qrow - let sliceS = int64(s) * slice_rows * srow - for (ri in range64(slice_rows)) { - let rr = int64(s) * slice_rows + ri - let grouped = repacked && ri < ng * mr - let g = ri / mr - let r = ri % mr - for (sbi in range64(nsb)) { - var dq = wqp + rr * qrow + sbi * qsb - var dsc = wsp + (rr * nsb + sbi) * 20l - if (grouped) { - let gq = qp + sliceQ + g * mr * qrow + sbi * qsb * mr - let gs = sp + sliceS + g * mr * srow + sbi * dssb * mr - for (m in range64(128l)) { // nibble region: verbatim per (blk, j, t) - dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] - } - if (fmt == KqFmt.k5) { - for (bj in range64(32l)) { - dq[128l + bj] = gq[128l * mr + bj * mr + r] - } - } elif (fmt == KqFmt.k6) { - for (p in range64(64l)) { - dq[128l + p] = gq[128l * mr + ((p / 4l) * mr + r) * 4l + p % 4l] - } - } - if (fmt == KqFmt.k6) { - for (idx in range64(16l)) { - dsc[idx] = gs[idx * mr + r] - } - dsc[16] = gs[16l * mr + 2l * r] - dsc[17] = gs[16l * mr + 2l * r + 1l] - dsc[18] = uint8(0) - dsc[19] = uint8(0) - } elif (fmt == KqFmt.q40) { - for (blk in range64(8l)) { // 8 per-block f16 d, mr-interleaved - dsc[blk * 2l] = gs[blk * 2l * mr + 2l * r] - dsc[blk * 2l + 1l] = gs[blk * 2l * mr + 2l * r + 1l] - } - for (idx in range64(16l, 20l)) { - dsc[idx] = uint8(0) - } - } else { - dsc[0] = gs[2l * r] - dsc[1] = gs[2l * r + 1l] - dsc[2] = gs[2l * mr + 2l * r] - dsc[3] = gs[2l * mr + 2l * r + 1l] - for (blk in range64(8l)) { - dsc[4l + blk] = gs[4l * mr + blk * mr + r] - dsc[12l + blk] = gs[12l * mr + blk * mr + r] - } - } - } else { - let rq = qp + sliceQ + ri * qrow + sbi * qsb - let rs = sp + sliceS + ri * srow + sbi * dssb - if (fmt == KqFmt.q40) { - for (m in range64(128l)) { // Q4_0 bytes already pair k / k+16 - dq[m] = rq[m] - } - } else { - for (m in range64(128l)) { // re-pair disk k/k+32 nibbles to k/k+16 - let k = (m / 16l) * 32l + m % 16l - let lo = fmt == KqFmt.k6 ? kq_disk_nib6(rq, k) : kq_disk_nib45(rq, k) - let hi = fmt == KqFmt.k6 ? kq_disk_nib6(rq, k + 16l) : kq_disk_nib45(rq, k + 16l) - dq[m] = uint8(lo | (hi << 4u)) - } - } - if (fmt == KqFmt.k5) { - for (bj in range64(32l)) { - let k = (bj / 4l) * 32l + (bj % 4l) * 4l - var hb = 0u - for (tt in range64(4l)) { - hb |= kq_disk_hbit5(rq + 128l, k + tt) << uint(tt) - hb |= kq_disk_hbit5(rq + 128l, k + 16l + tt) << uint(4l + tt) - } - dq[128l + bj] = uint8(hb) - } - } elif (fmt == KqFmt.k6) { - for (p in range64(64l)) { - dq[128l + p] = rq[128l + p] - } - } - if (fmt == KqFmt.k6 || fmt == KqFmt.q40) { - for (idx in range64(dssb)) { - dsc[idx] = rs[idx] - } - for (idx in range64(dssb, 20l)) { - dsc[idx] = uint8(0) - } - } else { - for (idx in range64(4l)) { // f16 d + f16 dmin verbatim - dsc[idx] = rs[idx] - } - for (blk in range64(8l)) { // decode the 6-bit sc/mn packing - let sm = k4_sc_mn(rs + 4l, blk) - dsc[4l + blk] = uint8(sm.x) - dsc[12l + blk] = uint8(sm.y) - } - } - } - } - } - } - } - } - } -} - def private moe_gpu_fmt_kq(f : KqFmt) : bool => f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.q40 // a plane the dense rail can serve: a format the tier has kernels for, repacked if native-kq @@ -1954,13 +2037,21 @@ def private dense_shexp_uploadable(t : Model; l : int64) : bool { && dense_plane_ok(t, fmt_at(t.w3_fmt, l))) } -// Gather one stack through its format's reader (q8 planes or native kq planes) and hand it to -// the tier's upload hook with the format tag. +// The device format of an EXPERT stack: q8 rides the cm2 native-block layout (q8n, fmt 6) when +// the armed tier asks for it; every other format ships as-is. Upload and dispatch must agree on +// the answer — both go through this one helper. +def private moe_gpu_expert_fmt(f : KqFmt) : KqFmt => f == KqFmt.q8 && moe_gpu_expert_q8n() ? KqFmt.q8n : f + +// Gather one stack through its format's reader (q8 planes, q8n interleaved blocks, or native kq +// planes) and hand it to the tier's upload hook with the format tag. def private moe_gpu_gather_upload(t : Model; f : KqFmt; woff : int64; n, rows, slice_rows : int64; mr, kgroup, wbias : int64; var wq : array; var ws : array; stream : bool = false) : bool { if (f == KqFmt.q8) { moe_gpu_gather_stack(t, woff, n, rows, slice_rows, mr, kgroup, wbias, wq, ws) + } elif (f == KqFmt.q8n) { + moe_gpu_gather_stack_q8n(t, woff, n, rows, slice_rows, mr, kgroup, wbias, wq) + ws |> resize(0) // no scale plane — the tier takes an EMPTY ws for fmt 6 } else { let kmr = t.kq_repacked ? (f == KqFmt.k4 ? t.kq_repack_mr4 : (f == KqFmt.k5 ? t.kq_repack_mr5 @@ -2085,7 +2176,10 @@ def private resident_place(t : Model; f : KqFmt; woff : int64; n, rows : int64; mr, kgroup, wbias : int64) : int64 { var wq : array var ws : array - if (f == KqFmt.q8) { + if (f == KqFmt.q8n) { + moe_gpu_gather_stack_q8n(t, woff, n, rows, rows, mr, kgroup, wbias, wq) + // no scale plane — the tier takes an EMPTY ws for fmt 6 + } elif (f == KqFmt.q8) { moe_gpu_gather_stack(t, woff, n, rows, rows, mr, kgroup, wbias, wq, ws) } else { let kmr = t.kq_repacked ? (f == KqFmt.k4 ? t.kq_repack_mr4 @@ -2099,11 +2193,34 @@ def private resident_place(t : Model; f : KqFmt; woff : int64; n, rows : int64; return blk } -def private rdec_blocks_for(n, rows : int64; f : KqFmt) : int64 => rows * (n / (f == KqFmt.q8 ? 32l : 256l)) +def private rdec_blocks_for(n, rows : int64; f : KqFmt) : int64 => ( + rows * (n / (f == KqFmt.q8 || f == KqFmt.q8n ? Q8_BLOCK_ELEMS : KQ_SUPERBLOCK_ELEMS))) + +// cm2 (mode 4) places dense q8 planes as q8n — decode-in-load prefill GEMMs, q8n decode GEMV. +// Promotion is per activation-group: members sharing one B-side image must agree on its form, +// so {q,k,v} and {gate,up} promote only whole; wo / down / cls are singletons. +def private resident_layer_fmts(t : Model; l : int64) : tuple { + var fq = fmt_at(t.wq_fmt, l) + var fk = fmt_at(t.wk_fmt, l) + var fv = fmt_at(t.wv_fmt, l) + if (fq == KqFmt.q8 && fk == KqFmt.q8 && fv == KqFmt.q8 && moe_gpu_expert_q8n()) { + fq = KqFmt.q8n + fk = KqFmt.q8n + fv = KqFmt.q8n + } + var f1 = fmt_at(t.w1_fmt, l) + var f3 = fmt_at(t.w3_fmt, l) + if (f1 == KqFmt.q8 && f3 == KqFmt.q8 && moe_gpu_expert_q8n()) { + f1 = KqFmt.q8n + f3 = KqFmt.q8n + } + return (fq = fq, fk = fk, fv = fv, fo = moe_gpu_expert_fmt(fmt_at(t.wo_fmt, l)), + f1 = f1, f3 = f3, f2 = moe_gpu_expert_fmt(fmt_at(t.w2_fmt, l))) +} //! Reserve arenas + place every dense plane + wire the resident decode driver. Returns false (and //! leaves the driver unarmed) if the model is not a servable dense shape or does not fit at seq_cap. -def resident_upload(var t : Model; seq_cap : int64) : bool { +def resident_upload(t : Model; seq_cap : int64) : bool { g_rdec_active = false if (!moe_gpu_resident_installed() || t.quant != QuantMode.q8 || t.config.n_expert > 0l) { return false @@ -2134,7 +2251,7 @@ def resident_upload(var t : Model; seq_cap : int64) : bool { return false } let neox = c.rope_neox - let cls_fmt = c.shared_weights ? t.emb_fmt : t.wcls_fmt + let cls_fmt = moe_gpu_expert_fmt(c.shared_weights ? t.emb_fmt : t.wcls_fmt) // reject non-uniform per-layer geometry (the recorder assumes fixed dims across layers) for (l in range64(c.n_layers)) { if (layer_hidden(t, l) != hid0 || layer_head_size(c, l) != hs || layer_kv_dim(c, l) != kvd @@ -2148,13 +2265,14 @@ def resident_upload(var t : Model; seq_cap : int64) : bool { var planes : array> planes |> reserve(int(c.n_layers * 7l + 1l)) for (l in range64(c.n_layers)) { - planes |> push((n = dim, rows = qd, f = fmt_at(t.wq_fmt, l))) - planes |> push((n = dim, rows = kvd, f = fmt_at(t.wk_fmt, l))) - planes |> push((n = dim, rows = kvd, f = fmt_at(t.wv_fmt, l))) - planes |> push((n = qd, rows = dim, f = fmt_at(t.wo_fmt, l))) - planes |> push((n = dim, rows = hid0, f = fmt_at(t.w1_fmt, l))) - planes |> push((n = dim, rows = hid0, f = fmt_at(t.w3_fmt, l))) - planes |> push((n = hid0, rows = dim, f = fmt_at(t.w2_fmt, l))) + let lf = resident_layer_fmts(t, l) + planes |> push((n = dim, rows = qd, f = lf.fq)) + planes |> push((n = dim, rows = kvd, f = lf.fk)) + planes |> push((n = dim, rows = kvd, f = lf.fv)) + planes |> push((n = qd, rows = dim, f = lf.fo)) + planes |> push((n = dim, rows = hid0, f = lf.f1)) + planes |> push((n = dim, rows = hid0, f = lf.f3)) + planes |> push((n = hid0, rows = dim, f = lf.f2)) } planes |> push((n = dim, rows = c.vocab_size, f = cls_fmt)) for (pl in planes) { @@ -2183,21 +2301,21 @@ def resident_upload(var t : Model; seq_cap : int64) : bool { } // place every plane, wire each layer for (l in range64(c.n_layers)) { - let bq = resident_place(t, fmt_at(t.wq_fmt, l), t.wq_offs[l], dim, qd, mr, kg, wb) - let bk = resident_place(t, fmt_at(t.wk_fmt, l), t.wk_offs[l], dim, kvd, mr, kg, wb) - let bv = resident_place(t, fmt_at(t.wv_fmt, l), t.wv_offs[l], dim, kvd, mr, kg, wb) - let bo = resident_place(t, fmt_at(t.wo_fmt, l), t.wo_offs[l], qd, dim, mr, kg, wb) - let b1 = resident_place(t, fmt_at(t.w1_fmt, l), t.w1_offs[l], dim, hid0, mr, kg, wb) - let b3 = resident_place(t, fmt_at(t.w3_fmt, l), t.w3_offs[l], dim, hid0, mr, kg, wb) - let b2 = resident_place(t, fmt_at(t.w2_fmt, l), t.w2_offs[l], hid0, dim, mr, kg, wb) + let lf = resident_layer_fmts(t, l) + let bq = resident_place(t, lf.fq, t.wq_offs[l], dim, qd, mr, kg, wb) + let bk = resident_place(t, lf.fk, t.wk_offs[l], dim, kvd, mr, kg, wb) + let bv = resident_place(t, lf.fv, t.wv_offs[l], dim, kvd, mr, kg, wb) + let bo = resident_place(t, lf.fo, t.wo_offs[l], qd, dim, mr, kg, wb) + let b1 = resident_place(t, lf.f1, t.w1_offs[l], dim, hid0, mr, kg, wb) + let b3 = resident_place(t, lf.f3, t.w3_offs[l], dim, hid0, mr, kg, wb) + let b2 = resident_place(t, lf.f2, t.w2_offs[l], hid0, dim, mr, kg, wb) // a placement can fail (arena exhausted); fail closed to the CPU loop rather than wire a -1 block if (bq < 0l || bk < 0l || bv < 0l || bo < 0l || b1 < 0l || b3 < 0l || b2 < 0l) { to_log(LOG_WARNING, "dasLLAMA: resident driver — arena placement failed at layer {l}, declining\n") return false } rdec_set_layer(l, bq, bk, bv, bo, b1, b3, b2, - int(fmt_at(t.wq_fmt, l)), int(fmt_at(t.wk_fmt, l)), int(fmt_at(t.wv_fmt, l)), int(fmt_at(t.wo_fmt, l)), - int(fmt_at(t.w1_fmt, l)), int(fmt_at(t.w3_fmt, l)), int(fmt_at(t.w2_fmt, l))) + int(lf.fq), int(lf.fk), int(lf.fv), int(lf.fo), int(lf.f1), int(lf.f3), int(lf.f2)) } let bcls = resident_place(t, cls_fmt, t.wcls_off, dim, c.vocab_size, mr, kg, wb) if (bcls < 0l) { @@ -2541,7 +2659,7 @@ def moe_gpu_drop_model() { // Upload the offloaded layers' expert stacks to the GPU tier from the PREPARED planes. Called at // the end of every load rail so DASLLAMA_GPU_MOE_LAYERS behaves identically however the model // arrived. Layers go resident from the end while the tier's VRAM budget holds. -def moe_gpu_upload_resident(var t : Model) { +def moe_gpu_upload_resident(t : Model) { set_moe_gpu_from_layer(t.config.n_layers) // nothing resident until an upload lands set_moe_gpu_stream_from(t.config.n_layers) set_moe_gpu_total_layers(t.config.n_layers) @@ -2565,6 +2683,25 @@ def moe_gpu_upload_resident(var t : Model) { && !gpu_want_shexp() && !gpu_want_qkv() && !gpu_want_cls())) { return // tier absent or nothing requested } + // size the tier's streamed-slot reserve from THIS model's largest expert layer — a fixed + // reserve wastes VRAM or asserts on big-expert geometries (GLM Air, Qwen1.5-MoE); must + // precede the first upload (the tier carves the reserve at its lazy init) + if (t.config.n_expert > 0l) { + var sgu = 0l + var sdn = 0l + for (sl in range64(t.config.n_layers + t.config.n_layer_nextn)) { + if (sl < t.config.n_layer_dense_lead || long_length(t.we1_offs) <= sl + || t.we1_offs[sl] < 0l || t.we2_offs[sl] < 0l || t.we3_offs[sl] < 0l) { + continue + } + let sne = t.config.n_expert + let snfe = t.config.n_ff_exp + sgu = max(sgu, moe_gpu_plane_bytes(t.config.dim, sne * snfe, int(fmt_at(t.we1_fmt, sl)))) + sgu = max(sgu, moe_gpu_plane_bytes(t.config.dim, sne * snfe, int(fmt_at(t.we3_fmt, sl)))) + sdn = max(sdn, moe_gpu_plane_bytes(snfe, sne * t.config.dim, int(fmt_at(t.we2_fmt, sl)))) + } + set_moe_gpu_stream_need(sgu > 0l ? 2l * sgu + sdn : 0l) + } // dense models that fully fit: try the whole-stack resident driver FIRST — it subsumes every // rail below (all planes + classifier resident, one-submit device prefill + mirror decode). // Any decline (shape, fit, placement) falls through to the per-op rails unchanged. @@ -2876,13 +3013,13 @@ def moe_gpu_upload_resident(var t : Model) { break } var got = 0 - if (moe_gpu_gather_upload(t, f1, t.we1_offs[l], dim, ne * nfe, nfe, mr, kgroup, wbias, wq, ws)) { + if (moe_gpu_gather_upload(t, moe_gpu_expert_fmt(f1), t.we1_offs[l], dim, ne * nfe, nfe, mr, kgroup, wbias, wq, ws)) { got++ } - if (got == 1 && moe_gpu_gather_upload(t, f3, t.we3_offs[l], dim, ne * nfe, nfe, mr, kgroup, wbias, wq, ws)) { + if (got == 1 && moe_gpu_gather_upload(t, moe_gpu_expert_fmt(f3), t.we3_offs[l], dim, ne * nfe, nfe, mr, kgroup, wbias, wq, ws)) { got++ } - if (got == 2 && moe_gpu_gather_upload(t, f2, t.we2_offs[l], nfe, ne * dim, dim, mr, kgroup, wbias, wq, ws)) { + if (got == 2 && moe_gpu_gather_upload(t, moe_gpu_expert_fmt(f2), t.we2_offs[l], nfe, ne * dim, dim, mr, kgroup, wbias, wq, ws)) { got++ } if (got < 3) { @@ -2915,13 +3052,13 @@ def moe_gpu_upload_resident(var t : Model) { break } var got = 0 - if (moe_gpu_gather_upload(t, f1, t.we1_offs[l], dim, ne * nfe, nfe, mr, kgroup, wbias, wq, ws, [stream = true])) { + if (moe_gpu_gather_upload(t, moe_gpu_expert_fmt(f1), t.we1_offs[l], dim, ne * nfe, nfe, mr, kgroup, wbias, wq, ws, [stream = true])) { got++ } - if (got == 1 && moe_gpu_gather_upload(t, f3, t.we3_offs[l], dim, ne * nfe, nfe, mr, kgroup, wbias, wq, ws, [stream = true])) { + if (got == 1 && moe_gpu_gather_upload(t, moe_gpu_expert_fmt(f3), t.we3_offs[l], dim, ne * nfe, nfe, mr, kgroup, wbias, wq, ws, [stream = true])) { got++ } - if (got == 2 && moe_gpu_gather_upload(t, f2, t.we2_offs[l], nfe, ne * dim, dim, mr, kgroup, wbias, wq, ws, [stream = true])) { + if (got == 2 && moe_gpu_gather_upload(t, moe_gpu_expert_fmt(f2), t.we2_offs[l], nfe, ne * dim, dim, mr, kgroup, wbias, wq, ws, [stream = true])) { got++ } if (got < 3) { @@ -2946,498 +3083,9 @@ def moe_gpu_upload_resident(var t : Model) { } } -//! Load a llama-architecture GGUF into the split layout at the requested precision. -// RepackReg: one mr-aligned row-chunk of a weight region. fmt: 0=q8, 1=mx4, 4/5/6=kq planes. -// Chunking splits huge regions (the classifier); 1024 is a multiple of every backend mr. - -// ===== metal-flavor blob transform (the blob-only .dlim redesign) ===== - -def private metal_blob_fmt_at(a : array; l : int64) : KqFmt => empty(a) ? KqFmt.q8 : a[l] - -// A q8 region's blob bind offset (off/32)*34 lands 16B-aligned iff off % 256 == 0; a k6 -// region's d-plane bind (NSB*16 + (off/256)*2) needs 4B alignment iff off % 512 == 0. -// k4/k5/q40 quant + scale binds are 16B-multiples by construction. -def private metal_blob_off_ok(off : int64; fmt : KqFmt) : bool { - if (off < 0l) return true - if (fmt == KqFmt.q8) return off % 256l == 0l - if (fmt == KqFmt.k6) return off % 512l == 0l - if (fmt == KqFmt.q51) return off % 128l == 0l // (off/32)*20 and (off/32)*4 both 16B-aligned - return true -} - -def private metal_blob_layout_ok(t : Model) : bool { - let c = t.config - if (long_length(t.wq_offs) < c.n_layers) { - return false - } - // a pure-MoE file (qwen3moe) carries no dense FFN arrays — its expert stacks bind instead - let has_ffn = long_length(t.w1_offs) >= c.n_layers - let has_exps = (c.moe && long_length(t.we1_offs) >= c.n_layers && - long_length(t.we2_offs) >= c.n_layers && long_length(t.we3_offs) >= c.n_layers) - let has_dn = c.recr_mask != 0ul && long_length(t.dnqkv_offs) >= c.n_layers - if (!has_ffn && !has_exps) { - return false - } - for (l in range64(c.n_layers)) { - // recurrent layers (Wave D): no attention tensors (offs -1, off_ok true); dn planes must be q8 - let dn_bad = (has_dn && layer_is_recurrent(c, l) && - (!dn_layer_q8(t, l) || - !metal_blob_off_ok(t.dnqkv_offs[l], KqFmt.q8) || - !metal_blob_off_ok(t.dngate_offs[l], KqFmt.q8) || - !metal_blob_off_ok(t.dnbeta_offs[l], KqFmt.q8) || - !metal_blob_off_ok(t.dnalpha_offs[l], KqFmt.q8) || - !metal_blob_off_ok(t.dnout_offs[l], KqFmt.q8))) - if (dn_bad || - !metal_blob_off_ok(t.wq_offs[l], metal_blob_fmt_at(t.wq_fmt, l)) || - !metal_blob_off_ok(t.wk_offs[l], metal_blob_fmt_at(t.wk_fmt, l)) || - !metal_blob_off_ok(t.wv_offs[l], metal_blob_fmt_at(t.wv_fmt, l)) || - !metal_blob_off_ok(t.wo_offs[l], metal_blob_fmt_at(t.wo_fmt, l)) || - (has_ffn && (!metal_blob_off_ok(t.w1_offs[l], metal_blob_fmt_at(t.w1_fmt, l)) || - !metal_blob_off_ok(t.w2_offs[l], metal_blob_fmt_at(t.w2_fmt, l)) || - !metal_blob_off_ok(t.w3_offs[l], metal_blob_fmt_at(t.w3_fmt, l)))) || - (has_exps && (!metal_blob_off_ok(t.we1_offs[l], metal_blob_fmt_at(t.we1_fmt, l)) || - !metal_blob_off_ok(t.we2_offs[l], metal_blob_fmt_at(t.we2_fmt, l)) || - !metal_blob_off_ok(t.we3_offs[l], metal_blob_fmt_at(t.we3_fmt, l))))) { - return false - } - } - // shexp planes: always q8, contiguous at base + l*she — base + stride covers every layer - if (c.n_ff_shexp > 0l) { - let she = c.dim * c.n_ff_shexp - if ((she % 256l) != 0l || - !metal_blob_off_ok(t.wsh1_off, KqFmt.q8) || - !metal_blob_off_ok(t.wsh2_off, KqFmt.q8) || - !metal_blob_off_ok(t.wsh3_off, KqFmt.q8)) { - return false - } - } - // E-series PLE gate/proj q8 planes bind per layer; the token table / model-proj stay CPU-side - if (has_ple(c)) { - let ples = c.dim * c.n_embd_per_layer - if ((ples % 256l) != 0l || - !metal_blob_off_ok(t.ple_gate_off, KqFmt.q8) || - !metal_blob_off_ok(t.ple_proj_off, KqFmt.q8)) { - return false - } - } - let cls_fmt = c.shared_weights ? t.emb_fmt : t.wcls_fmt - return (metal_blob_off_ok(t.wcls_off, cls_fmt) && - (!t.cls_q8 || metal_blob_off_ok(t.emb_q8_off, KqFmt.q8))) -} - -//! Convert a planar q8-mode Model to the metal-blob form IN PLACE (see the Model field doc). -//! False = the model cannot take the blob form (not q8, repacked planes, or a bind-alignment -//! violation) — the caller keeps the planar model. CPU inference on the converted model panics. -def convert_model_to_metal_blob(var t : Model) : bool { - if (t.metal_blob) { - return true - } - if (t.quant != QuantMode.q8 || t.kq_repacked || t.mx4_repacked || kernel_backend_needs_repack(active_kernel_backend())) { - return false - } - if (t.image_map != null) { - // image-backed planes are borrowed READ-ONLY views — the in-place k4s/k5s/k6s - // rewrites would fault. Transform an owned load (load_model_ / DASLLAMA_IMAGE=0). - to_log(LOG_WARNING, "dasLLAMA: metal-blob transform declined — the model is image-mapped (borrowed planes); transform an owned gguf load\n") - return false - } - if (!metal_blob_layout_ok(t)) { - to_log(LOG_WARNING, "dasLLAMA: metal-blob transform declined — a weight offset violates the GPU bind alignment\n") - return false - } - let ts = ref_time_ticks() - let nb = long_length(t.qblob) / 32l - if (nb > 0l) { - t.mblob |> resize(nb * 34l) - let s16 = t.wscale_f16 - unsafe { - var dp = addr(t.mblob[0]) - let qp = addr < int8 const? >(t.qblob[0]) - var sp : float const? - var sp16 : uint16 const? - if (s16) { - sp16 = addr(t.qscales16[0]) - } else { - sp = addr(t.qscales[0]) - } - let nch = int(clamp(nb / 262144l, 1l, 64l)) - maybe_parallel_for(0, int(nb / 32l) + (nb % 32l != 0l ? 1 : 0), is_job_que_available() ? nch : 1) $(cb, ce) { - unsafe { - for (b in range64(int64(cb) * 32l, min(int64(ce) * 32l, nb))) { - var ph = reinterpret(dp + b * 34l) - ph[0] = s16 ? sp16[b] : uint16(f32_to_f16(clamp(sp[b], -65504.0, 65504.0))) - memcpy(reinterpret(dp + b * 34l + 2l), reinterpret(qp + b * 32l), 32) - } - } - } - } - } - delete t.qblob - delete t.qscales - delete t.qscales16 - // k4/k5: the 16B verbatim disk scale block per superblock — the 4B pad is CPU-repack - // scratch (see the plane-layout comment in dasllama_gguf) and a blob model never repacks - if (!empty(t.k4s)) { - compact_kq_scale_strip(t.k4s) - } - if (!empty(t.k5s)) { - compact_kq_scale_strip(t.k5s) - } - // k6: the GPU split form — [nsb x 16B sub-scale strips][nsb x f16 d halfwords] - if (!empty(t.k6s)) { - var k6c : array - let nsb = long_length(t.k6s) / K6_SSB - k6c |> resize(nsb * K6_SSB) - unsafe { - var dp = addr(k6c[0]) - let sp = addr < uint8 const? >(t.k6s[0]) - for (sb in range64(nsb)) { - memcpy(reinterpret(dp + sb * 16l), reinterpret(sp + sb * 18l), 16) - dp[nsb * 16l + sb * 2l] = sp[sb * 18l + 16l] - dp[nsb * 16l + sb * 2l + 1l] = sp[sb * 18l + 17l] - } - } - t.k6s <- k6c - } - t.metal_blob = true - to_log(LOG_INFO, "dasLLAMA: metal-blob transform ({nb} blocks) in {get_time_usec(ts) / 1000} ms\n") - return true -} - -// 20B -> 16B in place: keep bytes [0..16) of every superblock strip, then shrink. Forward -// byte moves are safe — every destination sits below its source (regions can overlap). -def private compact_kq_scale_strip(var ks : array) { - let nsb = long_length(ks) / 20l - unsafe { - var p = addr(ks[0]) - for (sb in range64(1l, nsb)) { // sb 0 already sits at 0 - for (i in range64(16l)) { - p[sb * 16l + i] = p[sb * 20l + i] - } - } - } - ks |> resize(nsb * 16l) -} - -// A repack work item: one mr-aligned row-chunk of a weight region. fmt: 0 = q8 (qblob/qscales), -// 1 = mx4 (mxq/mxs), 4/5/6 = the kq plane pairs. Chunking splits huge regions (the classifier) for -// load balance; 1024 is a multiple of every backend mr, so row groups never straddle a boundary. -struct private RepackReg { - off : int64 - n : int64 - d : int64 - fmt : int -} - -def private push_repack(var regs : array; fmt : int; off, n, d : int64) { - var r0 = 0l - while (r0 < d) { - let rc = min(1024l, d - r0) - regs |> push(RepackReg(off = off + r0 * n, n = n, d = rc, fmt = fmt)) - r0 += rc - } -} - -// Run every collected repack across the jobque — regions are disjoint, so any lane count is -// bit-exact. Workers touch hoisted raw pointers + the hoisted backend fn values only. -def private repack_regions(var t : Model; regs : array) { - if (empty(regs)) { - return - } - let rq8 = g_repack_q8q8 - let rkq = g_repack_kq - let rmx = g_repack_mx4 - let rq51 = g_repack_q51 - unsafe { - var qbp : int8? = null - var qsp : float? = null - var mxqp : uint8? = null - var mxsp : uint8? = null - var k4qp : uint8? = null - var k4sp : uint8? = null - var k5qp : uint8? = null - var k5sp : uint8? = null - var k6qp : uint8? = null - var k6sp : uint8? = null - var q40qp : uint8? = null - var q40sp : uint8? = null - var q51qp : uint8? = null - var q51sp : uint8? = null - if (!empty(t.qblob)) { qbp = addr(t.qblob[0]) } - if (!empty(t.qscales)) { qsp = addr(t.qscales[0]) } - if (!empty(t.mxq)) { mxqp = addr(t.mxq[0]) } - if (!empty(t.mxs)) { mxsp = addr(t.mxs[0]) } - if (!empty(t.k4q)) { k4qp = addr(t.k4q[0]) } - if (!empty(t.k4s)) { k4sp = addr(t.k4s[0]) } - if (!empty(t.k5q)) { k5qp = addr(t.k5q[0]) } - if (!empty(t.k5s)) { k5sp = addr(t.k5s[0]) } - if (!empty(t.k6q)) { k6qp = addr(t.k6q[0]) } - if (!empty(t.k6s)) { k6sp = addr(t.k6s[0]) } - if (!empty(t.q40q)) { q40qp = addr(t.q40q[0]) } - if (!empty(t.q40s)) { q40sp = addr(t.q40s[0]) } - if (!empty(t.q51q)) { q51qp = addr(t.q51q[0]) } - if (!empty(t.q51s)) { q51sp = addr(t.q51s[0]) } - let rp = addr(regs[0]) - let nr = length(regs) - let njobs = is_job_que_available() ? min(nr, 4 * (get_total_hw_jobs() + 1)) : 1 - maybe_parallel_for(0, nr, njobs) $(rb, re) { - unsafe { - for (i in range(rb, re)) { - let f = rp[i].fmt - if (f == 0) { - invoke(rq8, qbp + rp[i].off, qsp + rp[i].off / 32l, rp[i].n, rp[i].d) - } elif (f == 1) { - invoke(rmx, mxqp + rp[i].off / 2l, mxsp + rp[i].off / 32l, rp[i].n, rp[i].d) - } elif (f == 2) { - invoke(rq51, q51qp + (rp[i].off / 32l) * 20l, q51sp + (rp[i].off / 32l) * 4l, rp[i].n, rp[i].d) - } else { - let sb = rp[i].off / 256l - let qsb = kq_qsb(f) - let ssb = kq_ssb(f) - var kqp = f == 4 ? k4qp : (f == 5 ? k5qp : (f == 6 ? k6qp : q40qp)) - var ksp = f == 4 ? k4sp : (f == 5 ? k5sp : (f == 6 ? k6sp : q40sp)) - invoke(rkq, f, kqp + sb * qsb, ksp + sb * ssb, rp[i].n, rp[i].d) - } - } - } - } - } -} - -// Repack every Q8 2D weight tensor in qblob/qscales into the active backend's interleaved layout, -// IN PLACE, once at load. Mirrors forward()'s per-tensor offsets/shapes exactly; the embedding + -// norms live in fblob and are never touched. -def private repack_q8_weights(var t : Model) { - let c = t.config - let dim = c.dim - let hidden = c.hidden_dim - var regs : array - for (l in range64(c.n_layers + c.n_layer_nextn)) { // + the MTP/NextN block at index n_layers - let kv = layer_kv_dim(c, l) - let qd = layer_qd(c, l) - // K-quant-tagged weights live in the kq planes (disk order, never repacked here) — their offsets are NOT qblob offsets - if (t.wq_offs[l] >= 0l && fmt_at(t.wq_fmt, l) == KqFmt.q8) { - push_repack(regs, 0, t.wq_offs[l], dim, qd * (c.q_gated ? 2l : 1l)) - } - if (t.wk_offs[l] >= 0l && fmt_at(t.wk_fmt, l) == KqFmt.q8) { // absent on E-series shared-KV layers - push_repack(regs, 0, t.wk_offs[l], dim, kv) - } - if (t.wv_offs[l] >= 0l && fmt_at(t.wv_fmt, l) == KqFmt.q8) { - push_repack(regs, 0, t.wv_offs[l], dim, kv) - } - if (t.wo_offs[l] >= 0l && fmt_at(t.wo_fmt, l) == KqFmt.q8) { - push_repack(regs, 0, t.wo_offs[l], qd, dim) - } - if (layer_is_recurrent(c, l)) { // deltanet projections (kq-tagged planes repack in the kq walk) - let cd = dn_conv_dim(c) - if (fmt_at(t.dnqkv_fmt, l) == KqFmt.q8) { - push_repack(regs, 0, t.dnqkv_offs[l], dim, cd) - } - if (fmt_at(t.dngate_fmt, l) == KqFmt.q8) { - push_repack(regs, 0, t.dngate_offs[l], dim, c.ssm_d_inner) - } - if (t.dnbeta_offs[l] >= 0l) { // -1 when beta/alpha ship F32 (fblob, no repack) - push_repack(regs, 0, t.dnbeta_offs[l], dim, c.ssm_dt_rank) - push_repack(regs, 0, t.dnalpha_offs[l], dim, c.ssm_dt_rank) - } - if (fmt_at(t.dnout_fmt, l) == KqFmt.q8) { - push_repack(regs, 0, t.dnout_offs[l], c.ssm_d_inner, dim) - } - } - if (has_ple(c)) { // gemma-4 E-series per-layer PLE gate/proj (the emb table stays linear, unpacked) - let ple = c.n_embd_per_layer - push_repack(regs, 0, t.ple_gate_off + l * dim * ple, dim, ple) - push_repack(regs, 0, t.ple_proj_off + l * ple * dim, ple, dim) - } - if (c.n_expert > 0l && l >= c.n_layer_dense_lead) { // MoE: each expert slice is its own 2D tensor, repacked per its shape - let ege = dim * c.n_ff_exp - if (!t.experts_mx4) { // native-MXFP4 stacks live in mxq/mxs, row-major (no Q8 repack) - for (e in range64(c.n_expert)) { - if (fmt_at(t.we1_fmt, l) == KqFmt.q8) { - push_repack(regs, 0, t.we1_offs[l] + e * ege, dim, c.n_ff_exp) - } - if (fmt_at(t.we2_fmt, l) == KqFmt.q8) { - push_repack(regs, 0, t.we2_offs[l] + e * ege, c.n_ff_exp, dim) - } - if (fmt_at(t.we3_fmt, l) == KqFmt.q8) { - push_repack(regs, 0, t.we3_offs[l] + e * ege, dim, c.n_ff_exp) - } - } - } - if (c.n_ff_shexp > 0l) { - let she = dim * c.n_ff_shexp - push_repack(regs, 0, t.wsh1_off + l * she, dim, c.n_ff_shexp) - push_repack(regs, 0, t.wsh2_off + l * she, c.n_ff_shexp, dim) - push_repack(regs, 0, t.wsh3_off + l * she, dim, c.n_ff_shexp) - } - if (c.moe_dense_shexp) { // gemma4: the dense parallel shared expert repacks like a plain FFN - if (fmt_at(t.w1_fmt, l) == KqFmt.q8) { - push_repack(regs, 0, t.w1_offs[l], dim, hidden) - } - if (fmt_at(t.w2_fmt, l) == KqFmt.q8) { - push_repack(regs, 0, t.w2_offs[l], hidden, dim) - } - if (fmt_at(t.w3_fmt, l) == KqFmt.q8) { - push_repack(regs, 0, t.w3_offs[l], dim, hidden) - } - } - } else { - // dense arch layer OR a glm4moe dense-lead layer (a plain FFN, no expert stacks) - let h = layer_hidden(t, l) - if (fmt_at(t.w1_fmt, l) == KqFmt.q8) { - push_repack(regs, 0, t.w1_offs[l], dim, h) - } - if (fmt_at(t.w2_fmt, l) == KqFmt.q8) { - push_repack(regs, 0, t.w2_offs[l], h, dim) - } - if (fmt_at(t.w3_fmt, l) == KqFmt.q8) { - push_repack(regs, 0, t.w3_offs[l], dim, h) - } - } - } - if ((!c.shared_weights && t.wcls_fmt == KqFmt.q8) || t.cls_q8) { - // the linear copy at emb_q8_off is deliberately NOT repacked — embedding-row dequant indexes it directly - push_repack(regs, 0, t.wcls_off, dim, c.vocab_size) - } - if (c.n_layer_nextn > 0l && t.mtp_ehproj_fmt == KqFmt.q8) { // NextN eh_proj: 2*dim -> dim - push_repack(regs, 0, t.mtp_ehproj_off, 2l * dim, dim) - } - repack_regions(t, regs) - t.mx4_repacked = true - delete regs -} - -// route one kq-tagged tensor to its format's plane pair (q8 tags no-op — those live in qblob) -def private push_repack_kq(var regs : array; fmt : KqFmt; woff, n, d : int64) { - if (fmt == KqFmt.k4) { - push_repack(regs, 4, woff, n, d) - } elif (fmt == KqFmt.k5) { - push_repack(regs, 5, woff, n, d) - } elif (fmt == KqFmt.k6) { - push_repack(regs, 6, woff, n, d) - } elif (fmt == KqFmt.q40) { - push_repack(regs, 40, woff, n, d) - } -} - -// Repack every K-quant-tagged weight tensor into the active backend's grp kq layout, IN -// PLACE — the kq sibling of repack_q8_weights, called only when the backend carries kq slots. -// The tied/untied classifier repacks too: embed_row gathers its rows through the grp layout after. -def private repack_kq_weights(var t : Model) { - let c = t.config - let dim = c.dim - var regs : array - for (l in range64(c.n_layers + c.n_layer_nextn)) { // + the MTP/NextN block at index n_layers - let kv = layer_kv_dim(c, l) - let qd = layer_qd(c, l) - // recurrent (deltanet) layers carry no attention projections — their kq planes are the dn triple - if (layer_is_recurrent(c, l)) { - let cd = dn_conv_dim(c) - push_repack_kq(regs, fmt_at(t.dnqkv_fmt, l), t.dnqkv_offs[l], dim, cd) - push_repack_kq(regs, fmt_at(t.dngate_fmt, l), t.dngate_offs[l], dim, c.ssm_d_inner) - push_repack_kq(regs, fmt_at(t.dnout_fmt, l), t.dnout_offs[l], c.ssm_d_inner, dim) - } - if (t.wq_offs[l] >= 0l) { - push_repack_kq(regs, fmt_at(t.wq_fmt, l), t.wq_offs[l], dim, qd * (c.q_gated ? 2l : 1l)) - } - if (t.wk_offs[l] >= 0l) { - push_repack_kq(regs, fmt_at(t.wk_fmt, l), t.wk_offs[l], dim, kv) - } - if (t.wv_offs[l] >= 0l) { - push_repack_kq(regs, fmt_at(t.wv_fmt, l), t.wv_offs[l], dim, kv) - } - if (t.wo_offs[l] >= 0l) { - push_repack_kq(regs, fmt_at(t.wo_fmt, l), t.wo_offs[l], qd, dim) - } - if (c.n_expert > 0l && l >= c.n_layer_dense_lead) { - if (!t.experts_mx4) { // each expert slice grp-repacks independently, like the Q8 loop - let ege = dim * c.n_ff_exp - for (e in range64(c.n_expert)) { - push_repack_kq(regs, fmt_at(t.we1_fmt, l), t.we1_offs[l] + e * ege, dim, c.n_ff_exp) - push_repack_kq(regs, fmt_at(t.we2_fmt, l), t.we2_offs[l] + e * ege, c.n_ff_exp, dim) - push_repack_kq(regs, fmt_at(t.we3_fmt, l), t.we3_offs[l] + e * ege, dim, c.n_ff_exp) - } - } - if (c.moe_dense_shexp) { - push_repack_kq(regs, fmt_at(t.w1_fmt, l), t.w1_offs[l], dim, c.hidden_dim) - push_repack_kq(regs, fmt_at(t.w2_fmt, l), t.w2_offs[l], c.hidden_dim, dim) - push_repack_kq(regs, fmt_at(t.w3_fmt, l), t.w3_offs[l], dim, c.hidden_dim) - } - } else { - // dense arch layer OR a glm4moe dense-lead layer; pure-MoE arches never reach here - let h = layer_hidden(t, l) - push_repack_kq(regs, fmt_at(t.w1_fmt, l), t.w1_offs[l], dim, h) - push_repack_kq(regs, fmt_at(t.w2_fmt, l), t.w2_offs[l], h, dim) - push_repack_kq(regs, fmt_at(t.w3_fmt, l), t.w3_offs[l], dim, h) - } - } - if ((!c.shared_weights && t.wcls_fmt != KqFmt.q8) || cls_kq(t)) { - push_repack_kq(regs, c.shared_weights ? t.emb_fmt : t.wcls_fmt, t.wcls_off, dim, c.vocab_size) - } - if (c.n_layer_nextn > 0l) { // NextN eh_proj (no-op for q8 tags) - push_repack_kq(regs, t.mtp_ehproj_fmt, t.mtp_ehproj_off, 2l * dim, dim) - } - repack_regions(t, regs) - delete regs -} - -// Repack the native-MXFP4 expert stacks into the active backend's interleaved mx layout, IN PLACE -// (each expert slice is its own 2D tensor). Mirrors repack_q8_weights' contract: called only after -// select_matmul_backend_for_load picks a repack backend; the pointer is otherwise identity. -def private repack_mx4_stacks(var t : Model) { - let c = t.config - let dim = c.dim - let ege = dim * c.n_ff_exp - var regs : array - for (l in range64(c.n_layers)) { - for (e in range64(c.n_expert)) { - let base = e * ege - push_repack(regs, 1, t.we1_offs[l] + base, dim, c.n_ff_exp) - push_repack(regs, 1, t.we2_offs[l] + base, c.n_ff_exp, dim) - push_repack(regs, 1, t.we3_offs[l] + base, dim, c.n_ff_exp) - } - } - repack_regions(t, regs) - delete regs -} - -// q51-native expert stacks -> the active backend's grp q51 layout (per-stack: a model can -// mix q51 downs with k4 gates, e.g. gemma-4-26B-A4B). No-op unless a stack is tagged q51. -def private repack_q51_stacks(var t : Model) { - let c = t.config - let dim = c.dim - let ege = dim * c.n_ff_exp - var regs : array - for (l in range64(c.n_layers)) { - let f1q = fmt_at(t.we1_fmt, l) == KqFmt.q51 - let f2q = fmt_at(t.we2_fmt, l) == KqFmt.q51 - let f3q = fmt_at(t.we3_fmt, l) == KqFmt.q51 - if (!(f1q || f2q || f3q)) { - continue - } - for (e in range64(c.n_expert)) { - let base = e * ege - if (f1q) { - push_repack(regs, 2, t.we1_offs[l] + base, dim, c.n_ff_exp) - } - if (f2q) { - push_repack(regs, 2, t.we2_offs[l] + base, c.n_ff_exp, dim) - } - if (f3q) { - push_repack(regs, 2, t.we3_offs[l] + base, dim, c.n_ff_exp) - } - } - } - if (!empty(regs)) { - repack_regions(t, regs) - to_log(LOG_INFO, "dasLLAMA: q51 expert planes repacked to grp{active_q51_layout_mr()}\n") - } - delete regs -} - -//! go straight into `mode`'s storage (pre-quantized tensors transcode block-for-block, no fp32 -//! round-trip) so a Q8/Q4 load never holds the dense fp32 weights. Embedding + norms stay fp32 in -//! fblob regardless; vocab is derived from token_embd (some files omit llama.vocab_size). +//! Load a llama-architecture GGUF into the split layout at the requested precision — straight into +//! `mode`'s storage (pre-quantized tensors transcode block-for-block; a Q8/Q4 load never holds the +//! dense fp32 weights). Norms stay fp32 in fblob; vocab derives from token_embd. var private g_weights_epoch = 0l //! Monotonic count of model loads this process. Weight caches keyed by HOST ADDRESS flush when it @@ -3471,6 +3119,9 @@ def private load_gguf_impl(path : string; var mode : QuantMode) : Model { g_weights_epoch++ var t = Model() t.quant = mode + if (has_env_variable("DASLLAMA_CONV_PROF")) { + set_conv_profile(true) + } let ts_load = ref_time_ticks() let shards <- gguf_shard_paths(path) if (length(shards) > 1) { @@ -3494,7 +3145,7 @@ def private load_gguf_impl(path : string; var mode : QuantMode) : Model { panic("dasLLAMA: cannot open gguf '{path}'") } fmap(f) $(var bytes : array#) { - let m <- parse_gguf_meta(bytes) + var inscope m <- parse_gguf_meta(bytes) load_gguf_parsed(t, m, bytes, mode, ts_load) } fclose(f) @@ -3640,12 +3291,14 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | if (gguf_find_tensor(m, "blk.0.attn_norm.weight") < 0) { panic("dasLLAMA: MTP-only sidecar GGUF (no trunk tensors) is not supported — use the full -MTP file") } - // glm4moe ships the draft head with its OWN embed_tokens/shared_head_head; the das MTP - // head drafts through the TRUNK's embedding/classifier instead, so drafting here would - // use the wrong weights — disable the head honestly rather than mis-draft. - if (gguf_find_tensor(m, "blk.{layers}.nextn.embed_tokens.weight") >= 0 - || gguf_find_tensor(m, "blk.{layers}.nextn.shared_head_head.weight") >= 0) { - to_log(LOG_WARNING, "dasLLAMA: NextN block carries its own embed_tokens/shared_head_head — MTP draft head disabled\n") + // glm4moe's MTP shares the trunk's embedding and classifier with the draft head, and the + // GGUF materializes copies of both — so draft through the TRUNK's tables and skip them. + // A shape disagreement means it is NOT that sharing, so fail closed. + let nemb = gguf_find_tensor(m, "blk.{layers}.nextn.embed_tokens.weight") + let nhead = gguf_find_tensor(m, "blk.{layers}.nextn.shared_head_head.weight") + if ((nemb >= 0 && m.tensors[nemb].n_elem != vocab * dim) + || (nhead >= 0 && m.tensors[nhead].n_elem != vocab * dim)) { + to_log(LOG_WARNING, "dasLLAMA: NextN embed_tokens/shared_head_head are not the trunk's shape — MTP draft head disabled\n") t.config.n_layer_nextn = 0l } } elif (gguf_find_tensor(m, "blk.{layers}.nextn.eh_proj.weight") >= 0 @@ -4260,11 +3913,8 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | } else { t.mtp_headnorm_off = -1l // reads fall back to output_norm (rms_final_off) } - // optional per-head embedding/classifier: no known qwen35 file ships them — fail closed - if (gguf_find_tensor(m, "blk.{layers}.nextn.embed_tokens.weight") >= 0 - || gguf_find_tensor(m, "blk.{layers}.nextn.shared_head_head.weight") >= 0) { - panic("dasLLAMA: NextN per-head embed_tokens/shared_head_head are not supported yet") - } + // nextn.embed_tokens / nextn.shared_head_head are the trunk's own tables under the + // head's prefix (shape-vetted at config parse) — deliberately never loaded } if (has_ple(t.config)) { // the per-layer token-embedding table [vocab x (ple*layers)], gathered per token, never repacked load_big(m, bytes, "per_layer_token_embd.weight", t, t.ple_emb_off, vocab * t.config.n_embd_per_layer * layers, scratch, 0l, t.ple_emb_fmt) @@ -4282,10 +3932,10 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | } // Qwen2 Q/K/V projection biases — small fp32 tensors, kept dense if (t.config.attn_qkv_bias) { - t.bq |> resize(layers * qd) - t.bk |> resize(layers * kv) - t.bv |> resize(layers * kv) - for (l in range64(layers)) { + t.bq |> resize(layers_ld * qd) + t.bk |> resize(layers_ld * kv) + t.bv |> resize(layers_ld * kv) + for (l in range64(layers_ld)) { gguf_read_tensor_f32(m, bytes, "blk.{l}.attn_q.bias", t.bq, l * qd, qd) gguf_read_tensor_f32(m, bytes, "blk.{l}.attn_k.bias", t.bk, l * kv, kv) gguf_read_tensor_f32(m, bytes, "blk.{l}.attn_v.bias", t.bv, l * kv, kv) @@ -4293,13 +3943,14 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | } // gpt-oss attention output-projection bias (dim per layer) if (t.config.attn_out_bias) { - t.bo |> resize(layers * dim) - for (l in range64(layers)) { + t.bo |> resize(layers_ld * dim) + for (l in range64(layers_ld)) { gguf_read_tensor_f32(m, bytes, "blk.{l}.attn_output.bias", t.bo, l * dim, dim) } } delete scratch to_log(LOG_INFO, "dasLLAMA: load stage weights: {get_time_usec(ts_stage) / 1000} ms\n") + conv_profile_report() // the per-conversion-kind split of the stage just reported ts_stage = ref_time_ticks() // arm64: switch to the laneq backend and repack the just-loaded Q8 weights into its interleaved // layout. Returns false off-ARM (portable stays, weights stay row-major). @@ -4637,6 +4288,9 @@ struct Session { // post-output-norm hidden of the LAST evaluated position (llama.cpp's t_h_nextn handoff); // mtp_h_pos1 is that position PLUS ONE (0 = cold, keeps Session zero-init-safe). mtp_h : array // h_nextn (dim) + mtp_h0 : array // the verify batch's ROW-0 normed hidden (= h_pos). forward_prefill's + // tail only stashes the LAST row, which a reject discards — this is + // what re-seeds mtp_h so a rejected step needs no re-forward mtp_h_pos1 : int64 mtp_cat : array // eh_proj input [enorm(embed(tok)) ; hnorm(h)] (2*dim) mtp_cat_b : array // prompt-warm eh_proj concat image ((npos-1) x 2*dim, scratch-sized per warm) @@ -4765,6 +4419,7 @@ def make_run_state(c : Config; kdt : KVDtype = KVDtype.f32; vdt : KVDtype = KVDt } if (c.n_layer_nextn > 0l) { // MTP/NextN: h_nextn handoff + eh_proj concat input + verify row-0 outputs s.mtp_h |> resize(c.dim) + s.mtp_h0 |> resize(c.dim) s.mtp_cat |> resize(2l * c.dim) s.mtp_logits |> resize(c.vocab_size) s.mtp_vbatch |> resize(2) @@ -8124,8 +7779,8 @@ def private heat_advise_layer(t : Model; l : int64; h : HeatLayer) { } if (want > 0l) { moe_gpu_heat_advise(l, t.we1_offs[l], t.we3_offs[l], t.we2_offs[l], - int(fmt_at(t.we1_fmt, l)), int(fmt_at(t.we3_fmt, l)), int(fmt_at(t.we2_fmt, l)), - c.dim, c.n_ff_exp, ids, want) + int(moe_gpu_expert_fmt(fmt_at(t.we1_fmt, l))), int(moe_gpu_expert_fmt(fmt_at(t.we3_fmt, l))), + int(moe_gpu_expert_fmt(fmt_at(t.we2_fmt, l))), c.dim, c.n_ff_exp, ids, want) } delete ids } @@ -8514,7 +8169,7 @@ def private moe_experts_apply(t : Model; var s : Session; l : int64) { } } matmul_moe_gpu_ffn(s.moe_dn, s.moe_offs1, s.moe_offs3, s.moe_offs2, k, - s.moe_gxq, s.moe_gxs, dim, nfe, k, int(f1), int(f3), int(f2), c.ffn_act == FfnAct.gelu) + s.moe_gxq, s.moe_gxs, dim, nfe, k, int(moe_gpu_expert_fmt(f1)), int(moe_gpu_expert_fmt(f3)), int(moe_gpu_expert_fmt(f2)), c.ffn_act == FfnAct.gelu) prof_add("mm_moe", ts_gu) } elif (nhits > 0l) { // split-FFN: compacted hit regions (pool coordinates, rows 0..nhits) dispatch async @@ -8548,7 +8203,7 @@ def private moe_experts_apply(t : Model; var s : Session; l : int64) { } let ts_sub = ref_time_ticks() matmul_moe_gpu_ffn_begin(s.moe_offs1, s.moe_offs3, s.moe_offs2, nhits, - s.moe_gxq, s.moe_gxs, dim, nfe, nhits, int(f1), int(f3), int(f2), c.ffn_act == FfnAct.gelu) + s.moe_gxq, s.moe_gxs, dim, nfe, nhits, int(moe_gpu_expert_fmt(f1)), int(moe_gpu_expert_fmt(f3)), int(moe_gpu_expert_fmt(f2)), c.ffn_act == FfnAct.gelu) prof_add("heat_sub", ts_sub) // dispatch cost ON the critical path (the split's tax) // the missed experts run the CPU groupn chain while the GPU crunches the hits var ts_down = ts_gu @@ -9484,6 +9139,7 @@ def private mtp_verify_row0(t : Model; var s : Session) { let c = t.config let dim = c.dim rms_at(s.xb, t, t.rms_final_off, s.x_b, 0l, dim) + copy_floats(s.xb, 0l, s.mtp_h0, 0l, dim) // = h_pos; a reject re-seeds the draft head from it mm_cls(s.mtp_logits, t, s.xb, s.xq, s.xs, s.xbs, dim, c.vocab_size) // mirror the plain classifier tail exactly — `truth` must match plain decode BY CONSTRUCTION if (c.final_logit_softcap > 0.0) { @@ -9514,6 +9170,7 @@ def private mtp_verify_logits(t : Model; var s : Session; npos : int64) { s.mtp_norm_b |> scratch_resize(npos * dim) s.mtp_logits_b |> scratch_resize(npos * vocab) rms_batch(s.mtp_norm_b, s.x_b, t, t.rms_final_off, dim, npos) + copy_floats(s.mtp_norm_b, 0l, s.mtp_h0, 0l, dim) // = h_pos; a reject re-seeds the draft head from it mm_b_f(s, s.mtp_logits_b, t, cls_fmt, t.wcls_off, s.mtp_norm_b, dim, vocab, npos) copy_floats(s.mtp_logits_b, 0l, s.mtp_logits, 0l, vocab) copy_floats(s.mtp_logits_b, (npos - 1l) * vocab, s.logits, 0l, vocab) @@ -9597,8 +9254,18 @@ def mtp_spec_eval(t : Model; var s : Session; tok : int64; var accepted : int64& s.n_past = pos + 2l return true } - // reject: the accepted row's KV from the verify batch is already correct; roll the recurrent - // state back and re-advance it through tok alone (s.logits = plain logits, like forward) + // Reject — but the verify already produced what a plain step would: row 0's logits and KV are + // correct, so re-forwarding discards real work. The re-forward is only needed to roll back + // recurrent state the verify advanced past the draft; an arch without any skips it. + if (c.recr_mask == 0ul) { + copy_floats(s.mtp_logits, 0l, s.logits, 0l, c.vocab_size) + // forward_prefill's tail stashed the LAST row's h — the draft's, now discarded — so re-seed + // the draft head from row 0's normed hidden the verify's own classifier computed + copy_floats(s.mtp_h0, 0l, s.mtp_h, 0l, c.dim) + s.mtp_h_pos1 = pos + 1l + s.n_past = pos + 1l + return false + } mtp_state_restore(s) forward(t, s, tok, pos) s.n_past = pos + 1l @@ -9656,7 +9323,11 @@ def generate_mtp_greedy(t : Model; var s : Session; prompt : array; total // ----- forward section profiler (decode AND prefill feed the same buckets) ----- // Measurement is UNCONDITIONAL and negligible vs the matmuls; reset() starts a window, report() dumps it. -// Prefill pass-threading thresholds (M1 Max ~90us dispatch, runtime knobs, 0 = always thread). + +// Batch widths at or below this are speculative-verify shaped, so they follow the DECODE dispatch +// policy, not the prefill work threshold — see prefill_attention (lcpp runs n=1..3, vLLM's GLM 1). +let private ATTN_NARROW_NPOS = 8l + var g_attn_par_threshold = 100_000l // Thread the per-position activation requant (mm_b) once n × npos clears this. var g_requant_par_threshold = 256_000l @@ -10392,7 +10063,8 @@ def private prefill_attention(var s : Session; kcl, vcl : uint8 const?; npos, st is_sliding : bool; sliding_window : int64; attn_softcap : float; sinksp : float const?) { // this layer's per-head sink logits; null = no sinks let maxctx = start_pos + npos // per-head att row width (= max cnt over this call) - let attn_par = npos * maxctx * n_heads >= g_attn_par_threshold + // narrow batches are decode-shaped: under the prefill work threshold they lose every lane + let attn_par = npos <= ATTN_NARROW_NPOS || npos * maxctx * n_heads >= g_attn_par_threshold let kdt = s.kv_dtype_k let vdt = s.kv_dtype_v // ONE run list over the layer's widest window; workers intersect their sub-windows with each run @@ -11646,11 +11318,11 @@ def private ffn_moe_prefill_grouped(t : Model; var s : Session; l : int64; npos s.moe_inv[s.moe_bkt[j]] = uint(j) } matmul_moe_gpu_ffn_combined(s.moe_eout, s.moe_offs1, s.moe_offs3, s.moe_offs2, nreg, - s.moe_gxq, s.moe_gxs, s.moe_w_b, s.moe_inv, dim, nfe, nk, npos, int(f1), int(f3), int(f2), c.ffn_act == FfnAct.gelu) + s.moe_gxq, s.moe_gxs, s.moe_w_b, s.moe_inv, dim, nfe, nk, npos, int(moe_gpu_expert_fmt(f1)), int(moe_gpu_expert_fmt(f3)), int(moe_gpu_expert_fmt(f2)), c.ffn_act == FfnAct.gelu) prof_add("moe_gpu", ts_g1) } elif (fused_gpu) { // combine off: per-row gout + the park/CPU-reduce path below matmul_moe_gpu_ffn(s.moe_gout, s.moe_offs1, s.moe_offs3, s.moe_offs2, nreg, - s.moe_gxq, s.moe_gxs, dim, nfe, nk, int(f1), int(f3), int(f2), c.ffn_act == FfnAct.gelu) + s.moe_gxq, s.moe_gxs, dim, nfe, nk, int(moe_gpu_expert_fmt(f1)), int(moe_gpu_expert_fmt(f3)), int(moe_gpu_expert_fmt(f2)), c.ffn_act == FfnAct.gelu) prof_add("moe_gpu", ts_g1) } else { if (fused_kq) { // native-plane: one batched region-list GEMM per projection (never biased) @@ -12030,10 +11702,10 @@ def private gemma4_moe_prefill_grouped(t : Model; var s : Session; l : int64; np s.moe_inv[s.moe_bkt[j]] = uint(j) } matmul_moe_gpu_ffn_combined(s.moe_eout, s.moe_offs1, s.moe_offs3, s.moe_offs2, nreg, - s.moe_gxq, s.moe_gxs, s.moe_w_b, s.moe_inv, dim, nfe, nk, npos, int(f1), int(f3), int(f2), c.ffn_act == FfnAct.gelu) + s.moe_gxq, s.moe_gxs, s.moe_w_b, s.moe_inv, dim, nfe, nk, npos, int(moe_gpu_expert_fmt(f1)), int(moe_gpu_expert_fmt(f3)), int(moe_gpu_expert_fmt(f2)), c.ffn_act == FfnAct.gelu) } else { matmul_moe_gpu_ffn(s.moe_gout, s.moe_offs1, s.moe_offs3, s.moe_offs2, nreg, - s.moe_gxq, s.moe_gxs, dim, nfe, nk, int(f1), int(f3), int(f2), c.ffn_act == FfnAct.gelu) + s.moe_gxq, s.moe_gxs, dim, nfe, nk, int(moe_gpu_expert_fmt(f1)), int(moe_gpu_expert_fmt(f3)), int(moe_gpu_expert_fmt(f2)), c.ffn_act == FfnAct.gelu) } prof_add("moe_gpu", ts_g1) if (!gpu_comb) { // combine off: park the raw down rows in bucket order for the reduce diff --git a/modules/dasLLAMA/dasllama/dasllama_env.das b/modules/dasLLAMA/dasllama/dasllama_env.das index 0d72bbc4ba..0609e1ac90 100644 --- a/modules/dasLLAMA/dasllama/dasllama_env.das +++ b/modules/dasLLAMA/dasllama/dasllama_env.das @@ -110,6 +110,10 @@ def private reg_engine(var o : array) { "A/B rail for the dispatch caller's thread priority, -2..2; unset claims the top notch.") k(o, "DASLLAMA_TRUTH_REFRESH", EnvKind.flag, EnvArea.engine, "off", "Regenerate the stored parity truth files instead of comparing against them.") + k(o, "DASLLAMA_CONV_PROF", EnvKind.flag, EnvArea.engine, "off", + "Bucket gguf -> image conversion time by kind over the weight walk; one clock pair per tensor.") + k(o, "DASLLAMA_ALLOW_INTERP_LOAD", EnvKind.flag, EnvArea.engine, "off", + "Permit a big gguf load without -jit; the transforms run interpreted, so expect minutes per GB.") } def private reg_gpu_tier(var o : array) { @@ -219,6 +223,14 @@ def private reg_vulkan(var o : array) { "Cooperative-matrix mode; the flash-attention twin needs it even when the GEMM runs sdot4.") k(o, "DASLLAMA_MM_SMALL", EnvKind.number, EnvArea.vulkan, "32", "Small-batch tier: 32 = sdot4 (default, beats both coopmat tiles below the crossover), 64 = coopmat M, 128 = always-L.") + k(o, "DASLLAMA_MM_SMALLD", EnvKind.number, EnvArea.vulkan, "64", + "Small-d cutoff routing narrow roles (k/v) to the small tier; widening measured worse, so this is an instrument.") + k(o, "DASLLAMA_VK_FUSE", EnvKind.flag, EnvArea.vulkan, "on", + "Fused decode tail (add+rms+requant, qk-norm+rope); 0 pins the split dispatches for a same-build A/B.") + k(o, "DASLLAMA_VK_XFERQ", EnvKind.flag, EnvArea.vulkan, "on", + "Stream expert uploads on the dedicated transfer queue, overlapped via a timeline semaphore; 0 keeps the single-queue rail.") + k(o, "DASLLAMA_VK_MEMPRIO", EnvKind.flag, EnvArea.vulkan, "on", + "Tag allocations high-priority (VK_EXT_memory_priority) so the driver demotes desktop memory, not ours.") k(o, "DASLLAMA_VK_FA", EnvKind.flag, EnvArea.vulkan, "on", "Vulkan flash-attention kernel; 0 falls back to the chunked path.") k(o, "DASLLAMA_VK_REBAR", EnvKind.flag, EnvArea.vulkan, "on", @@ -302,6 +314,10 @@ def private reg_benchmark(var o : array) { "Simdgroups per threadgroup in the Metal attention lab.") k(o, "DASMETAL_LAB_DUMP_MSL", EnvKind.flag, EnvArea.benchmark, "off", "Dump generated MSL from the Metal labs instead of running it.") + k(o, "PROBE_PATH", EnvKind.path, EnvArea.benchmark, "_wcliff.bin", + "Scratch file the write-cliff probe writes, cwd-relative by default; point it at the drive under test.") + k(o, "PROBE_GB", EnvKind.number, EnvArea.benchmark, "50", + "Gigabytes the write-cliff probe writes before reporting.") } def private reg_profiling(var o : array) { diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das index 2972befdaf..1b66261e44 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das @@ -38,6 +38,25 @@ def q8q8_repack_type(mr : int; wbias : int = 0; kgroup : int = 4) : Q8RepackType //! stamped from one manifest entry so both decline in lockstep (M4). let GEMM_REFERENCE_MR = 4 +//! GPU-tier device-plane shape of the Q8 rail: 32-weight blocks, 32B quants + one f16 scale +//! halfword per block. Every q8 plane sizing (gathers, arena/stack math) derives from these. +let Q8_BLOCK_ELEMS = 32l +let Q8_QPB = 32l // quant bytes per block +let Q8_SPB = 2l // scale bytes per block (f16 halfword) + +//! kq superblock element count, and the DEVICE scale-row stride: the gathers decode every kq +//! format's scales to 20B rows on device (kq_ssb below is the DISK/plane stride, per format). +let KQ_SUPERBLOCK_ELEMS = 256l +let KQ_DEV_SSB = 20l + +//! q5_1 per-32-block plane strides: 16B nibbles + 4B qh quants, f16 d + f16 m scales. +let Q51_QPB = 20l +let Q51_SPB = 4l + +//! q8n: the same q8 weights as gguf-native 34B interleaved blocks ({f16 d; int8 qs[32]}) — +//! ONE plane (no separate scale plane), the cm2 decode-in-load device layout. +let Q8N_BPB = 34l + //! Quant-plane bytes per 256-weight superblock per row for a kq format id (4/5/6 = Q4_K/Q5_K/ //! Q6_K, 40 = Q4_0). The ONE stride source for every fmt-branched kq walker/kernel/repack — //! an unknown id panics instead of silently walking another format's stride. diff --git a/modules/dasLLAMA/dasllama/dasllama_gemma4a.das b/modules/dasLLAMA/dasllama/dasllama_gemma4a.das index 59daf0448b..f431de770d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemma4a.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemma4a.das @@ -197,7 +197,7 @@ def load_gemma4a_encoder(path : string) : Gemma4aEncoder { panic("dasLLAMA gemma4a: cannot open mmproj gguf '{path}'") } fmap(f) $(var bytes : array#) { - let m <- parse_gguf_meta(bytes) + var inscope m <- parse_gguf_meta(bytes) let proj = (gguf_has(m, "clip.audio.projector_type") ? gguf_str(m, bytes, "clip.audio.projector_type") : gguf_str(m, bytes, "clip.projector_type")) diff --git a/modules/dasLLAMA/dasllama/dasllama_image.das b/modules/dasLLAMA/dasllama/dasllama_image.das index 3f94fa7694..6c0076b519 100644 --- a/modules/dasLLAMA/dasllama/dasllama_image.das +++ b/modules/dasLLAMA/dasllama/dasllama_image.das @@ -11,6 +11,7 @@ require dasllama/dasllama_tokenizer // Tokenizer — the bulk tokenizer serial require daslib/archive require daslib/array_boost // nolint:STYLE030 — temp_array is [template]; STYLE030 misses template-fn refs (same note as dasllama_common) require dasllama/dasllama_common +require dasllama/dasllama_layout // convert_model_to_metal_blob — the blob-flavor image transform require dasllama/dasllama_math // active_kernel_backend — half the image identity // The prepared-model image (spec: fmap-model-image plan, locked 2026-07-16): the post-load @@ -25,8 +26,12 @@ require dasllama/dasllama_math // active_kernel_backend — half the image ide // WhisperModel.enc) contribute their planes under dotted names ("enc.fblob"); string-array // fields ride the meta blob via serialize_strings — raw string pointers can't be planes. -// bump on ANY layout/serialization change — a mismatched image regenerates from the gguf -let IMAGE_VERSION = 5 // v5: Model grew the deltanet projection format tags (dn kq-native planes) +// Bump on ANY change to the image's layout/serialization OR to WHAT THE LOADER PUTS IN IT — the +// identity below is computed before the gguf is parsed, so it can carry box/knob state but nothing +// config-derived. A loader change that alters which tensors load leaves a structurally VALID stale +// image that silently represents a different model; only this version catches it. Mismatch +// regenerates from the gguf, never migrates. +let IMAGE_VERSION = 6 // v6: glm4moe NextN heads now load (blk.n_layers + its qkv biases) //! The metal (blob-only) flavor's identity tag: q8 planes ride the 34B block_q8_0 blob and the //! kq scale planes their GPU forms (convert_model_to_metal_blob) — flavors are per-config and @@ -621,6 +626,24 @@ def load_image(path : string; var t; tag : string = "") : bool { return true } +// interp-load guard: the gguf path's O(model) das loops run ~10x slower interpreted (tinyllama +// load 53s vs 5.5s jitted; repack itself is native tune kernels, 0ms either way) — a 69GB +// hybrid extrapolates to an hour. Prepared images (the slice path) stay interp-friendly. +def private guard_interp_gguf_load(path : string) { + if (jit_enabled() || is_standalone_exe()) { + return + } + var st : FStat + if (!stat(path, st) || int64(st.size) <= 4_294_967_296l) { + return + } + if (env_flag("DASLLAMA_ALLOW_INTERP_LOAD", false)) { + to_log(LOG_WARNING, "dasLLAMA: interpreted gguf load of {int64(st.size) >> 20l} MB — expect minutes per GB (DASLLAMA_ALLOW_INTERP_LOAD)\n") + return + } + panic("dasLLAMA: loading a {int64(st.size) >> 30l} GB gguf without -jit runs the load transforms interpreted (an hour on big models) — run with -jit, load a prepared .dlim, or set DASLLAMA_ALLOW_INTERP_LOAD=1") +} + //! Model entry point — see load_image (`tag` = the flavor, like save_model_image). def load_model_image(path : string; var t : Model; tag : string = "") : bool { if (!load_image(path, t, tag)) { @@ -637,24 +660,39 @@ def load_model_image(path : string; var t : Model; tag : string = "") : bool { //! (``DASLLAMA_IMAGE=0`` disables); under an active Metal mode the image is the BLOB-ONLY metal flavor (CPU inference against it panics) when servable, the planar CPU flavor otherwise. Call inside ``with_job_que()``. def load_model_cached(path : string; mode : QuantMode = QuantMode.fp32) : Model { if (path |> ends_with(".dlim")) { - // a prepared image named directly — first-class once the source gguf is gone. Try both - // identities, EXPECTED flavor first (metal under an active Metal mode, else planar) so - // the normal case loads without the wrong-tag decline warning; a dual mismatch panics + // a prepared image named directly — first-class once the source gguf is gone. EXPECTED + // flavor first (metal mode > armed vulkan tier > planar) so the normal case loads + // without the wrong-tag decline warning; a full mismatch panics apply_box_profile_runtime() let ts_img = ref_time_ticks() var t = Model() let metal_first = get_metal_mode() != MetalMode.off && has_decode_override("metal") - let loaded = (metal_first - ? load_model_image(path, t, METAL_IMAGE_TAG) || load_model_image(path, t) - : load_model_image(path, t) || load_model_image(path, t, METAL_IMAGE_TAG)) + let vk_cfg = metal_first ? "" : moe_gpu_bake_tag() + let vk_tag = vk_cfg == "" ? "" : "vulkan|{vk_cfg}" + var loaded = false + if (vk_tag != "") { + vulkan_bake_slice_begin() + loaded = load_model_image(path, t, vk_tag) + if (loaded) { + vulkan_bake_slice_end(t) + } else { + vulkan_bake_disarm() + } + } if (!loaded) { - panic("dasLLAMA: '{path}' is not a prepared image for this box/knobs (identity {image_identity()} / {image_identity(METAL_IMAGE_TAG)}) — regenerate it from the source gguf") + loaded = (metal_first + ? load_model_image(path, t, METAL_IMAGE_TAG) || load_model_image(path, t) + : load_model_image(path, t) || load_model_image(path, t, METAL_IMAGE_TAG)) + } + if (!loaded) { + let vk_ident = vk_tag != "" ? " / {image_identity(vk_tag)}" : "" + panic("dasLLAMA: '{path}' is not a prepared image for this box/knobs (identity {image_identity()} / {image_identity(METAL_IMAGE_TAG)}{vk_ident}) — regenerate it from the source gguf") } to_log(LOG_INFO, "dasLLAMA: prepared image mapped in {get_time_usec(ts_img) / 1000} ms — {path}\n") return <- t } - // metal mode at load time picks the image FLAVOR: blob-only metal when the drivers can - // serve the config, planar CPU otherwise (flavors are per-config and self-sufficient) + // load-time flavor pick: blob-only metal when its drivers serve the config; the vulkan + // flavor (planar + baked GPU planes) when the vulkan tier arms; planar CPU otherwise let want_metal = mode == QuantMode.q8 && get_metal_mode() != MetalMode.off && has_decode_override("metal") if (mode == QuantMode.q8 && env_flag("DASLLAMA_IMAGE", true)) { apply_box_profile_runtime() // a backend pin must precede the identity (it names the backend) @@ -667,8 +705,33 @@ def load_model_cached(path : string; mode : QuantMode = QuantMode.fp32) : Model return <- t } } + // the vulkan flavor: the GPU walk slices the baked device-layout planes off the mapping + // instead of re-gathering; ANY vulkan config switch changes the tag and rebakes below + let vk_cfg = want_metal ? "" : moe_gpu_bake_tag() + let vk_tag = vk_cfg == "" ? "" : "vulkan|{vk_cfg}" + if (vk_tag != "") { + let img_v = image_path_for(path, vk_tag) + vulkan_bake_slice_begin() + if (load_model_image(img_v, t, vk_tag)) { + vulkan_bake_slice_end(t) + to_log(LOG_INFO, "dasLLAMA: prepared vulkan image mapped in {get_time_usec(ts_img) / 1000} ms — {img_v}\n") + return <- t + } + vulkan_bake_disarm() + vulkan_bake_begin() // whatever loads below runs the GPU walk — collect and bake + } let img = image_path_for(path) if (load_model_image(img, t)) { + if (vk_tag != "" && vulkan_bake_take(t)) { + // the planar image served this run's uploads; bake the flavor for the next one + let ts_save = ref_time_ticks() + let img_v = image_path_for(path, vk_tag) + if (save_model_image(t, img_v, vk_tag)) { + to_log(LOG_INFO, "dasLLAMA: vulkan image baked in {get_time_usec(ts_save) / 1000} ms — {img_v}\n") + } + delete t.vkblob // this run is already uploaded — the bake only serves the NEXT load + delete t.vkplan + } if (!(want_metal && metal_model_servable(t))) { to_log(LOG_INFO, "dasLLAMA: prepared image mapped in {get_time_usec(ts_img) / 1000} ms — {img}\n") return <- t @@ -677,6 +740,7 @@ def load_model_cached(path : string; mode : QuantMode = QuantMode.fp32) : Model // and regenerate the blob flavor (the planar file stays for CPU-mode loads) delete t } + guard_interp_gguf_load(path) var m <- load_model_(path, mode) let ts_save = ref_time_ticks() if (want_metal && metal_model_servable(m) && convert_model_to_metal_blob(m)) { @@ -684,11 +748,25 @@ def load_model_cached(path : string; mode : QuantMode = QuantMode.fp32) : Model if (save_model_image(m, img_m, METAL_IMAGE_TAG)) { to_log(LOG_INFO, "dasLLAMA: prepared image saved in {get_time_usec(ts_save) / 1000} ms — {img_m}\n") } - } elif (save_model_image(m, img)) { - to_log(LOG_INFO, "dasLLAMA: prepared image saved in {get_time_usec(ts_save) / 1000} ms — {img}\n") + } else { + // planar saves FIRST (vkblob still empty — the planar flavor must stay planar), + // then the take moves the collected planes in and the vulkan flavor saves + if (save_model_image(m, img)) { + to_log(LOG_INFO, "dasLLAMA: prepared image saved in {get_time_usec(ts_save) / 1000} ms — {img}\n") + } + if (vk_tag != "" && vulkan_bake_take(m)) { + let ts_vk = ref_time_ticks() + let img_v = image_path_for(path, vk_tag) + if (save_model_image(m, img_v, vk_tag)) { + to_log(LOG_INFO, "dasLLAMA: vulkan image baked in {get_time_usec(ts_vk) / 1000} ms — {img_v}\n") + } + delete m.vkblob + delete m.vkplan + } } return <- m } + guard_interp_gguf_load(path) var m <- load_model_(path, mode) // no image cache: the metal drivers are blob-only, so metal mode still takes the in-RAM form if (want_metal && metal_model_servable(m)) { diff --git a/modules/dasLLAMA/dasllama/dasllama_kernel_access.das b/modules/dasLLAMA/dasllama/dasllama_kernel_access.das index 4f2b6af7cc..6e9bbc58b1 100644 --- a/modules/dasLLAMA/dasllama/dasllama_kernel_access.das +++ b/modules/dasLLAMA/dasllama/dasllama_kernel_access.das @@ -140,9 +140,9 @@ class private AccessVisitor : AstVisitor { def override preVisitExprCall(expr : ExprCall?) : void { let cname = string(expr.name) // GPU intrinsics that take a whole array and write (or only read) through it: - // dasSpirv coopmat pair + the dasMetal simdgroup pair store through arg 1; - // tg_store_float4(dst, idx, v) stores through arg 0 - if (cname == "coopmatStore" || cname == "simdgroup_store") { + // dasSpirv coopmat pair (+ the cm2 tensor-addressed forms) + the dasMetal simdgroup + // pair store through arg 1; tg_store_float4(dst, idx, v) stores through arg 0 + if (cname == "coopmatStore" || cname == "simdgroup_store" || cname == "coopmatStoreTensor") { if (length(expr.arguments) >= 2) { note_write(expr.arguments[1], false) } @@ -154,7 +154,8 @@ class private AccessVisitor : AstVisitor { } return } - if (cname == "coopmatLoad" || cname == "simdgroup_load") { + if (cname == "coopmatLoad" || cname == "simdgroup_load" + || cname == "coopmatLoadTensor" || cname == "coopmatLoadTensorDecode") { return // src argument is a plain read — the generic read pass records it } callees |> insert(cname) diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das new file mode 100644 index 0000000000..e968f01a5a --- /dev/null +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -0,0 +1,820 @@ +options gen2 +options _comment_hygiene = true + +module dasllama_layout shared public + +require dasllama/dasllama_common +require dasllama/dasllama_math // g_repack_* backend pointers + active_* layout knobs +require dasllama/dasllama_math_default // k4_sc_mn — the k4 6-bit sc/mn decoder +require dasllama/dasllama_gguf // K6_SSB — the k6 disk scale stride +require dasllama/dasllama_gemm_schema // kq_qsb/kq_ssb — the fmt-id plane strides +require dasllama/dasllama_par // maybe_parallel_for + jobque probes +require math + +// Disk-format -> compute-layout transforms: the metal-flavor blob transform, the CPU repack +// walkers, and the GPU tier gathers. Requires dasllama_common back for Model/KqFmt; the loader +// reaches the entry points through the hooks registered at [init] (register_model_layout). + +//! KqFmt -> the gemm_schema kq format id space (4/5/6/40) — the ONE bridge between the enum +//! and the stride tables. q8/q51 are not kq superblock formats and panic. +def kq_schema_id(f : KqFmt) : int { + if (f == KqFmt.k4) return 4 + if (f == KqFmt.k5) return 5 + if (f == KqFmt.k6) return 6 + if (f == KqFmt.q40) return 40 + panic("kq_schema_id: not a kq superblock format") + return 0 +} + +// ===== metal-flavor blob transform (the blob-only .dlim redesign) ===== + +// A q8 region's blob bind offset (off/32)*34 lands 16B-aligned iff off % 256 == 0; a k6 +// region's d-plane bind (NSB*16 + (off/256)*2) needs 4B alignment iff off % 512 == 0. +// k4/k5/q40 quant + scale binds are 16B-multiples by construction. +def private metal_blob_off_ok(off : int64; fmt : KqFmt) : bool { + if (off < 0l) return true + if (fmt == KqFmt.q8) return off % 256l == 0l + if (fmt == KqFmt.k6) return off % 512l == 0l + if (fmt == KqFmt.q51) return off % 128l == 0l // (off/32)*20 and (off/32)*4 both 16B-aligned + return true +} + +def private metal_blob_layout_ok(t : Model) : bool { + let c = t.config + if (long_length(t.wq_offs) < c.n_layers) { + return false + } + // a pure-MoE file (qwen3moe) carries no dense FFN arrays — its expert stacks bind instead + let has_ffn = long_length(t.w1_offs) >= c.n_layers + let has_exps = (c.moe && long_length(t.we1_offs) >= c.n_layers && + long_length(t.we2_offs) >= c.n_layers && long_length(t.we3_offs) >= c.n_layers) + let has_dn = c.recr_mask != 0ul && long_length(t.dnqkv_offs) >= c.n_layers + if (!has_ffn && !has_exps) { + return false + } + for (l in range64(c.n_layers)) { + // recurrent layers (Wave D): no attention tensors (offs -1, off_ok true); dn planes must be q8 + let dn_bad = (has_dn && layer_is_recurrent(c, l) && + (!dn_layer_q8(t, l) || + !metal_blob_off_ok(t.dnqkv_offs[l], KqFmt.q8) || + !metal_blob_off_ok(t.dngate_offs[l], KqFmt.q8) || + !metal_blob_off_ok(t.dnbeta_offs[l], KqFmt.q8) || + !metal_blob_off_ok(t.dnalpha_offs[l], KqFmt.q8) || + !metal_blob_off_ok(t.dnout_offs[l], KqFmt.q8))) + if (dn_bad || + !metal_blob_off_ok(t.wq_offs[l], fmt_at(t.wq_fmt, l)) || + !metal_blob_off_ok(t.wk_offs[l], fmt_at(t.wk_fmt, l)) || + !metal_blob_off_ok(t.wv_offs[l], fmt_at(t.wv_fmt, l)) || + !metal_blob_off_ok(t.wo_offs[l], fmt_at(t.wo_fmt, l)) || + (has_ffn && (!metal_blob_off_ok(t.w1_offs[l], fmt_at(t.w1_fmt, l)) || + !metal_blob_off_ok(t.w2_offs[l], fmt_at(t.w2_fmt, l)) || + !metal_blob_off_ok(t.w3_offs[l], fmt_at(t.w3_fmt, l)))) || + (has_exps && (!metal_blob_off_ok(t.we1_offs[l], fmt_at(t.we1_fmt, l)) || + !metal_blob_off_ok(t.we2_offs[l], fmt_at(t.we2_fmt, l)) || + !metal_blob_off_ok(t.we3_offs[l], fmt_at(t.we3_fmt, l))))) { + return false + } + } + // shexp planes: always q8, contiguous at base + l*she — base + stride covers every layer + if (c.n_ff_shexp > 0l) { + let she = c.dim * c.n_ff_shexp + if ((she % 256l) != 0l || + !metal_blob_off_ok(t.wsh1_off, KqFmt.q8) || + !metal_blob_off_ok(t.wsh2_off, KqFmt.q8) || + !metal_blob_off_ok(t.wsh3_off, KqFmt.q8)) { + return false + } + } + // E-series PLE gate/proj q8 planes bind per layer; the token table / model-proj stay CPU-side + if (has_ple(c)) { + let ples = c.dim * c.n_embd_per_layer + if ((ples % 256l) != 0l || + !metal_blob_off_ok(t.ple_gate_off, KqFmt.q8) || + !metal_blob_off_ok(t.ple_proj_off, KqFmt.q8)) { + return false + } + } + let cls_fmt = c.shared_weights ? t.emb_fmt : t.wcls_fmt + return (metal_blob_off_ok(t.wcls_off, cls_fmt) && + (!t.cls_q8 || metal_blob_off_ok(t.emb_q8_off, KqFmt.q8))) +} + +//! Convert a planar q8-mode Model to the metal-blob form IN PLACE (see the Model field doc). +//! False = the model cannot take the blob form (not q8, repacked planes, or a bind-alignment +//! violation) — the caller keeps the planar model. CPU inference on the converted model panics. +def convert_model_to_metal_blob(var t : Model) : bool { + if (t.metal_blob) { + return true + } + if (t.quant != QuantMode.q8 || t.kq_repacked || t.mx4_repacked || kernel_backend_needs_repack(active_kernel_backend())) { + return false + } + if (t.image_map != null) { + // image-backed planes are borrowed READ-ONLY views — the in-place k4s/k5s/k6s + // rewrites would fault. Transform an owned load (load_model_ / DASLLAMA_IMAGE=0). + to_log(LOG_WARNING, "dasLLAMA: metal-blob transform declined — the model is image-mapped (borrowed planes); transform an owned gguf load\n") + return false + } + if (!metal_blob_layout_ok(t)) { + to_log(LOG_WARNING, "dasLLAMA: metal-blob transform declined — a weight offset violates the GPU bind alignment\n") + return false + } + let ts = ref_time_ticks() + let nb = long_length(t.qblob) / Q8_BLOCK_ELEMS + let mbb = Q8_QPB + Q8_SPB // the 34B gguf-native block_q8_0 + if (nb > 0l) { + t.mblob |> resize(nb * mbb) + let s16 = t.wscale_f16 + unsafe { + var dp = addr(t.mblob[0]) + let qp = addr < int8 const? >(t.qblob[0]) + var sp : float const? + var sp16 : uint16 const? + if (s16) { + sp16 = addr(t.qscales16[0]) + } else { + sp = addr(t.qscales[0]) + } + let nch = int(clamp(nb / 262144l, 1l, 64l)) + maybe_parallel_for(0, int(nb / 32l) + (nb % 32l != 0l ? 1 : 0), is_job_que_available() ? nch : 1) $(cb, ce) { + unsafe { + for (b in range64(int64(cb) * 32l, min(int64(ce) * 32l, nb))) { + var ph = reinterpret(dp + b * mbb) + ph[0] = s16 ? sp16[b] : uint16(f32_to_f16(clamp(sp[b], -65504.0, 65504.0))) + memcpy(reinterpret(dp + b * mbb + Q8_SPB), reinterpret(qp + b * Q8_QPB), 32) + } + } + } + } + } + delete t.qblob + delete t.qscales + delete t.qscales16 + // k4/k5: the 16B verbatim disk scale block per superblock — the 4B pad is CPU-repack + // scratch (see the plane-layout comment in dasllama_gguf) and a blob model never repacks + if (!empty(t.k4s)) { + compact_kq_scale_strip(t.k4s) + } + if (!empty(t.k5s)) { + compact_kq_scale_strip(t.k5s) + } + // k6: the GPU split form — [nsb x 16B sub-scale strips][nsb x f16 d halfwords] + if (!empty(t.k6s)) { + var k6c : array + let nsb = long_length(t.k6s) / K6_SSB + k6c |> resize(nsb * K6_SSB) + unsafe { + var dp = addr(k6c[0]) + let sp = addr < uint8 const? >(t.k6s[0]) + for (sb in range64(nsb)) { + memcpy(reinterpret(dp + sb * 16l), reinterpret(sp + sb * 18l), 16) + dp[nsb * 16l + sb * 2l] = sp[sb * 18l + 16l] + dp[nsb * 16l + sb * 2l + 1l] = sp[sb * 18l + 17l] + } + } + t.k6s <- k6c + } + t.metal_blob = true + to_log(LOG_INFO, "dasLLAMA: metal-blob transform ({nb} blocks) in {get_time_usec(ts) / 1000} ms\n") + return true +} + +// 20B -> 16B in place: keep bytes [0..16) of every superblock strip, then shrink. Forward +// byte moves are safe — every destination sits below its source (regions can overlap). +def private compact_kq_scale_strip(var ks : array) { + let nsb = long_length(ks) / 20l + unsafe { + var p = addr(ks[0]) + for (sb in range64(1l, nsb)) { // sb 0 already sits at 0 + for (i in range64(16l)) { + p[sb * 16l + i] = p[sb * 20l + i] + } + } + } + ks |> resize(nsb * 16l) +} + +// ===== CPU repack walkers (backend grp interleave, in place at load) ===== + +// A repack work item: one mr-aligned row-chunk of a weight region. fmt: 0 = q8 (qblob/qscales), +// 1 = mx4 (mxq/mxs), 4/5/6 = the kq plane pairs. Chunking splits huge regions (the classifier) for +// load balance; 1024 is a multiple of every backend mr, so row groups never straddle a boundary. +struct private RepackReg { + off : int64 + n : int64 + d : int64 + fmt : int +} + +def private push_repack(var regs : array; fmt : int; off, n, d : int64) { + var r0 = 0l + while (r0 < d) { + let rc = min(1024l, d - r0) + regs |> push(RepackReg(off = off + r0 * n, n = n, d = rc, fmt = fmt)) + r0 += rc + } +} + +// Run every collected repack across the jobque — regions are disjoint, so any lane count is +// bit-exact. Workers touch hoisted raw pointers + the hoisted backend fn values only. +def private repack_regions(var t : Model; regs : array) { + if (empty(regs)) { + return + } + let rq8 = g_repack_q8q8 + let rkq = g_repack_kq + let rmx = g_repack_mx4 + let rq51 = g_repack_q51 + unsafe { + var qbp : int8? = null + var qsp : float? = null + var mxqp : uint8? = null + var mxsp : uint8? = null + var k4qp : uint8? = null + var k4sp : uint8? = null + var k5qp : uint8? = null + var k5sp : uint8? = null + var k6qp : uint8? = null + var k6sp : uint8? = null + var q40qp : uint8? = null + var q40sp : uint8? = null + var q51qp : uint8? = null + var q51sp : uint8? = null + if (!empty(t.qblob)) { qbp = addr(t.qblob[0]) } + if (!empty(t.qscales)) { qsp = addr(t.qscales[0]) } + if (!empty(t.mxq)) { mxqp = addr(t.mxq[0]) } + if (!empty(t.mxs)) { mxsp = addr(t.mxs[0]) } + if (!empty(t.k4q)) { k4qp = addr(t.k4q[0]) } + if (!empty(t.k4s)) { k4sp = addr(t.k4s[0]) } + if (!empty(t.k5q)) { k5qp = addr(t.k5q[0]) } + if (!empty(t.k5s)) { k5sp = addr(t.k5s[0]) } + if (!empty(t.k6q)) { k6qp = addr(t.k6q[0]) } + if (!empty(t.k6s)) { k6sp = addr(t.k6s[0]) } + if (!empty(t.q40q)) { q40qp = addr(t.q40q[0]) } + if (!empty(t.q40s)) { q40sp = addr(t.q40s[0]) } + if (!empty(t.q51q)) { q51qp = addr(t.q51q[0]) } + if (!empty(t.q51s)) { q51sp = addr(t.q51s[0]) } + let rp = addr(regs[0]) + let nr = length(regs) + let njobs = is_job_que_available() ? min(nr, 4 * (get_total_hw_jobs() + 1)) : 1 + maybe_parallel_for(0, nr, njobs) $(rb, re) { + unsafe { + for (i in range(rb, re)) { + let f = rp[i].fmt + if (f == 0) { + invoke(rq8, qbp + rp[i].off, qsp + rp[i].off / Q8_BLOCK_ELEMS, rp[i].n, rp[i].d) + } elif (f == 1) { + invoke(rmx, mxqp + rp[i].off / 2l, mxsp + rp[i].off / 32l, rp[i].n, rp[i].d) + } elif (f == 2) { + invoke(rq51, q51qp + (rp[i].off / 32l) * Q51_QPB, q51sp + (rp[i].off / 32l) * Q51_SPB, rp[i].n, rp[i].d) + } else { + let sb = rp[i].off / 256l + let qsb = kq_qsb(f) + let ssb = kq_ssb(f) + var kqp = f == 4 ? k4qp : (f == 5 ? k5qp : (f == 6 ? k6qp : q40qp)) + var ksp = f == 4 ? k4sp : (f == 5 ? k5sp : (f == 6 ? k6sp : q40sp)) + invoke(rkq, f, kqp + sb * qsb, ksp + sb * ssb, rp[i].n, rp[i].d) + } + } + } + } + } +} + +// Repack every Q8 2D weight tensor in qblob/qscales into the active backend's interleaved layout, +// IN PLACE, once at load. Mirrors forward()'s per-tensor offsets/shapes exactly; the embedding + +// norms live in fblob and are never touched. +def private repack_q8_weights(var t : Model) { + let c = t.config + let dim = c.dim + let hidden = c.hidden_dim + var regs : array + for (l in range64(c.n_layers + c.n_layer_nextn)) { // + the MTP/NextN block at index n_layers + let kv = layer_kv_dim(c, l) + let qd = layer_qd(c, l) + // K-quant-tagged weights live in the kq planes (disk order, never repacked here) — their offsets are NOT qblob offsets + if (t.wq_offs[l] >= 0l && fmt_at(t.wq_fmt, l) == KqFmt.q8) { + push_repack(regs, 0, t.wq_offs[l], dim, qd * (c.q_gated ? 2l : 1l)) + } + if (t.wk_offs[l] >= 0l && fmt_at(t.wk_fmt, l) == KqFmt.q8) { // absent on E-series shared-KV layers + push_repack(regs, 0, t.wk_offs[l], dim, kv) + } + if (t.wv_offs[l] >= 0l && fmt_at(t.wv_fmt, l) == KqFmt.q8) { + push_repack(regs, 0, t.wv_offs[l], dim, kv) + } + if (t.wo_offs[l] >= 0l && fmt_at(t.wo_fmt, l) == KqFmt.q8) { + push_repack(regs, 0, t.wo_offs[l], qd, dim) + } + if (layer_is_recurrent(c, l)) { // deltanet projections (kq-tagged planes repack in the kq walk) + let cd = dn_conv_dim(c) + if (fmt_at(t.dnqkv_fmt, l) == KqFmt.q8) { + push_repack(regs, 0, t.dnqkv_offs[l], dim, cd) + } + if (fmt_at(t.dngate_fmt, l) == KqFmt.q8) { + push_repack(regs, 0, t.dngate_offs[l], dim, c.ssm_d_inner) + } + if (t.dnbeta_offs[l] >= 0l) { // -1 when beta/alpha ship F32 (fblob, no repack) + push_repack(regs, 0, t.dnbeta_offs[l], dim, c.ssm_dt_rank) + push_repack(regs, 0, t.dnalpha_offs[l], dim, c.ssm_dt_rank) + } + if (fmt_at(t.dnout_fmt, l) == KqFmt.q8) { + push_repack(regs, 0, t.dnout_offs[l], c.ssm_d_inner, dim) + } + } + if (has_ple(c)) { // gemma-4 E-series per-layer PLE gate/proj (the emb table stays linear, unpacked) + let ple = c.n_embd_per_layer + push_repack(regs, 0, t.ple_gate_off + l * dim * ple, dim, ple) + push_repack(regs, 0, t.ple_proj_off + l * ple * dim, ple, dim) + } + if (c.n_expert > 0l && l >= c.n_layer_dense_lead) { // MoE: each expert slice is its own 2D tensor, repacked per its shape + let ege = dim * c.n_ff_exp + if (!t.experts_mx4) { // native-MXFP4 stacks live in mxq/mxs, row-major (no Q8 repack) + for (e in range64(c.n_expert)) { + if (fmt_at(t.we1_fmt, l) == KqFmt.q8) { + push_repack(regs, 0, t.we1_offs[l] + e * ege, dim, c.n_ff_exp) + } + if (fmt_at(t.we2_fmt, l) == KqFmt.q8) { + push_repack(regs, 0, t.we2_offs[l] + e * ege, c.n_ff_exp, dim) + } + if (fmt_at(t.we3_fmt, l) == KqFmt.q8) { + push_repack(regs, 0, t.we3_offs[l] + e * ege, dim, c.n_ff_exp) + } + } + } + if (c.n_ff_shexp > 0l) { + let she = dim * c.n_ff_shexp + push_repack(regs, 0, t.wsh1_off + l * she, dim, c.n_ff_shexp) + push_repack(regs, 0, t.wsh2_off + l * she, c.n_ff_shexp, dim) + push_repack(regs, 0, t.wsh3_off + l * she, dim, c.n_ff_shexp) + } + if (c.moe_dense_shexp) { // gemma4: the dense parallel shared expert repacks like a plain FFN + if (fmt_at(t.w1_fmt, l) == KqFmt.q8) { + push_repack(regs, 0, t.w1_offs[l], dim, hidden) + } + if (fmt_at(t.w2_fmt, l) == KqFmt.q8) { + push_repack(regs, 0, t.w2_offs[l], hidden, dim) + } + if (fmt_at(t.w3_fmt, l) == KqFmt.q8) { + push_repack(regs, 0, t.w3_offs[l], dim, hidden) + } + } + } else { + // dense arch layer OR a glm4moe dense-lead layer (a plain FFN, no expert stacks) + let h = layer_hidden(t, l) + if (fmt_at(t.w1_fmt, l) == KqFmt.q8) { + push_repack(regs, 0, t.w1_offs[l], dim, h) + } + if (fmt_at(t.w2_fmt, l) == KqFmt.q8) { + push_repack(regs, 0, t.w2_offs[l], h, dim) + } + if (fmt_at(t.w3_fmt, l) == KqFmt.q8) { + push_repack(regs, 0, t.w3_offs[l], dim, h) + } + } + } + if ((!c.shared_weights && t.wcls_fmt == KqFmt.q8) || t.cls_q8) { + // the linear copy at emb_q8_off is deliberately NOT repacked — embedding-row dequant indexes it directly + push_repack(regs, 0, t.wcls_off, dim, c.vocab_size) + } + if (c.n_layer_nextn > 0l && t.mtp_ehproj_fmt == KqFmt.q8) { // NextN eh_proj: 2*dim -> dim + push_repack(regs, 0, t.mtp_ehproj_off, 2l * dim, dim) + } + repack_regions(t, regs) + t.mx4_repacked = true + delete regs +} + +// route one kq-tagged tensor to its format's plane pair (q8 tags no-op — those live in qblob) +def private push_repack_kq(var regs : array; fmt : KqFmt; woff, n, d : int64) { + if (fmt == KqFmt.k4) { + push_repack(regs, 4, woff, n, d) + } elif (fmt == KqFmt.k5) { + push_repack(regs, 5, woff, n, d) + } elif (fmt == KqFmt.k6) { + push_repack(regs, 6, woff, n, d) + } elif (fmt == KqFmt.q40) { + push_repack(regs, 40, woff, n, d) + } +} + +// Repack every K-quant-tagged weight tensor into the active backend's grp kq layout, IN +// PLACE — the kq sibling of repack_q8_weights, called only when the backend carries kq slots. +// The tied/untied classifier repacks too: embed_row gathers its rows through the grp layout after. +def private repack_kq_weights(var t : Model) { + let c = t.config + let dim = c.dim + var regs : array + for (l in range64(c.n_layers + c.n_layer_nextn)) { // + the MTP/NextN block at index n_layers + let kv = layer_kv_dim(c, l) + let qd = layer_qd(c, l) + // recurrent (deltanet) layers carry no attention projections — their kq planes are the dn triple + if (layer_is_recurrent(c, l)) { + let cd = dn_conv_dim(c) + push_repack_kq(regs, fmt_at(t.dnqkv_fmt, l), t.dnqkv_offs[l], dim, cd) + push_repack_kq(regs, fmt_at(t.dngate_fmt, l), t.dngate_offs[l], dim, c.ssm_d_inner) + push_repack_kq(regs, fmt_at(t.dnout_fmt, l), t.dnout_offs[l], c.ssm_d_inner, dim) + } + if (t.wq_offs[l] >= 0l) { + push_repack_kq(regs, fmt_at(t.wq_fmt, l), t.wq_offs[l], dim, qd * (c.q_gated ? 2l : 1l)) + } + if (t.wk_offs[l] >= 0l) { + push_repack_kq(regs, fmt_at(t.wk_fmt, l), t.wk_offs[l], dim, kv) + } + if (t.wv_offs[l] >= 0l) { + push_repack_kq(regs, fmt_at(t.wv_fmt, l), t.wv_offs[l], dim, kv) + } + if (t.wo_offs[l] >= 0l) { + push_repack_kq(regs, fmt_at(t.wo_fmt, l), t.wo_offs[l], qd, dim) + } + if (c.n_expert > 0l && l >= c.n_layer_dense_lead) { + if (!t.experts_mx4) { // each expert slice grp-repacks independently, like the Q8 loop + let ege = dim * c.n_ff_exp + for (e in range64(c.n_expert)) { + push_repack_kq(regs, fmt_at(t.we1_fmt, l), t.we1_offs[l] + e * ege, dim, c.n_ff_exp) + push_repack_kq(regs, fmt_at(t.we2_fmt, l), t.we2_offs[l] + e * ege, c.n_ff_exp, dim) + push_repack_kq(regs, fmt_at(t.we3_fmt, l), t.we3_offs[l] + e * ege, dim, c.n_ff_exp) + } + } + if (c.moe_dense_shexp) { + push_repack_kq(regs, fmt_at(t.w1_fmt, l), t.w1_offs[l], dim, c.hidden_dim) + push_repack_kq(regs, fmt_at(t.w2_fmt, l), t.w2_offs[l], c.hidden_dim, dim) + push_repack_kq(regs, fmt_at(t.w3_fmt, l), t.w3_offs[l], dim, c.hidden_dim) + } + } else { + // dense arch layer OR a glm4moe dense-lead layer; pure-MoE arches never reach here + let h = layer_hidden(t, l) + push_repack_kq(regs, fmt_at(t.w1_fmt, l), t.w1_offs[l], dim, h) + push_repack_kq(regs, fmt_at(t.w2_fmt, l), t.w2_offs[l], h, dim) + push_repack_kq(regs, fmt_at(t.w3_fmt, l), t.w3_offs[l], dim, h) + } + } + if ((!c.shared_weights && t.wcls_fmt != KqFmt.q8) || cls_kq(t)) { + push_repack_kq(regs, c.shared_weights ? t.emb_fmt : t.wcls_fmt, t.wcls_off, dim, c.vocab_size) + } + if (c.n_layer_nextn > 0l) { // NextN eh_proj (no-op for q8 tags) + push_repack_kq(regs, t.mtp_ehproj_fmt, t.mtp_ehproj_off, 2l * dim, dim) + } + repack_regions(t, regs) + delete regs +} + +// Repack the native-MXFP4 expert stacks into the active backend's interleaved mx layout, IN PLACE +// (each expert slice is its own 2D tensor). Mirrors repack_q8_weights' contract: called only after +// select_matmul_backend_for_load picks a repack backend; the pointer is otherwise identity. +def private repack_mx4_stacks(var t : Model) { + let c = t.config + let dim = c.dim + let ege = dim * c.n_ff_exp + var regs : array + for (l in range64(c.n_layers)) { + for (e in range64(c.n_expert)) { + let base = e * ege + push_repack(regs, 1, t.we1_offs[l] + base, dim, c.n_ff_exp) + push_repack(regs, 1, t.we2_offs[l] + base, c.n_ff_exp, dim) + push_repack(regs, 1, t.we3_offs[l] + base, dim, c.n_ff_exp) + } + } + repack_regions(t, regs) + delete regs +} + +// q51-native expert stacks -> the active backend's grp q51 layout (per-stack: a model can +// mix q51 downs with k4 gates, e.g. gemma-4-26B-A4B). No-op unless a stack is tagged q51. +def private repack_q51_stacks(var t : Model) { + let c = t.config + let dim = c.dim + let ege = dim * c.n_ff_exp + var regs : array + for (l in range64(c.n_layers)) { + let f1q = fmt_at(t.we1_fmt, l) == KqFmt.q51 + let f2q = fmt_at(t.we2_fmt, l) == KqFmt.q51 + let f3q = fmt_at(t.we3_fmt, l) == KqFmt.q51 + if (!(f1q || f2q || f3q)) { + continue + } + for (e in range64(c.n_expert)) { + let base = e * ege + if (f1q) { + push_repack(regs, 2, t.we1_offs[l] + base, dim, c.n_ff_exp) + } + if (f2q) { + push_repack(regs, 2, t.we2_offs[l] + base, c.n_ff_exp, dim) + } + if (f3q) { + push_repack(regs, 2, t.we3_offs[l] + base, dim, c.n_ff_exp) + } + } + } + if (!empty(regs)) { + repack_regions(t, regs) + to_log(LOG_INFO, "dasLLAMA: q51 expert planes repacked to grp{active_q51_layout_mr()}\n") + } + delete regs +} + +// ===== GPU tier gathers (prepared planes -> device-layout bytes) ===== + +// Gather one PREPARED Q8 weight stack into the GPU tier's two row-major planes — int8 quants + +// f16 halfword scales. slice_rows is the repack unit (each expert slice interleaves independently). +// Reads whatever the active backend prepared (grp interleave or plain row-major). +def moe_gpu_gather_stack(t : Model; woff : int64; n, rows, slice_rows : int64; mr, kgroup, wbias : int64; + var wq : array; var ws : array) { + let nb = n / Q8_BLOCK_ELEMS + assert(rows * n <= 2147483647l, "GPU MoE stack exceeds the int32 gather-buffer rail") + wq |> resize(int(rows * n)) + ws |> resize(int(rows * nb * Q8_SPB)) + assert(wbias == 0l || wbias == 128l, "GPU MoE gather: unknown q8 plane bias") + let nslices = int(rows / slice_rows) + unsafe { + let wp = addr(t.qblob[woff]) + var s16p : uint16 const? = null + var s32p : float const? = null + if (t.wscale_f16) { + s16p = addr(t.qscales16[woff / Q8_BLOCK_ELEMS]) + } else { + s32p = addr(t.qscales[woff / Q8_BLOCK_ELEMS]) + } + var wqp = addr(wq[0]) + var wsp = addr(ws[0]) + let njobs = is_job_que_available() ? min(nslices, 4 * (get_total_hw_jobs() + 1)) : 1 + maybe_parallel_for(0, nslices, njobs) $(sb, se) { + unsafe { + let ng = slice_rows / mr + for (s in range(sb, se)) { + let swoff = int64(s) * slice_rows * n + let ssoff = int64(s) * slice_rows * nb + for (rr in range64(slice_rows)) { + let grouped = rr < ng * mr + let g = rr / mr + let r = rr % mr + for (bi in range64(nb)) { + // scale: group-interleaved [g][bi][r] or the row-major tail; the f32 + // fallback takes the same clamped rounding wscale_convert_f16 applies + let si = grouped ? ssoff + (g * nb + bi) * mr + r : ssoff + rr * nb + bi + wsp[ssoff + rr * nb + bi] = (s16p != null ? s16p[si] + : uint16(f32_to_f16(clamp(s32p[si], -65504.0, 65504.0)))) + var dst = wqp + swoff + rr * n + bi * 32l + if (grouped) { + let gbase = swoff + g * mr * n + r * kgroup + var kg = (bi * 32l) / kgroup + var o = 0l + for (_kk in range64(32l / kgroup)) { + let src = gbase + kg * kgroup * mr + for (j in range64(kgroup)) { + dst[o + j] = uint8(int(wp[src + j]) ^ int(wbias)) + } + kg++ + o += kgroup + } + } else { + let src = swoff + rr * n + bi * 32l + for (j in range64(32l)) { + dst[j] = uint8(wp[src + j]) + } + } + } + } + } + } + } + } +} + +// Gather one PREPARED Q8 weight stack into gguf-native 34B interleaved blocks (the q8n/cm2 +// decode-in-load device layout, ONE plane; upload = straight memcpy). Same source contract +// as moe_gpu_gather_stack: grp interleave or row-major, backend wbias XOR on grouped rows. +def moe_gpu_gather_stack_q8n(t : Model; woff : int64; n, rows, slice_rows : int64; mr, kgroup, wbias : int64; + var wq : array) { + let nb = n / Q8_BLOCK_ELEMS + assert(rows * nb * Q8N_BPB <= 2147483647l, "GPU q8n stack exceeds the int32 gather-buffer rail") + wq |> resize(int(rows * nb * Q8N_BPB)) + assert(wbias == 0l || wbias == 128l, "GPU q8n gather: unknown q8 plane bias") + let nslices = int(rows / slice_rows) + unsafe { + let wp = addr(t.qblob[woff]) + var s16p : uint16 const? = null + var s32p : float const? = null + if (t.wscale_f16) { + s16p = addr(t.qscales16[woff / Q8_BLOCK_ELEMS]) + } else { + s32p = addr(t.qscales[woff / Q8_BLOCK_ELEMS]) + } + var wqp = addr(wq[0]) + let njobs = is_job_que_available() ? min(nslices, 4 * (get_total_hw_jobs() + 1)) : 1 + maybe_parallel_for(0, nslices, njobs) $(sb, se) { + unsafe { + let ng = slice_rows / mr + for (s in range(sb, se)) { + let swoff = int64(s) * slice_rows * n + let ssoff = int64(s) * slice_rows * nb + for (rr in range64(slice_rows)) { + let grouped = rr < ng * mr + let g = rr / mr + let r = rr % mr + for (bi in range64(nb)) { + var dst = wqp + (ssoff + rr * nb + bi) * Q8N_BPB + // scale halfword first — same grouped/tail source index as the + // split-plane gather, same f32 fallback rounding + let si = grouped ? ssoff + (g * nb + bi) * mr + r : ssoff + rr * nb + bi + let sw = (s16p != null ? uint(s16p[si]) + : f32_to_f16(clamp(s32p[si], -65504.0, 65504.0))) + dst[0] = uint8(sw & 0xFFu) + dst[1] = uint8((sw >> 8u) & 0xFFu) + if (grouped) { + let gbase = swoff + g * mr * n + r * kgroup + var kg = (bi * 32l) / kgroup + var o = 2l + for (_kk in range64(32l / kgroup)) { + let src = gbase + kg * kgroup * mr + for (j in range64(kgroup)) { + dst[o + j] = uint8(int(wp[src + j]) ^ int(wbias)) + } + kg++ + o += kgroup + } + } else { + let src = swoff + rr * n + bi * 32l + for (j in range64(32l)) { + dst[2l + j] = uint8(wp[src + j]) + } + } + } + } + } + } + } + } +} + +// Disk-order K-quant plane accessors for the kq gather's un-repacked/tail rows (source of +// truth: the private twins beside repack_k4/k5/k6_grp in dasllama_math_gen.das — replicated +// because math_gen mounts conditionally behind the dasLLVM guard). + +// low 4 bits of weight k off a disk-order k4/k5 nibble row (128 bytes per superblock) +def private kq_disk_nib45(rowq : uint8 const?; k : int64) : uint { + let js = k / 64l + let rem = k % 64l + let b = uint(unsafe(rowq[js * 32l + rem % 32l])) + return rem < 32l ? b & 15u : b >> 4u +} + +// the 5th bit of weight k off a disk-order k5 qh row (32 bytes per superblock) +def private kq_disk_hbit5(rowh : uint8 const?; k : int64) : uint { + let js = k / 64l + let rem = k % 64l + return (uint(unsafe(rowh[rem % 32l])) >> uint(2l * js + (rem < 32l ? 0l : 1l))) & 1u +} + +// low 4 bits of weight k off a disk-order k6 ql row (128 bytes per superblock) +def private kq_disk_nib6(rowq : uint8 const?; k : int64) : uint { + let h = k / 128l + let g = (k % 128l) / 32l + let b = uint(unsafe(rowq[h * 64l + (g % 2l) * 32l + k % 32l])) + return g < 2l ? b & 15u : b >> 4u +} + +// Gather one PREPARED K-quant weight stack into the GPU tier's device planes — quant payloads +// row-major per superblock in the k/k+16 nibble pairing, plus DECODED 20B scale rows. Reads +// whatever the load prepared (grp interleave or disk-order); device bytes are identical either way. +def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice_rows : int64; + repacked : bool; mr : int64; var wq : array; var ws : array) { + let nsb = n / KQ_SUPERBLOCK_ELEMS + let sid = kq_schema_id(fmt) + let qsb = kq_qsb(sid) // quant plane stride per superblock (disk == device for quants) + let dssb = kq_ssb(sid) // disk/plane scale stride; device rows decode to KQ_DEV_SSB + assert(n % KQ_SUPERBLOCK_ELEMS == 0l, "GPU MoE kq gather: row length must be a superblock multiple") + assert(rows * nsb * qsb <= 2147483647l, "GPU MoE kq stack exceeds the int32 gather-buffer rail") + wq |> resize(int(rows * nsb * qsb)) + ws |> resize(int(rows * nsb * KQ_DEV_SSB)) + let nslices = int(rows / slice_rows) + let qrow = nsb * qsb + let srow = nsb * dssb + let ng = repacked ? slice_rows / mr : 0l + unsafe { + let sb0 = woff / KQ_SUPERBLOCK_ELEMS + let qp = (fmt == KqFmt.k4 ? addr(t.k4q[sb0 * kq_qsb(4)]) + : (fmt == KqFmt.k5 ? addr(t.k5q[sb0 * kq_qsb(5)]) + : (fmt == KqFmt.k6 ? addr(t.k6q[sb0 * kq_qsb(6)]) : addr(t.q40q[sb0 * kq_qsb(40)])))) + let sp = (fmt == KqFmt.k4 ? addr(t.k4s[sb0 * kq_ssb(4)]) + : (fmt == KqFmt.k5 ? addr(t.k5s[sb0 * kq_ssb(5)]) + : (fmt == KqFmt.k6 ? addr(t.k6s[sb0 * kq_ssb(6)]) : addr(t.q40s[sb0 * kq_ssb(40)])))) + var wqp = addr(wq[0]) + var wsp = addr(ws[0]) + let njobs = is_job_que_available() ? min(nslices, 4 * (get_total_hw_jobs() + 1)) : 1 + maybe_parallel_for(0, nslices, njobs) $(ssb, sse) { + unsafe { + for (s in range(ssb, sse)) { + let sliceQ = int64(s) * slice_rows * qrow + let sliceS = int64(s) * slice_rows * srow + for (ri in range64(slice_rows)) { + let rr = int64(s) * slice_rows + ri + let grouped = repacked && ri < ng * mr + let g = ri / mr + let r = ri % mr + for (sbi in range64(nsb)) { + var dq = wqp + rr * qrow + sbi * qsb + var dsc = wsp + (rr * nsb + sbi) * KQ_DEV_SSB + if (grouped) { + let gq = qp + sliceQ + g * mr * qrow + sbi * qsb * mr + let gs = sp + sliceS + g * mr * srow + sbi * dssb * mr + for (m in range64(128l)) { // nibble region: verbatim per (blk, j, t) + dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] + } + if (fmt == KqFmt.k5) { + for (bj in range64(32l)) { + dq[128l + bj] = gq[128l * mr + bj * mr + r] + } + } elif (fmt == KqFmt.k6) { + for (p in range64(64l)) { + dq[128l + p] = gq[128l * mr + ((p / 4l) * mr + r) * 4l + p % 4l] + } + } + if (fmt == KqFmt.k6) { + for (idx in range64(16l)) { + dsc[idx] = gs[idx * mr + r] + } + dsc[16] = gs[16l * mr + 2l * r] + dsc[17] = gs[16l * mr + 2l * r + 1l] + dsc[18] = uint8(0) + dsc[19] = uint8(0) + } elif (fmt == KqFmt.q40) { + for (blk in range64(8l)) { // 8 per-block f16 d, mr-interleaved + dsc[blk * 2l] = gs[blk * 2l * mr + 2l * r] + dsc[blk * 2l + 1l] = gs[blk * 2l * mr + 2l * r + 1l] + } + for (idx in range64(16l, 20l)) { + dsc[idx] = uint8(0) + } + } else { + dsc[0] = gs[2l * r] + dsc[1] = gs[2l * r + 1l] + dsc[2] = gs[2l * mr + 2l * r] + dsc[3] = gs[2l * mr + 2l * r + 1l] + for (blk in range64(8l)) { + dsc[4l + blk] = gs[4l * mr + blk * mr + r] + dsc[12l + blk] = gs[12l * mr + blk * mr + r] + } + } + } else { + let rq = qp + sliceQ + ri * qrow + sbi * qsb + let rs = sp + sliceS + ri * srow + sbi * dssb + if (fmt == KqFmt.q40) { + for (m in range64(128l)) { // Q4_0 bytes already pair k / k+16 + dq[m] = rq[m] + } + } else { + for (m in range64(128l)) { // re-pair disk k/k+32 nibbles to k/k+16 + let k = (m / 16l) * 32l + m % 16l + let lo = fmt == KqFmt.k6 ? kq_disk_nib6(rq, k) : kq_disk_nib45(rq, k) + let hi = fmt == KqFmt.k6 ? kq_disk_nib6(rq, k + 16l) : kq_disk_nib45(rq, k + 16l) + dq[m] = uint8(lo | (hi << 4u)) + } + } + if (fmt == KqFmt.k5) { + for (bj in range64(32l)) { + let k = (bj / 4l) * 32l + (bj % 4l) * 4l + var hb = 0u + for (tt in range64(4l)) { + hb |= kq_disk_hbit5(rq + 128l, k + tt) << uint(tt) + hb |= kq_disk_hbit5(rq + 128l, k + 16l + tt) << uint(4l + tt) + } + dq[128l + bj] = uint8(hb) + } + } elif (fmt == KqFmt.k6) { + for (p in range64(64l)) { + dq[128l + p] = rq[128l + p] + } + } + if (fmt == KqFmt.k6 || fmt == KqFmt.q40) { + for (idx in range64(dssb)) { + dsc[idx] = rs[idx] + } + for (idx in range64(dssb, KQ_DEV_SSB)) { + dsc[idx] = uint8(0) + } + } else { + for (idx in range64(4l)) { // f16 d + f16 dmin verbatim + dsc[idx] = rs[idx] + } + for (blk in range64(8l)) { // decode the 6-bit sc/mn packing + let sm = k4_sc_mn(rs + 4l, blk) + dsc[4l + blk] = uint8(sm.x) + dsc[12l + blk] = uint8(sm.y) + } + } + } + } + } + } + } + } + } +} + +// ===== hook registration (the loader in dasllama_common reaches back through these) ===== + +[init] +def private register_layout_hooks { + register_model_layout(@@repack_q8_weights, @@repack_kq_weights, @@repack_mx4_stacks, @@repack_q51_stacks, + @@moe_gpu_gather_stack, @@moe_gpu_gather_stack_kq, @@moe_gpu_gather_stack_q8n) +} diff --git a/modules/dasLLAMA/dasllama/dasllama_math.das b/modules/dasLLAMA/dasllama/dasllama_math.das index e0b89af73f..316c24becb 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math.das +++ b/modules/dasLLAMA/dasllama/dasllama_math.das @@ -2153,6 +2153,31 @@ def public set_moe_gpu_budget_hooks(budget : function<() : int64>; plane : funct //! Device-local bytes the armed tier may spend on resident weights (0 = no tier). def public moe_gpu_weight_budget() : int64 => invoke(g_moe_gpu_budget_report) +def private moe_gpu_no_q8n : bool => false +var private g_moe_gpu_expert_q8n = @@moe_gpu_no_q8n + +//! The tier asks the loader to gather q8 EXPERT stacks as gguf-native q8n blocks (fmt 6, the +//! cm2 decode-in-load layout, no scale plane). Registered beside the budget hooks; the probe +//! may create the device — it must answer before the first gather. +def public set_moe_gpu_expert_q8n_probe(f : function<() : bool>) { + g_moe_gpu_expert_q8n = f +} + +def private moe_gpu_no_bake_tag : string => "" + +var private g_moe_gpu_bake_tag = @@moe_gpu_no_bake_tag + +//! The GPU tier's config-identity tag for the vulkan-flavor image ("" = tier absent/unarmed — +//! no flavor). Everything that shapes the load-time GPU walk must fold in: a config switch +//! must miss the flavor's identity and rebake. +def public set_moe_gpu_bake_tag(f : function<() : string>) { + g_moe_gpu_bake_tag = f +} + +def public moe_gpu_bake_tag : string => invoke(g_moe_gpu_bake_tag) + +def public moe_gpu_expert_q8n() : bool => invoke(g_moe_gpu_expert_q8n) + //! Device-layout bytes one [rows x n] plane of `fmt` occupies (quants + scales). 0 = no tier. def public moe_gpu_plane_bytes(n, rows : int64; fmt : int) : int64 => invoke(g_moe_gpu_plane_bytes_fn, n, rows, fmt) @@ -2165,6 +2190,17 @@ def public set_moe_gpu_total_layers(n : int64) { g_moe_gpu_total_layers = n } +var private g_moe_gpu_stream_need = 0l + +//! Loader contract: worst-case streamed-slot bytes of THIS model (2x gate/up + down device +//! planes of the largest expert layer, via moe_gpu_plane_bytes). The tier carves its stream +//! reserve from this instead of a fixed guess; 0 = unknown (tier falls back to its constant). +def public set_moe_gpu_stream_need(bytes : int64) { + g_moe_gpu_stream_need = bytes +} + +def public moe_gpu_stream_need() : int64 => g_moe_gpu_stream_need + //! Loader contract: "" = the loaded model rides the armed tier; otherwise the human reason it //! cannot (reset per load; meaningless while the tier is unarmed). def public set_moe_gpu_model_support(reason : string) { @@ -2214,8 +2250,12 @@ def public moe_gpu_upload_stack(wq : array | #; ws : array | #; wo if (!g_moe_gpu_installed) { return false } + var wsp : uint8 const? // q8n stacks carry no scale plane — an empty ws hands the tier null + if (!empty(ws)) { + wsp = unsafe(addr(ws[0])) + } unsafe { - return invoke(g_moe_gpu_upload, addr(wq[0]), addr(ws[0]), woff, n, rows, fmt, stream) + return invoke(g_moe_gpu_upload, addr(wq[0]), wsp, woff, n, rows, fmt, stream) } } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_vulkan.das b/modules/dasLLAMA/dasllama/dasllama_math_vulkan.das index 73fe41fa8a..fc7b11b881 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_vulkan.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_vulkan.das @@ -4,6 +4,7 @@ options _comment_hygiene = true module dasllama_math_vulkan shared public require dasllama/dasllama_math +require dasllama/dasllama_gemm_schema // kq_qsb/kq_ssb + the q8/kq plane consts — the stride source require dasllama/dasllama_env // the shared typed env readers + the knob registry require dasllama/dasllama_par require dasllama/dasllama_vulkan_lens @@ -63,6 +64,7 @@ var @ssbo @binding = 0 vk_wq4 : array // vec4 view of the same weight w var @ssbo @binding = 1 vk_ws : array var @ssbo @binding = 2 vk_meta : array var @ssbo @binding = 3 vk_xq : array +var @ssbo @binding = 3 vk_xq4 : array // vec4 view of the activation words (16B loads) var @ssbo @binding = 4 vk_xs : array var @ssbo @binding = 5 vk_y : array @@ -969,6 +971,148 @@ def kq_moe_batch_q40_cmf16 { } } +// ===== the cm2 (NV_cooperative_matrix2) prefill twin ===== +// Mode 4, fmt-6 q8n stacks: 34B blocks decoded IN-LOAD, wg-scope tensor tiles, f16 activations. +// P5a winner geometry (n2x256); layout dims bound every access — edge tiles clamp, no guard code. + +struct VkBlockQ8N { + d : float16 + qs : uint16[16] +} + +var @ssbo @binding = 0 vk_wqn : array +var @ssbo @binding = 0 vk_wq16 : array // u16 view of the same blocks (17/blk — the GEMV path) +var @ssbo @binding = 3 vk_xf16 : array // f16 activation plane (the twin's B side) +var @ssbo @binding = 0 vk_of16 : array // f16 output view (the dequant convert kernel) + +[spirv_decode, unused_argument(bc)] +def decode_q8n(blk : VkBlockQ8N; bc, cib : uint2) : float16 { + let s = uint(blk.qs[int((cib.y & 30u) >> 1u)]) + let sh = 24 - int(cib.y & 1u) * 8 + let q = (int(s) << sh) >> 24 + return blk.d * float16(float(q)) +} + +[compute_shader(local_size_x=256, name="q8n_moe_batch_cm2_spv")] +def q8n_moe_batch_cm2 { + let n = vk_meta[0u] + let d = vk_meta[1u] + let rid = vk_meta[vk_meta[3u] + gl_WorkGroupID.x] + let rb = 4u + rid * 4u + let wblk0 = vk_meta[rb] // region base in 32-weight blocks (= 34B array elements) + let row0 = vk_meta[rb + 1u] // window-relative activation row base + let cnt = vk_meta[rb + 2u] + let tix = gl_WorkGroupID.x - vk_meta[rb + 3u] + let ttiles = (cnt + 127u) / 128u + let xt = tix % ttiles // token 128-tile (fastest) + let wt = tix / ttiles // weight 64-tile + // A: the region's [d x n] q8n plane — block addressing, decode-in-load + var tla : tensorLayout2DPad + tensorLayoutCreate(tla) + tensorLayoutSetBlockSize(tla, 1u, 32u) + tensorLayoutSetDimension(tla, d, n) + tensorLayoutSetStride(tla, n / 32u, 1u) + // B: the f16 activation rows, transpose-viewed; the dimension cuts at the region's end + var tlb : tensorLayout2DPad + tensorLayoutCreate(tlb) + tensorLayoutSetDimension(tlb, row0 + cnt, n) + tensorLayoutSetStride(tlb, n, 1u) + // out: token-major y through the transpose view. MUST be the clamp-Constant layout type: + // only clamped layouts DISCARD out-of-bounds stores — an Undefined-clamp store from a + // partial token tile stomps the neighboring region's rows (lcpp's store type is clamp 1 too) + var tlo : tensorLayout2DPad + tensorLayoutCreate(tlo) + tensorLayoutSetDimension(tlo, row0 + cnt, d) + tensorLayoutSetStride(tlo, d, 1u) + var tv : tensorView2Dt + tensorViewCreate(tv) + var a : coopmatWgA_f16_64x16 + var b0 : coopmatWgB_f16_16x64 + var b1 : coopmatWgB_f16_16x64 + var acc0 : coopmatWgAcc_f16_64x64 + var acc1 : coopmatWgAcc_f16_64x64 + let t0 = row0 + xt * 128u + var k = 0u + while (k < n) { + coopmatLoadTensorDecode(a, vk_wqn, wblk0, tla, wt * 64u, 64u, k, 16u, @@decode_q8n) + coopmatLoadTensor(b0, vk_xf16, 0u, tlb, t0, 64u, k, 16u, tv) + coopmatLoadTensor(b1, vk_xf16, 0u, tlb, t0 + 64u, 64u, k, 16u, tv) + acc0 = coopmatMulAdd(a, b0, acc0) + acc1 = coopmatMulAdd(a, b1, acc1) + k += 16u + } + coopmatClamp(acc0, -65504.0, 65504.0) + coopmatClamp(acc1, -65504.0, 65504.0) + var accw0 : coopmatWgAcc_f32_64x64 + coopmatConvert(accw0, acc0) + coopmatStoreTensor(accw0, vk_y, 0u, tlo, t0, 64u, wt * 64u, 64u, tv) + var accw1 : coopmatWgAcc_f32_64x64 + coopmatConvert(accw1, acc1) + coopmatStoreTensor(accw1, vk_y, 0u, tlo, t0 + 64u, 64u, wt * 64u, 64u, tv) +} + +// the q8n decode GEMV: q8_moe_groupn's region skeleton on the 34B blocks (P4's L2 arm — 17 u16 +// per block, u16-pair loads combined to words, 8 lanes per block, d broadcast-read from u16[0]). +// Activations stay Q8_0 — decode never leaves the int-dot path, only the weight addressing moves. +[compute_shader(local_size_x=64, local_size_x_id=0, name="q8n_moe_groupn_spv")] +def q8n_moe_groupn { + let n = vk_meta[0u] + let d = vk_meta[1u] + let nreg = vk_meta[2u] + let rg = gl_WorkGroupID.x * gl_NumSubgroups + gl_SubgroupID + if (rg < nreg * d) { + let r = rg / d + let row = rg % d + let nb = n / 32u + let wb0 = vk_meta[4u + r * 2u] + row * nb + let xb0 = vk_meta[5u + r * 2u] + let lane = gl_SubgroupInvocationID + let my_word = lane % 8u + let my_boff = lane / 8u + let bpp = gl_SubgroupSize / 8u + var acc = 0.0 + var bi0 = 0u + while (bi0 < nb) { + let b = bi0 + my_boff + var idot = 0 + var dbits = 0u + if (b < nb) { + let base = (wb0 + b) * 17u + let lo = uint(vk_wq16[base + 1u + my_word * 2u]) + let hi = uint(vk_wq16[base + 2u + my_word * 2u]) + idot = sdot4(lo | (hi << 16u), vk_xq[(xb0 + b) * 8u + my_word]) + dbits = uint(vk_wq16[base]) + } + idot += subgroupShuffleXor(idot, 1u) + idot += subgroupShuffleXor(idot, 2u) + idot += subgroupShuffleXor(idot, 4u) + if (my_word == 0u && b < nb) { + acc += float(idot) * unpackHalf2x16(dbits).x * vk_xs[xb0 + b] + } + bi0 += bpp + } + let total = subgroupAdd(acc) + if (subgroupElect()) { + vk_y[vk_meta[3u] + rg] = total + } + } +} + +// the cm2 chain's activation feed: dequantize the uploaded Q8_0 image to the f16 plane the twin +// loads as B. One uint word (4 quants) per thread; ffn-meta word 3 carries the word count. +[compute_shader(local_size_x=256, name="q8_moe_xf16_spv")] +def q8_moe_xf16 { + let w = gl_GlobalInvocationID.x + if (w < vk_meta[3u]) { + let v = int(vk_xq[w]) + let s = vk_xs[w / 8u] + vk_of16[w * 4u] = float16(float((v << 24) >> 24) * s) + vk_of16[w * 4u + 1u] = float16(float((v << 16) >> 24) * s) + vk_of16[w * 4u + 2u] = float16(float((v << 8) >> 24) * s) + vk_of16[w * 4u + 3u] = float16(float(v >> 24) * s) + } +} + // ===== the FFN mid-chain kernels ===== // ONE fused act+requant step between gate+up and down GEMMs — hidden activations never cross PCIe. // Binding 0 doubles as a float view here (vk_upf) vs uint weights in the GEMMs — SPIR-V only emits referenced globals. @@ -1057,6 +1201,34 @@ def q8k_moe_actrq { } } +// the cm2 chain's mid-step: act(gate) * up straight to the f16 hidden plane the down twin loads +// as B — no quantization on this route (same bindings as actrq: up@0, params@2, out@3, gate@5). +[compute_shader(local_size_x=256, name="q8_moe_actf16_spv")] +def q8_moe_actf16 { + let e = gl_GlobalInvocationID.x * 4u + if (e < vk_meta[0u]) { + let gelu = vk_meta[1u] + vk_xf16[e] = float16(act_mul(vk_y[e], vk_upf[e], gelu)) + vk_xf16[e + 1u] = float16(act_mul(vk_y[e + 1u], vk_upf[e + 1u], gelu)) + vk_xf16[e + 2u] = float16(act_mul(vk_y[e + 2u], vk_upf[e + 2u], gelu)) + vk_xf16[e + 3u] = float16(act_mul(vk_y[e + 3u], vk_upf[e + 3u], gelu)) + } +} + +// the arena cm2 chain's activation feed: straight f32 -> f16 rows for the twin's B side — the +// resident prefill's quant-role stand-in (no quantize on the fmt-6 route). Meta word 0 carries +// the element count (always a 32-multiple, so the 4-wide guard is exact). +[compute_shader(local_size_x=256, name="rd_f16cvt_spv")] +def rd_f16cvt { + let e = gl_GlobalInvocationID.x * 4u + if (e < vk_meta[0u]) { + vk_xf16[e] = float16(vk_upf[e]) + vk_xf16[e + 1u] = float16(vk_upf[e + 1u]) + vk_xf16[e + 2u] = float16(vk_upf[e + 2u]) + vk_xf16[e + 3u] = float16(vk_upf[e + 3u]) + } +} + // ===== the combine kernel ===== // One workgroup per output POSITION, summing its k slots' down-GEMM rows ascending j (matches // the decode accumulation order); r0 == 0 starts the sum, chunk-spanning positions accumulate across dispatches. @@ -1089,6 +1261,7 @@ def moe_combine { var @ssbo @binding = 1 vk_wsf : array // float view: the persistent per-head f32 state var @ssbo @binding = 3 vk_xqf : array // float view: the DN_WS panel/o scratch +var @ssbo @binding = 5 vk_yq : array // uint view of the y binding: the fused ar+requant's quant words // conv output rows (f32, the scan's input) ride binding 0's float view (vk_upf) @@ -1980,6 +2153,74 @@ def ar_add_rms_b { } } +// ar_add_rms + dn_requant in ONE dispatch: the normed row quantizes out of ar_row, so xb never +// reaches memory. Verbatim reduce/amax/rounding => BIT-IDENTICAL to the split pair. Bindings 0 x | +// 1 a | 2 meta | 3 w+eps/ascale | 4 xs | 5 xq. meta 0=dim 1=add_on 2=woff 3=eps off (4 is xs now). +[compute_shader(local_size_x=256, name="ar_add_rms_rq_spv")] +def ar_add_rms_rq { + let dim = vk_meta[0u] + let add_on = vk_meta[1u] + let woff = vk_meta[2u] + let soff = vk_meta[3u] + let eps = vk_xqf[soff] + let ascale = vk_xqf[soff + 1u] + let tid = gl_LocalInvocationID.x + var ss = 0.0 + var k = tid + while (k < dim) { + let vv = add_on != 0u ? (vk_upf[k] + vk_wsf[k]) * ascale : vk_upf[k] + vk_upf[k] = vv + ar_row[k] = vv + ss += vv * vv + k += 256u + } + ss = subgroupAdd(ss) + if (gl_SubgroupInvocationID == 0u) { + ar_part[gl_SubgroupID] = ss + } + barrier() + if (tid == 0u) { + var tot = 0.0 + var sgi = 0u + while (sgi < gl_NumSubgroups) { + tot += ar_part[sgi] + sgi++ + } + ar_inv[0u] = 1.0 / sqrt(tot / float(dim) + eps) // the CPU rmsnorm's own form, not rsqrt + } + barrier() + let inv = ar_inv[0u] + // quantize the normed row instead of storing it: 8 lanes per 32-block, 32 blocks per pass. The + // 8 lanes of a block are consecutive AND 8-aligned, so the xor-1/2/4 fold only ever exchanges + // with co-active lanes even when other groups in the subgroup have left the loop. + let nblk = dim / 32u + let lane8 = tid % 8u + var b = tid / 8u + while (b < nblk) { + let base = b * 32u + lane8 * 4u + let v0 = vk_xqf[woff + base] * (ar_row[base] * inv) + let v1 = vk_xqf[woff + base + 1u] * (ar_row[base + 1u] * inv) + let v2 = vk_xqf[woff + base + 2u] * (ar_row[base + 2u] * inv) + let v3 = vk_xqf[woff + base + 3u] * (ar_row[base + 3u] * inv) + var m = max(max(abs(v0), abs(v1)), max(abs(v2), abs(v3))) + m = max(m, subgroupShuffleXor(m, 1u)) + m = max(m, subgroupShuffleXor(m, 2u)) + m = max(m, subgroupShuffleXor(m, 4u)) + let d = m / 127.0 + let id = d != 0.0 ? 1.0 / d : 0.0 + if (lane8 == 0u) { + vk_xs[b] = d + } + let q0 = int(round(v0 * id)) + let q1 = int(round(v1 * id)) + let q2 = int(round(v2 * id)) + let q3 = int(round(v3 * id)) + vk_yq[b * 8u + lane8] = ((uint(q0) & 0xFFu) | ((uint(q1) & 0xFFu) << 8u) + | ((uint(q2) & 0xFFu) << 16u) | ((uint(q3) & 0xFFu) << 24u)) + b += 32u + } +} + // per-head RMSNorm on q/k between the projections and rope (qk_norm models), one wg per head-row, // in place: q rows in vk_upf, k rows in vk_y; weight rows in vk_xqf; eps = vk_xs[0]. meta 0=hs // 1=nh 2=nkvh 3=qstride 4=kstride 5=kbase 6=qwoff 7=kwoff; dispatch rows*(nh+nkvh) wgs. @@ -2068,6 +2309,75 @@ def rope_kv_store { } } +// qk_rms + rope_kv_store in ONE dispatch (qk_norm only), one wg per head-row = qk_rms's own shape; +// verbatim reduce + rotation => BIT-IDENTICAL. rms rows + eps ride the cos buffer (binding 3 is the +// V mirror). 0 kv|1 Kmir|2 meta|3 Vmir|4 cos+rms+eps|5 q. meta = rope's, 5=nh 9=qw 10=kw 11=eps. +var @workgroup qr_row : float[256] // this head's q or k row (hs <= 256, same cap as da_q) + +[compute_shader(local_size_x=256, name="qkn_rope_kv_spv")] +def qkn_rope_kv { + let qd = vk_meta[0u] + let hs = vk_meta[2u] + let half = vk_meta[3u] + let neox = vk_meta[4u] + let nh = vk_meta[5u] + let wid = gl_WorkGroupID.x + let is_k = wid >= nh + let h = is_k ? wid - nh : wid + let tid = gl_LocalInvocationID.x + // q rows live in vk_y, k rows at qd in vk_upf (the kv projection buffer) + let sbase = is_k ? qd + h * hs : h * hs + var v = 0.0 + if (tid < hs) { + v = is_k ? vk_upf[sbase + tid] : vk_y[sbase + tid] + } + var ss = tid < hs ? v * v : 0.0 + ss = subgroupAdd(ss) + if (gl_SubgroupInvocationID == 0u) { + ar_part[gl_SubgroupID] = ss + } + barrier() + if (tid == 0u) { + var tot = 0.0 + var sgi = 0u + while (sgi < gl_NumSubgroups) { + tot += ar_part[sgi] + sgi++ + } + ar_inv[0u] = 1.0 / sqrt(tot / float(hs) + vk_xs[vk_meta[11u]]) + } + barrier() + if (tid < hs) { + qr_row[tid] = vk_xs[(is_k ? vk_meta[10u] : vk_meta[9u]) + tid] * (v * ar_inv[0u]) + } + barrier() + // rope this head's pairs off the normed row. NORM (llama): adjacent pair (2j, 2j+1). + // NEOX (qwen/gemma): (j, j+half). Either way the pairs partition [0, hs). + if (tid < half) { + let fcr = vk_xs[tid] + let fci = vk_xs[half + tid] + let e0 = neox != 0u ? tid : tid * 2u + let e1 = neox != 0u ? tid + half : tid * 2u + 1u + let a0 = qr_row[e0] + let a1 = qr_row[e1] + let r0 = a0 * fcr - a1 * fci + let r1 = a0 * fci + a1 * fcr + if (is_k) { + let kb = vk_meta[7u] + h * hs + vk_wsf[kb + e0] = r0 + vk_wsf[kb + e1] = r1 + } else { + let qb = h * hs + vk_y[qb + e0] = r0 + vk_y[qb + e1] = r1 + } + } + // v is not roped or normed — the k-head wgs copy their own raw row into the V plane + if (is_k && tid < hs) { + vk_xqf[vk_meta[8u] + h * hs + tid] = vk_upf[qd + vk_meta[1u] + h * hs + tid] + } +} + // decode attention (one query row): one wg/head, GQA, f32 K/V mirror (K vk_wsf, V vk_xqf). // Online softmax over 256-position chunks (running max/denom, alpha-rescaled V accumulators) — // shared use is chunk-sized, ANY cnt works. meta 0=cnt 2=kvd 3=hs 4=kv_mul; scale vk_xs[0]. @@ -2610,6 +2920,63 @@ var @ssbo @binding = 1 vk_wsu : array def private k5_dep(hb : uint) : uint => ( ((hb & 1u) << 4u) | ((hb & 2u) << 11u) | ((hb & 4u) << 18u) | ((hb & 8u) << 25u)) +// ===== the K-quant decode GEMV family: ONE LANE PER 32-BLOCK, uint4 loads ===== +// uint4 index `wsb*stride4 + blk` IS block blk's 4 quant words: 4x bytes in flight/warp (512B vs +// 128B, q8_moe_groupn's lever) + one scale decode/block, not four. DRIFT-CLASS (1 float term/block). + +let private KQ_LOW = 0x0F0F0F0Fu // nibble mask +let private KQ_ONES = 0x01010101u // sdot4 against this sums 4 signed bytes + +// a block's activation sum (the min/offset terms fold against it) +def private kq_blk_sum(xl, xh : uint4) : int { + let bs = sdot4(xl.x, KQ_ONES) + sdot4(xl.y, KQ_ONES) + sdot4(xl.z, KQ_ONES) + sdot4(xl.w, KQ_ONES) + return bs + sdot4(xh.x, KQ_ONES) + sdot4(xh.y, KQ_ONES) + sdot4(xh.z, KQ_ONES) + sdot4(xh.w, KQ_ONES) +} + +// a 4-bit block's dot: weight word j's LOW nibbles hit x word j, its HIGH nibbles x word j+4 +def private kq_blk_dot(wv, xl, xh : uint4) : int { + var idot = sdot4(wv.x & KQ_LOW, xl.x) + sdot4(wv.y & KQ_LOW, xl.y) + idot += sdot4(wv.z & KQ_LOW, xl.z) + sdot4(wv.w & KQ_LOW, xl.w) + idot += sdot4((wv.x >> 4u) & KQ_LOW, xh.x) + sdot4((wv.y >> 4u) & KQ_LOW, xh.y) + return idot + sdot4((wv.z >> 4u) & KQ_LOW, xh.z) + sdot4((wv.w >> 4u) & KQ_LOW, xh.w) +} + +// Q6_K's dot, halves kept apart: the lo/hi nibble halves ARE the block's two 16-weight sub-scale +// groups, so each needs its own dot and activation sum. qh contributes 2 bits per weight. +// Returns (ilo, ihi, blo, bhi) — an int4, since a shader function cannot return a tuple. +def private k6_blk_dot(wv, xl, xh, qhl, qhh : uint4; shift : uint) : int4 { + let m2 = 0x03030303u + let l0 = (wv.x & KQ_LOW) | (((qhl.x >> shift) & m2) << 4u) + let l1 = (wv.y & KQ_LOW) | (((qhl.y >> shift) & m2) << 4u) + let l2 = (wv.z & KQ_LOW) | (((qhl.z >> shift) & m2) << 4u) + let l3 = (wv.w & KQ_LOW) | (((qhl.w >> shift) & m2) << 4u) + let h0 = ((wv.x >> 4u) & KQ_LOW) | (((qhh.x >> shift) & m2) << 4u) + let h1 = ((wv.y >> 4u) & KQ_LOW) | (((qhh.y >> shift) & m2) << 4u) + let h2 = ((wv.z >> 4u) & KQ_LOW) | (((qhh.z >> shift) & m2) << 4u) + let h3 = ((wv.w >> 4u) & KQ_LOW) | (((qhh.w >> shift) & m2) << 4u) + return int4(sdot4(l0, xl.x) + sdot4(l1, xl.y) + sdot4(l2, xl.z) + sdot4(l3, xl.w), + sdot4(h0, xh.x) + sdot4(h1, xh.y) + sdot4(h2, xh.z) + sdot4(h3, xh.w), + sdot4(xl.x, KQ_ONES) + sdot4(xl.y, KQ_ONES) + sdot4(xl.z, KQ_ONES) + sdot4(xl.w, KQ_ONES), + sdot4(xh.x, KQ_ONES) + sdot4(xh.y, KQ_ONES) + sdot4(xh.z, KQ_ONES) + sdot4(xh.w, KQ_ONES)) +} + +// Q5_K's dot: the same fold with the 5th bit deposited from the block's qh word, byte j serving +// weight word j (low nibble -> lo half, high nibble -> hi half) +def private k5_blk_dot(wv, xl, xh : uint4; qh : uint) : int { + let h0 = qh & 0xFFu + let h1 = (qh >> 8u) & 0xFFu + let h2 = (qh >> 16u) & 0xFFu + let h3 = (qh >> 24u) & 0xFFu + var idot = sdot4((wv.x & KQ_LOW) | k5_dep(h0 & 0xFu), xl.x) + idot += sdot4((wv.y & KQ_LOW) | k5_dep(h1 & 0xFu), xl.y) + idot += sdot4((wv.z & KQ_LOW) | k5_dep(h2 & 0xFu), xl.z) + idot += sdot4((wv.w & KQ_LOW) | k5_dep(h3 & 0xFu), xl.w) + idot += sdot4(((wv.x >> 4u) & KQ_LOW) | k5_dep(h0 >> 4u), xh.x) + idot += sdot4(((wv.y >> 4u) & KQ_LOW) | k5_dep(h1 >> 4u), xh.y) + idot += sdot4(((wv.z >> 4u) & KQ_LOW) | k5_dep(h2 >> 4u), xh.z) + return idot + sdot4(((wv.w >> 4u) & KQ_LOW) | k5_dep(h3 >> 4u), xh.w) +} + // Q4_K: w = d*sc*q - dmin*mn*1, q in [0,15] — per 32-block xs*(d*sc*idot - dmin*mn*bsum) [compute_shader(local_size_x=64, local_size_x_id=0, name="kq_moe_gemv_k4_spv")] def kq_moe_gemv_k4 { @@ -2624,30 +2991,26 @@ def kq_moe_gemv_k4 { let wsb0 = vk_meta[4u + r * 2u] + row * nsb let xsb0 = vk_meta[5u + r * 2u] let lane = gl_SubgroupInvocationID - let j = lane % 4u - let brel = lane / 4u - let bstride = gl_SubgroupSize / 4u let nb = nsb * 8u var acc = 0.0 var b0 = 0u while (b0 < nb) { - let bb = b0 + brel + let bb = b0 + lane if (bb < nb) { let sb = bb / 8u let blk = bb % 8u let wsb = wsb0 + sb - let w = vk_wq[wsb * 32u + blk * 4u + j] - let xw = (xsb0 + sb) * 64u + blk * 8u + j - let xlo = vk_xq[xw] - let xhi = vk_xq[xw + 4u] - let idot = sdot4(w & 0x0F0F0F0F, xlo) + sdot4((w >> 4u) & 0x0F0F0F0F, xhi) - let bs = sdot4(xlo, 0x01010101) + sdot4(xhi, 0x01010101) + let xb4 = (xsb0 + sb) * 16u + blk * 2u + let xl = vk_xq4[xb4] + let xh = vk_xq4[xb4 + 1u] + let idot = kq_blk_dot(vk_wq4[wsb * 8u + blk], xl, xh) + let bs = kq_blk_sum(xl, xh) let dm = unpackHalf2x16(vk_wsu[wsb * 5u]) let sc = int((vk_wsu[wsb * 5u + 1u + blk / 4u] >> ((blk % 4u) * 8u)) & 0xFF) let mn = int((vk_wsu[wsb * 5u + 3u + blk / 4u] >> ((blk % 4u) * 8u)) & 0xFF) acc += vk_xs[xsb0 + sb] * (dm.x * float(sc * idot) - dm.y * float(mn * bs)) } - b0 += bstride + b0 += gl_SubgroupSize } let total = subgroupAdd(acc) if (subgroupElect()) { @@ -2671,29 +3034,24 @@ def kq_moe_gemv_q40 { let wsb0 = vk_meta[4u + r * 2u] + row * nsb let xsb0 = vk_meta[5u + r * 2u] let lane = gl_SubgroupInvocationID - let j = lane % 4u - let brel = lane / 4u - let bstride = gl_SubgroupSize / 4u let nb = nsb * 8u var acc = 0.0 var b0 = 0u while (b0 < nb) { - let bb = b0 + brel + let bb = b0 + lane if (bb < nb) { let sb = bb / 8u let blk = bb % 8u let wsb = wsb0 + sb - let w = vk_wq[wsb * 32u + blk * 4u + j] - let xw = (xsb0 + sb) * 64u + blk * 8u + j - let xlo = vk_xq[xw] - let xhi = vk_xq[xw + 4u] - let idot = sdot4(w & 0x0F0F0F0F, xlo) + sdot4((w >> 4u) & 0x0F0F0F0F, xhi) - let bs = sdot4(xlo, 0x01010101) + sdot4(xhi, 0x01010101) + let xb4 = (xsb0 + sb) * 16u + blk * 2u + let xl = vk_xq4[xb4] + let xh = vk_xq4[xb4 + 1u] + let idot = kq_blk_dot(vk_wq4[wsb * 8u + blk], xl, xh) let dp = unpackHalf2x16(vk_wsu[wsb * 5u + blk / 2u]) // 8 f16 d packed in words 0..3 let dv = blk % 2u == 0u ? dp.x : dp.y - acc += vk_xs[xsb0 + sb] * dv * float(idot - 8 * bs) + acc += vk_xs[xsb0 + sb] * dv * float(idot - 8 * kq_blk_sum(xl, xh)) } - b0 += bstride + b0 += gl_SubgroupSize } let total = subgroupAdd(acc) if (subgroupElect()) { @@ -2716,32 +3074,26 @@ def kq_moe_gemv_k5 { let wsb0 = vk_meta[4u + r * 2u] + row * nsb let xsb0 = vk_meta[5u + r * 2u] let lane = gl_SubgroupInvocationID - let j = lane % 4u - let brel = lane / 4u - let bstride = gl_SubgroupSize / 4u let nb = nsb * 8u var acc = 0.0 var b0 = 0u while (b0 < nb) { - let bb = b0 + brel + let bb = b0 + lane if (bb < nb) { let sb = bb / 8u let blk = bb % 8u let wsb = wsb0 + sb - let w = vk_wq[wsb * 40u + blk * 4u + j] - let hb = (vk_wq[wsb * 40u + 32u + blk] >> (j * 8u)) & 0xFF - let xw = (xsb0 + sb) * 64u + blk * 8u + j - let xlo = vk_xq[xw] - let xhi = vk_xq[xw + 4u] - let idot = (sdot4((w & 0x0F0F0F0F) | k5_dep(hb & 0xF), xlo) - + sdot4(((w >> 4u) & 0x0F0F0F0F) | k5_dep(hb >> 4u), xhi)) - let bs = sdot4(xlo, 0x01010101) + sdot4(xhi, 0x01010101) + let xb4 = (xsb0 + sb) * 16u + blk * 2u + let xl = vk_xq4[xb4] + let xh = vk_xq4[xb4 + 1u] + let idot = k5_blk_dot(vk_wq4[wsb * 10u + blk], xl, xh, vk_wq[wsb * 40u + 32u + blk]) + let bs = kq_blk_sum(xl, xh) let dm = unpackHalf2x16(vk_wsu[wsb * 5u]) let sc = int((vk_wsu[wsb * 5u + 1u + blk / 4u] >> ((blk % 4u) * 8u)) & 0xFF) let mn = int((vk_wsu[wsb * 5u + 3u + blk / 4u] >> ((blk % 4u) * 8u)) & 0xFF) acc += vk_xs[xsb0 + sb] * (dm.x * float(sc * idot) - dm.y * float(mn * bs)) } - b0 += bstride + b0 += gl_SubgroupSize } let total = subgroupAdd(acc) if (subgroupElect()) { @@ -2766,37 +3118,26 @@ def kq_moe_gemv_k6 { let wsb0 = vk_meta[4u + r * 2u] + row * nsb let xsb0 = vk_meta[5u + r * 2u] let lane = gl_SubgroupInvocationID - let j = lane % 4u - let brel = lane / 4u - let bstride = gl_SubgroupSize / 4u let nb = nsb * 8u var acc = 0.0 var b0 = 0u while (b0 < nb) { - let bb = b0 + brel + let bb = b0 + lane if (bb < nb) { let sb = bb / 8u let blk = bb % 8u let wsb = wsb0 + sb - let w = vk_wq[wsb * 48u + blk * 4u + j] - let qhw = wsb * 48u + 32u + (blk / 4u) * 8u + j - let shift = (blk % 4u) * 2u - let lo6 = (w & 0x0F0F0F0F) | (((vk_wq[qhw] >> shift) & 0x03030303) << 4u) - let hi6 = ((w >> 4u) & 0x0F0F0F0F) | (((vk_wq[qhw + 4u] >> shift) & 0x03030303) << 4u) - let xw = (xsb0 + sb) * 64u + blk * 8u + j - let xlo = vk_xq[xw] - let xhi = vk_xq[xw + 4u] - let ilo = sdot4(lo6, xlo) - let ihi = sdot4(hi6, xhi) - let blo = sdot4(xlo, 0x01010101) - let bhi = sdot4(xhi, 0x01010101) + let qh4 = wsb * 12u + 8u + (blk / 4u) * 2u + let xb4 = (xsb0 + sb) * 16u + blk * 2u + let dot = k6_blk_dot(vk_wq4[wsb * 12u + blk], vk_xq4[xb4], vk_xq4[xb4 + 1u], + vk_wq4[qh4], vk_wq4[qh4 + 1u], (blk % 4u) * 2u) let s4 = int4(unpack8(int(vk_wsu[wsb * 5u + blk / 2u]))) // 4 signed sub-scales let slo = (blk % 2u) == 0u ? s4.x : s4.z let shi = (blk % 2u) == 0u ? s4.y : s4.w let dd = unpackHalf2x16(vk_wsu[wsb * 5u + 4u]).x - acc += vk_xs[xsb0 + sb] * dd * float(slo * (ilo - 32 * blo) + shi * (ihi - 32 * bhi)) + acc += vk_xs[xsb0 + sb] * dd * float(slo * (dot.x - 32 * dot.z) + shi * (dot.y - 32 * dot.w)) } - b0 += bstride + b0 += gl_SubgroupSize } let total = subgroupAdd(acc) if (subgroupElect()) { @@ -3338,9 +3679,9 @@ let private BATCH_Y_BYTES = 67_108_864l // one output plane (two exist: gate/ let private BATCH_META_BYTES = 262_144l // per-stack batch params + regions + workgroup map let private BATCH_TILE = 32l // kernel tile edge (rows and cols) -// the Q8_0 batch kernel's tile edge — 128 for the mul_mm L-tile kernel (mode 3), 32 otherwise. +// the Q8_0 batch kernel's tile edge — 128 for the mul_mm L-tile kernel (modes 3/4), 32 otherwise. // Meta builders MUST use the same edge the bound kernel derives (wtiles = ceil(d / edge)). -def private q8_batch_tile : int64 => g_gpu.coopmat_mode == 3 ? 128l : BATCH_TILE +def private q8_batch_tile : int64 => g_gpu.coopmat_mode >= 3 ? 128l : BATCH_TILE let private MM_SMALL_CNT = 192l // measured crossover (qwen3-4B pp: sdot4 +42% at 128, L +2.5% at 192) @@ -3356,14 +3697,26 @@ def private mm_small_tile : int64 { return g_mm_small } +// the small-d cutoff (DASLLAMA_MM_SMALLD). k/v (d = kv_dim) gets only 32 wgs for 36 SMs at +// d 1024 / cnt 512 — but widening that grid MEASURES WORSE, so k/v is limited by the L-tile's +// arithmetic efficiency, not parallelism: pp512 L128 4297 > M64 4101 > sdot4-32 3906. +var private g_mm_smalld = -1l + +def private mm_small_d : int64 { + if (g_mm_smalld < 0l) { + g_mm_smalld = env_int64("DASLLAMA_MM_SMALLD", 64l) + } + return g_mm_smalld +} + // per-dispatch tile router (the lcpp l/m ladder, ours resolved by measurement): in mm mode a // small-cnt or small-d dispatch drops to the small tier — the L tile starves the grid and pads // its B panel there. Pure in (d, cnt, mode), so pipe pick and meta fill can never disagree. def private q8_batch_tile_for(d, cnt : int64) : int64 { - if (g_gpu.coopmat_mode != 3) { + if (g_gpu.coopmat_mode < 3) { return q8_batch_tile() } - return d <= 64l || cnt < MM_SMALL_CNT ? mm_small_tile() : 128l + return d <= mm_small_d() || cnt < MM_SMALL_CNT ? mm_small_tile() : 128l } // aligned-L eligibility: every tile interior (d and cnt both 128-multiples) — the guard-free @@ -3373,6 +3726,14 @@ def private q8_batch_aligned(d, cnt : int64) : bool def private q8_batch_pipe_for(d, cnt : int64) : Pipeline => q8_batch_pipe(q8_batch_tile_for(d, cnt), q8_batch_aligned(d, cnt)) + +// resident prefill GEMM pipe: a fmt-6 role rides the cm2 twin, everything else the q8 tile router +def private pf_gemm_pipe(fmt6 : bool; d, cnt : int64) : Pipeline { + if (fmt6) { + return weak_copy(g_gpu.batch_cm2) + } + return q8_batch_pipe_for(d, cnt) +} let private BATCH_CHUNK_ROWS = 8_192l // dispatch row-window cap (bounds y and the wg map) let private BATCH_HQ_BYTES = 33_554_432l // device requantized-hidden quants (the down GEMM's activations) let private BATCH_HS_BYTES = 8_388_608l // ... its scale plane @@ -3380,10 +3741,19 @@ let private FFN_META_BYTES = 64l // act/requant params (element count, // + the combine's [n, k, r0, r1] at words 4..7 let private COMB_WI_BYTES = 2_097_152l // combine w / inv plane cap (npos*k grid entries x 4B) let private CLS_META_BYTES = 64l // classifier GEMV params (one region, filled once) -let private STREAM_RESERVE = 1_100_000_000l // VRAM reserved out of the weight budget for the two - // streaming slots when DASLLAMA_GPU_MOE_STREAM is on - // (35B kq measured: 2 x 532.7MB slots = 1065MB; - // ensure_stream_slots asserts actual <= reserve) +var private g_coopmat_mode_force = -1 + +//! Test hook: pin the GEMM mode (0 sdot4 / 1 f16 / 2 int8 / 3 mm) ahead of the mm default and +//! DASLLAMA_COOPMAT — must run before the tier's lazy device init. The CPU-reference tier tests +//! pin 0 so their oracles stay reference-kernel-deterministic under any router default. +def vk_force_coopmat_mode(m : int) { + g_coopmat_mode_force = m +} + +let private STREAM_RESERVE = 1_100_000_000l // LEGACY fallback stream-slot carve — used only when + // the loader never reported the model's exact need + // (moe_gpu_stream_need; big-expert geometries like + // GLM Air exceed this constant by design) // deltanet chain capacities (npos chunks over DN_WINDOW windows, state carried device-side). // Maxima match qwen35 (ds 128, 32 v-heads, conv dim 8192, dconv 4) — vk_moe_dn asserts them @@ -3509,6 +3879,7 @@ struct private StreamSlot { s1 : int // pseudo-stack indices: gate / up / down s3 : int s2 : int + wait_value : uint64 // timeline value of the slot's in-flight transfer-queue fill (0 = none) } // pre-recorded decode-form FFN chain per (gate, up, down) stack triple: GEMV pair, fused @@ -3580,6 +3951,11 @@ struct private GpuState { batch_coopmat_mm : Pipeline // prefill Q8_0 GEMM, the mul_mm 128x128 L-tile (DASLLAMA_COOPMAT=mm) batch_coopmat_mm_m : Pipeline // ... its 64x64 M-tile twin, routed in for small-cnt dispatches batch_coopmat_mm_a : Pipeline // ... the aligned-L twin (no staging guards/edge path — 2.2x the L rate) + batch_cm2 : Pipeline // the cm2 decode-in-load prefill twin (mode 4, fmt-6 q8n stacks) + gemv_q8n : Pipeline // the q8n native-block decode GEMV (mode 4's fmt-6 groupn twin) + xf16_pipeline : Pipeline // cm2 chain: Q8_0 activation image -> f16 plane + actf16_pipeline : Pipeline // cm2 chain: act(g)*u -> f16 hidden plane (the actrq stand-in) + f16cvt_pipeline : Pipeline // arena cm2 chain: f32 rows -> f16 plane (the quant-role stand-in) actrq_pipeline : Pipeline // fused act(g)*u + q8_0 requant of the hidden rows gemv_k4 : Pipeline // native K-quant GEMVs (per-format decode kernels) gemv_k5 : Pipeline @@ -3601,7 +3977,9 @@ struct private GpuState { at_attn_pipeline : Pipeline // flash-style causal attention (GQA, online softmax, out-gate) q8k_rq_pipeline : Pipeline // plain Q8_K requant (the kq wo GEMM's activation image) ar_pipeline : Pipeline // resident driver: fused residual-add + RMSNorm (meta[1]=0 = norm only) + ar_rq_pipeline : Pipeline // resident driver: the above + the Q8_0 requant, one pass (DASLLAMA_VK_FUSE) rks_pipeline : Pipeline // resident driver: fused rope + KV-row store (f32 mirror) + qkn_rope_pipeline : Pipeline // resident driver: per-head q/k rmsnorm + rope + KV store, one pass da_pipeline : Pipeline // resident driver: decode attention over the f32 KV mirror ar_pipeline_b : Pipeline // prefill: batched add+rmsnorm rks_b_pipeline : Pipeline // prefill: batched rope + KV-store @@ -3612,7 +3990,8 @@ struct private GpuState { fence : VkFence // the one reusable submit fence (reset after every wait) rows_per_wg : int64 // subgroups per workgroup = output rows per workgroup has_coopmat : bool // device supports VK_KHR_cooperative_matrix (the f16/int8 tensor tiles) - coopmat_mode : int // Q8_0 prefill GEMM: 0 = sdot4 (default), 1 = f16 coopmat, 2 = int8 coopmat, 3 = mul_mm L-tile + has_coopmat2 : bool // device supports NV_cooperative_matrix2 (tensor-addressed wg tiles + decode fns) + coopmat_mode : int // Q8_0 prefill GEMM: 0 = sdot4, 1 = f16 coopmat, 2 = int8 coopmat, 3 = mul_mm L-tile (default), 4 = cm2 decode-in-load weight_budget : int64 // resident-weight cap — queried heap budget minus reserve, or VRAM_BUDGET @do_not_delete stacks : array // the resident driver's weight arenas, one per format present (see the arena section). Separate @@ -3673,6 +4052,7 @@ struct private GpuState { dev_mem : table // device buffer -> its VkDeviceMemory (rollback destroy needs it) dev_mapped : table // ReBAR: device buffer -> persistent CPU mapping (uploads write direct) rebar_type : int = -1 // DL|HV|HC memory type on the full-VRAM heap (-1 = staging path only) + has_memprio : bool // device enabled VK_EXT_memory_priority — allocations chain priority 1.0 host_mem : table // host-visible buffer -> memory: every make_host_buf result except // the process-lifetime pair (staging, profiler readback) — the // whole-model drop sweeps this @@ -3696,6 +4076,12 @@ struct private GpuState { batch_actrq_set : VkDescriptorSet // batch twin of FfnCmd's fused act+requant set (y1/y2 planes) batch_cmd : VkCommandBuffer // one reusable batch cmd buffer, re-recorded per submit batch_ready : bool + // cm2 batch state (lazy — first q8n batch dispatch creates it; mode 4 only) + batch_xf_dev : uint64 // f16 gathered-activation plane (the gate/up twins' B side) + batch_hf_dev : uint64 // f16 hidden plane (the down twin's B side) + batch_xf_set : VkDescriptorSet // the Q8_0 -> f16 dequant dispatch + batch_hf_set : VkDescriptorSet // the act -> f16 dispatch + cm2_batch_ready : bool // combined-output state (lazy — first combined prefill dispatch creates it) comb_stage : HostBuf // staging: routing w plane + grid->bucket map (device copies at first-chunk head) comb_w_dev : uint64 @@ -3714,6 +4100,16 @@ struct private GpuState { stream_max_dn_wq : int64 stream_max_dn_ws : int64 stream_ready : bool + stream_reserve : int64 // bytes carved out of weight_budget for the 2 stream slots + // dedicated transfer family (phase 5): mirror->slot copies overlap compute, handed off via a + // timeline semaphore the next compute submit waits on. -1 = single-queue mode (the old rail). + xfer_fam : int = -1 + xfer_queue : VkQueue + xfer_pool : CommandPool + xfer_cmd : VkCommandBuffer + xfer_sem : VkSemaphore + xfer_value : uint64 // last signaled timeline value + pend_wait : uint64 // value the NEXT compute submit must wait for (0 = none) // deltanet chain state (lazy — first DN dispatch creates it) dn_meta : HostBuf // dn kernel params staging dn_meta_dev : uint64 @@ -3801,12 +4197,17 @@ def private find_host_mem(type_bits : uint; cached : bool) : uint { return find_memory_type(g_gpu.phys, type_bits, want) } -def private make_host_buf(bytes : int64; storage : bool; cached : bool = false) : HostBuf { +def private make_host_buf(bytes : int64; storage : bool; cached : bool = false; xfer_shared : bool = false) : HostBuf { var bci : BufferCreateInfo bci.size = uint64(bytes) bci.usage.storage_buffer = storage bci.usage.transfer_src = !storage bci.usage.transfer_dst = true // readback staging targets (y DMA copies land here) + if (xfer_shared && g_gpu.xfer_fam >= 0) { + // stream mirrors: transfer-queue fills + compute-queue heat slices — CONCURRENT dodges ownership transfers + bci.sharingMode = VkSharingMode.CONCURRENT + bci.pQueueFamilyIndices <- [g_gpu.fam, uint(g_gpu.xfer_fam)] + } var b <- create_buffer(g_gpu.device, bci) // deliberately never finalized (process-lifetime) let req = get_buffer_memory_requirements(g_gpu.device, b) let mai = MemoryAllocateInfo(allocationSize = req.size, @@ -3822,19 +4223,32 @@ def private make_host_buf(bytes : int64; storage : bool; cached : bool = false) return HostBuf(buf = b._vk, mem = m._vk, mapped = mapped, bytes = bytes) } -def private make_device_buf(bytes : int64) : uint64 { +def private make_device_buf(bytes : int64; xfer_shared : bool = false) : uint64 { var bci : BufferCreateInfo bci.size = uint64(bytes) bci.usage.storage_buffer = true bci.usage.transfer_dst = true bci.usage.transfer_src = true // y planes DMA back out (device -> host staging) + if (xfer_shared && g_gpu.xfer_fam >= 0) { + // stream slots: transfer-queue fills + compute-queue reads — CONCURRENT dodges ownership transfers + bci.sharingMode = VkSharingMode.CONCURRENT + bci.pQueueFamilyIndices <- [g_gpu.fam, uint(g_gpu.xfer_fam)] + } var b <- create_buffer(g_gpu.device, bci) let req = get_buffer_memory_requirements(g_gpu.device, b) + // priority 1.0 (default 0.5): under pressure the driver demotes lower-priority (desktop) memory first + var prio = VkMemoryPriorityAllocateInfoEXT() + prio.priority = 1.0 if (g_gpu.rebar_type >= 0 && (req.memoryTypeBits & (1u << uint(g_gpu.rebar_type))) != 0u) { // ReBAR attempt — raw alloc so a decline falls back to plain device-local below var rmai = VkMemoryAllocateInfo() rmai.allocationSize = req.size rmai.memoryTypeIndex = uint(g_gpu.rebar_type) + if (g_gpu.has_memprio) { + unsafe { + rmai.pNext = addr(prio) + } + } var rmem : VkDeviceMemory if (vkAllocateMemory(dev_raw(), rmai, null, rmem) == VkResult.SUCCESS) { vk_check(vkBindBufferMemory(dev_raw(), unsafe(reinterpret(b._vk)), rmem, 0ul), null) @@ -3850,8 +4264,13 @@ def private make_device_buf(bytes : int64) : uint64 { } var want : VkMemoryPropertyFlags want.device_local = true - let mai = MemoryAllocateInfo(allocationSize = req.size, + var mai = MemoryAllocateInfo(allocationSize = req.size, memoryTypeIndex = find_memory_type(g_gpu.phys, req.memoryTypeBits, want)) + if (g_gpu.has_memprio) { + unsafe { + mai.next = addr(prio) + } + } var m <- allocate_memory(g_gpu.device, mai) bind_buffer_memory(g_gpu.device, b, m, 0ul) g_gpu.dev_mem |> insert(b._vk, m._vk) @@ -3907,19 +4326,41 @@ def private vk_moe_init : bool { g_gpu.fam = select_graphics_queue_family(g_gpu.phys) // coopmat requested whenever supported — the fa twin needs it even in sdot4 GEMM mode g_gpu.has_coopmat = cooperative_matrix_supported(g_gpu.phys) - g_gpu.coopmat_mode = 0 + g_gpu.has_coopmat2 = g_gpu.has_coopmat && cooperative_matrix2_supported(g_gpu.phys) + // fastest MEASURED default (mm; cm2 opt-in until it beats mm); DASLLAMA_COOPMAT overrides + g_gpu.coopmat_mode = g_gpu.has_coopmat ? 3 : 0 if (g_gpu.has_coopmat) { let mode = env_str("DASLLAMA_COOPMAT", "") - if (mode == "f16") { + if (mode == "sdot4") { + g_gpu.coopmat_mode = 0 + } elif (mode == "f16") { g_gpu.coopmat_mode = 1 } elif (mode == "int8") { g_gpu.coopmat_mode = 2 } elif (mode == "mm") { g_gpu.coopmat_mode = 3 + } elif (mode == "cm2" && g_gpu.has_coopmat2) { + g_gpu.coopmat_mode = 4 } } - if (g_gpu.has_coopmat) { - g_gpu.device <- create_device_storage_8_16_int_dot_coopmat(g_gpu.phys, g_gpu.fam) + if (g_coopmat_mode_force >= 0) { + g_gpu.coopmat_mode = g_gpu.has_coopmat ? g_coopmat_mode_force : 0 + } + if (g_gpu.coopmat_mode == 4 && !g_gpu.has_coopmat2) { + g_gpu.coopmat_mode = 3 // forced/requested cm2 without the extension: mm + } + // dedicated transfer family: streamed copies overlap compute (DASLLAMA_VK_XFERQ=0 = bisect hatch) + var xfam = -1 + if (g_gpu.has_coopmat && env_flag("DASLLAMA_VK_XFERQ", true) + && timeline_semaphore_supported(g_gpu.phys)) { + xfam = select_transfer_queue_family(g_gpu.phys) + } + if (g_gpu.has_coopmat2 && g_gpu.coopmat_mode == 4) { + // #77's creator: coopmat2 + BDA + vulkanMemoryModel + the query-then-enable extras + g_gpu.device <- create_device_storage_8_16_int_dot_coopmat2(g_gpu.phys, g_gpu.fam, xfam) + to_log(LOG_INFO, "dasLLAMA vulkan tier: Q8_0 prefill on NV_coopmat2 tensor tiles (decode-in-load)\n") + } elif (g_gpu.has_coopmat) { + g_gpu.device <- create_device_storage_8_16_int_dot_coopmat(g_gpu.phys, g_gpu.fam, xfam) if (g_gpu.coopmat_mode != 0) { let cm_name = g_gpu.coopmat_mode == 1 ? "f16" : (g_gpu.coopmat_mode == 2 ? "int8" : "mul_mm L-tile") to_log(LOG_INFO, "dasLLAMA vulkan tier: Q8_0 prefill on {cm_name} cooperative-matrix tiles\n") @@ -3929,6 +4370,34 @@ def private vk_moe_init : bool { } volkLoadDevice(dev_raw()) g_gpu.queue = get_device_queue(g_gpu.device, g_gpu.fam, 0u) + if (xfam >= 0) { + g_gpu.xfer_fam = xfam + g_gpu.xfer_queue = get_device_queue(g_gpu.device, uint(xfam), 0u) + var xpci : CommandPoolCreateInfo + xpci.queueFamilyIndex = uint(xfam) + xpci.flags.reset_command_buffer = true + g_gpu.xfer_pool <- create_command_pool(g_gpu.device, xpci) + var xai = VkCommandBufferAllocateInfo() + xai.commandPool = boost_value_to_vk(g_gpu.xfer_pool) + xai.level = VkCommandBufferLevel.PRIMARY + xai.commandBufferCount = 1u + var tci = VkSemaphoreTypeCreateInfo() + tci.semaphoreType = VkSemaphoreType.TIMELINE + tci.initialValue = 0ul + var sci = VkSemaphoreCreateInfo() + unsafe { + vk_check(vkAllocateCommandBuffers(dev_raw(), xai, addr(g_gpu.xfer_cmd)), null) + sci.pNext = addr(tci) + vk_check(vkCreateSemaphore(dev_raw(), sci, null, g_gpu.xfer_sem), null) + } + to_log(LOG_INFO, "dasLLAMA vulkan tier: transfer queue armed (family {xfam})\n") + } + // WDDM residency shield: allocations chain priority 1.0 (DASLLAMA_VK_MEMPRIO=0 = bisect hatch) + g_gpu.has_memprio = (g_gpu.has_coopmat && memory_priority_supported(g_gpu.phys) + && env_flag("DASLLAMA_VK_MEMPRIO", true)) + if (g_gpu.has_memprio) { + to_log(LOG_INFO, "dasLLAMA vulkan tier: memory-priority residency shield armed\n") + } let sg = subgroup_properties(g_gpu.phys).subgroupSize if (sg < 32u) { // the gid-mapped quant kernels reduce with xor masks up to 16 (q8k amax spans 32 lanes) @@ -3969,8 +4438,15 @@ def private vk_moe_init : bool { to_log(LOG_INFO, "dasLLAMA vulkan tier: device heap budget {int64(mb.heapBudget[best]) / 1000000l} MB (used {int64(mb.heapUsage[best]) / 1000000l}), weight cap {g_gpu.weight_budget / 1000000l} MB\n") } } - if (gpu_want_moe_stream() > 0l) { - g_gpu.weight_budget = max(g_gpu.weight_budget - STREAM_RESERVE, 0l) // room for the 2 slots + // stream-slot carve: exact loader-reported need, else the legacy constant (direct-upload paths) + if (gpu_want_moe_stream() != 0l) { + let need = moe_gpu_stream_need() + if (need > 0l) { + g_gpu.stream_reserve = 2l * need + } elif (gpu_want_moe_stream() > 0l) { + g_gpu.stream_reserve = STREAM_RESERVE + } + g_gpu.weight_budget = max(g_gpu.weight_budget - g_gpu.stream_reserve, 0l) } // ReBAR: DL|HV|HC on the full-VRAM heap (the >1GB check excludes the 256MiB legacy window) if (env_flag("DASLLAMA_VK_REBAR", true)) { @@ -4021,10 +4497,18 @@ def private vk_moe_init : bool { g_gpu.batch_coopmat_q40 <- make_kernel_pipe("kq_moe_batch_q40_cmf16", kq_moe_batch_q40_cmf16_spv) } elif (g_gpu.coopmat_mode == 2) { // native-s8 tensor-core prefill GEMM g_gpu.batch_coopmat_i8 <- make_kernel_pipe("q8_moe_batch_cmi8", q8_moe_batch_cmi8_spv) - } elif (g_gpu.coopmat_mode == 3) { // the mul_mm L/M-tile prefill GEMMs (routed per dispatch) + } elif (g_gpu.coopmat_mode >= 3) { // the mul_mm L/M-tile prefill GEMMs (routed per dispatch; + // mode 4 keeps the whole mm ladder for fmt-0 stacks and adds the cm2 rails for fmt-6) g_gpu.batch_coopmat_mm <- make_kernel_pipe("q8_moe_batch_mm", q8_moe_batch_mm_spv) g_gpu.batch_coopmat_mm_m <- make_kernel_pipe("q8_moe_batch_mm_m", q8_moe_batch_mm_m_spv) g_gpu.batch_coopmat_mm_a <- make_kernel_pipe("q8_moe_batch_mm_a", q8_moe_batch_mm_a_spv) + if (g_gpu.coopmat_mode == 4) { + g_gpu.batch_cm2 <- make_kernel_pipe("q8n_moe_batch_cm2", q8n_moe_batch_cm2_spv) + g_gpu.gemv_q8n <- make_kernel_pipe("q8n_moe_groupn", q8n_moe_groupn_spv) + g_gpu.xf16_pipeline <- make_kernel_pipe("q8_moe_xf16", q8_moe_xf16_spv) + g_gpu.actf16_pipeline <- make_kernel_pipe("q8_moe_actf16", q8_moe_actf16_spv) + g_gpu.f16cvt_pipeline <- make_kernel_pipe("rd_f16cvt", rd_f16cvt_spv) + } } g_gpu.actrq_pipeline <- make_kernel_pipe("q8_moe_actrq", q8_moe_actrq_spv) g_gpu.gemv_k4 <- make_kernel_pipe("kq_moe_gemv_k4", kq_moe_gemv_k4_spv) @@ -4056,6 +4540,8 @@ def private vk_moe_init : bool { g_gpu.da_b128_pipeline <- make_kernel_pipe("da_attn_b_h128", da_attn_b_h128_spv) } g_gpu.qkn_pipeline <- make_kernel_pipe("qk_rms", qk_rms_spv) + g_gpu.ar_rq_pipeline <- make_kernel_pipe("ar_add_rms_rq", ar_add_rms_rq_spv) + g_gpu.qkn_rope_pipeline <- make_kernel_pipe("qkn_rope_kv", qkn_rope_kv_spv) var fence <- create_fence(g_gpu.device, FenceCreateInfo()) // never finalized (process-lifetime) g_gpu.fence = boost_value_to_vk(fence) @@ -4091,14 +4577,14 @@ def private upload_region_at(dst : uint64; dst_off : int64; src : uint8 const?; let t0 = ref_time_ticks() if (key_exists(g_gpu.dev_mapped, dst)) { unsafe { - par_memcpy(reinterpret(g_gpu.dev_mapped[dst] + uint64(dst_off)), reinterpret(src), total) + par_memcpy(reinterpret(g_gpu.dev_mapped[dst] + uint64(dst_off)), reinterpret(src), total) } } else { var off = 0l while (off < total) { let chunk = min(total - off, STAGING_BYTES) unsafe { - par_memcpy(g_gpu.staging.mapped, reinterpret(src + off), chunk) + par_memcpy(g_gpu.staging.mapped, reinterpret(src + off), chunk) } staged_upload(dst, dst_off + off, chunk) off += chunk @@ -4143,13 +4629,17 @@ def private stack_plane_bytes(n, rows : int64; fmt : int) : tuple push(st) return length(g_gpu.stacks) - 1 } +// the descriptor range for a stack's ws binding — the 64B floor of the shell above +def private ws_range(si : int) : int64 => max(g_gpu.stacks[si].wsbytes, 64l) + // decode/GEMV scratch for one stack (per-stack host bufs + the written 6-binding set). Lazy: // only stacks that actually serve GEMV or cls dispatches pay for it — dense-GEMM planes never do. def private ensure_gemv_scratch(si : int) { @@ -4164,7 +4654,7 @@ def private ensure_gemv_scratch(si : int) { g_gpu.stacks[si].y_dev = make_device_buf(Y_BYTES) var writes : array write_buf_desc(writes, g_gpu.stacks[si].raw_set, 0u, g_gpu.stacks[si].wqbuf, g_gpu.stacks[si].wqbytes) - write_buf_desc(writes, g_gpu.stacks[si].raw_set, 1u, g_gpu.stacks[si].wsbuf, g_gpu.stacks[si].wsbytes) + write_buf_desc(writes, g_gpu.stacks[si].raw_set, 1u, g_gpu.stacks[si].wsbuf, ws_range(si)) write_buf_desc(writes, g_gpu.stacks[si].raw_set, 2u, g_gpu.stacks[si].meta.buf, META_BYTES) write_buf_desc(writes, g_gpu.stacks[si].raw_set, 3u, g_gpu.stacks[si].xq.buf, XQ_BYTES) write_buf_desc(writes, g_gpu.stacks[si].raw_set, 4u, g_gpu.stacks[si].xs.buf, XS_BYTES) @@ -4181,10 +4671,13 @@ def private ensure_gemv_scratch(si : int) { def private vk_moe_stream_stack(wq : uint8 const?; ws : uint8 const?; woff : int64; n, rows : int64; fmt : int) : bool { let pb = stack_plane_bytes(n, rows, fmt) var ss = StreamStack(base = woff, elems = rows * n, fmt = fmt, wqbytes = pb.wq, wsbytes = pb.ws, - hwq = make_host_buf(pb.wq, false), hws = make_host_buf(pb.ws, false)) + hwq = make_host_buf(pb.wq, false, [xfer_shared = true]), + hws = make_host_buf(max(pb.ws, 64l), false, [xfer_shared = true])) unsafe { - par_memcpy(ss.hwq.mapped, reinterpret(wq), pb.wq) - par_memcpy(ss.hws.mapped, reinterpret(ws), pb.ws) + par_memcpy(ss.hwq.mapped, reinterpret(wq), pb.wq) + if (pb.ws > 0l) { // q8n mirrors carry no scale plane + par_memcpy(ss.hws.mapped, reinterpret(ws), pb.ws) + } } if (length(g_gpu.stream_stacks) % 3 == 2) { // walk order gate/up/down: this one is a down g_gpu.stream_max_dn_wq = max(g_gpu.stream_max_dn_wq, pb.wq) @@ -4255,6 +4748,10 @@ def vk_drop_model_state { return } vk_check(vkQueueWaitIdle(g_gpu.queue), null) + if (g_gpu.xfer_fam >= 0 && g_gpu.xfer_value > 0ul) { + xfer_host_wait(g_gpu.xfer_value) // in-flight transfer-queue copies retire before the sweeps + } + g_gpu.pend_wait = 0ul assert(g_gpu.ffn_pend_s2 == -1, "dasLLAMA vulkan tier: model drop with an async FFN in flight") // the resident driver's das-side shell first (its VK objects die in the sweeps below) if (g_rd != null) { @@ -4382,18 +4879,38 @@ struct private ArenaFmt { // per-block plane strides — the block is 32 weights for q8 and a 256-weight superblock for kq. // Mirrors stack_plane_bytes: it computes the same totals as count * these strides. +// the tier's stack fmt space is int(KqFmt) — bridge to the gemm_schema kq id space (4/5/6/40) +def private vk_kq_schema_id(fmt : int) : int { + if (fmt == 1) return 4 + if (fmt == 2) return 5 + if (fmt == 3) return 6 + if (fmt == 4) return 40 + panic("vk_kq_schema_id: not a kq stack fmt") + return 0 +} + +// fmt space here = int(KqFmt): 0 q8, 6 q8n (both 32-elem blocks), 1-4 the kq lattice + q40 +def private vk_fmt_b32(fmt : int) : bool => fmt == 0 || fmt == 6 + +// meta/activation block unit of a stack format — q8 AND q8n count 32-weight blocks, the kq +// lattice counts 256-weight superblocks +def private fmt_unit(fmt : int) : int64 => vk_fmt_b32(fmt) ? 32l : 256l + def private arena_block_bytes(fmt : int) : tuple { if (fmt == 0) { - return (wq = 32l, ws = 2l) // 32 int8 quants + one f16 scale + return (wq = Q8_QPB, ws = Q8_SPB) // 32 int8 quants + one f16 scale + } + if (fmt == 6) { + return (wq = Q8N_BPB, ws = 0l) // q8n: one interleaved 34B plane, scale inside } - let q = fmt == 2 ? 160l : (fmt == 3 ? 192l : 128l) // k4 and q40 share the 128B nibble tiling - return (wq = q, ws = 20l) // decoded scale row: [f16 d][f16 dmin][8 sc][8 mn] + // decoded scale row: [f16 d][f16 dmin][8 sc][8 mn] + return (wq = kq_qsb(vk_kq_schema_id(fmt)), ws = KQ_DEV_SSB) } def private arena_blocks_for(n, rows : int64; fmt : int) : int64 { - assert(fmt == 0 || n % 256l == 0l, "dasLLAMA vulkan arena: kq row length must be a superblock multiple") - assert(fmt != 0 || n % 32l == 0l, "dasLLAMA vulkan arena: q8 row length must be a 32-block multiple") - return rows * (n / (fmt == 0 ? 32l : 256l)) + assert(vk_fmt_b32(fmt) || n % KQ_SUPERBLOCK_ELEMS == 0l, "dasLLAMA vulkan arena: kq row length must be a superblock multiple") + assert(!vk_fmt_b32(fmt) || n % Q8_BLOCK_ELEMS == 0l, "dasLLAMA vulkan arena: q8 row length must be a 32-block multiple") + return rows * (n / (vk_fmt_b32(fmt) ? Q8_BLOCK_ELEMS : KQ_SUPERBLOCK_ELEMS)) } // Reserve one format's arena up front. Sizing is a decision, not a walk: the resident driver knows @@ -4408,7 +4925,8 @@ def private arena_reserve(fmt : int; cap_blocks : int64) : bool { } let bb = arena_block_bytes(fmt) let qbytes = cap_blocks * bb.wq - let sbytes = cap_blocks * bb.ws + // q8n has no scale plane — floor the ws buffer so its descriptor slots stay valid (stack-shell rule) + let sbytes = max(cap_blocks * bb.ws, 64l) // maxStorageBufferRange is the real ceiling (uint32 indexing reaches 16GiB); NV/AMD report 4GiB var lp : VkPhysicalDeviceProperties vkGetPhysicalDeviceProperties(g_gpu.phys, lp) @@ -4439,7 +4957,9 @@ def private arena_place(wq : uint8 const?; ws : uint8 const?; n, rows : int64; f a.blocks += need let bb = arena_block_bytes(fmt) upload_region_at(a.wqbuf, blk * bb.wq, wq, need * bb.wq) - upload_region_at(a.wsbuf, blk * bb.ws, ws, need * bb.ws) + if (bb.ws > 0l) { + upload_region_at(a.wsbuf, blk * bb.ws, ws, need * bb.ws) + } return blk } } @@ -4456,10 +4976,11 @@ def vk_arena_reserve(fmt : int; cap_blocks : int64) : bool { //! Arena seam: place + upload one [rows x n] tensor from the loader's gathered device-layout //! planes. Returns its block index (what the meta's weight base wants), or -1 if exhausted. def vk_arena_place(wq : array; ws : array; n : int64; rows : int64; fmt : int) : int64 { - if (!vk_moe_init() || empty(wq) || empty(ws)) { + // q8n (fmt 6) carries its scale inside the interleaved plane — an EMPTY ws is its contract + if (!vk_moe_init() || empty(wq) || (empty(ws) && fmt != 6)) { return -1l } - return arena_place(unsafe(addr(wq[0])), unsafe(addr(ws[0])), n, rows, fmt) + return arena_place(unsafe(addr(wq[0])), empty(ws) ? null : unsafe(addr(ws[0])), n, rows, fmt) } // Bindings 0/1 are the arena planes; 2-5 are per-dispatch scratch, written exactly as @@ -4746,7 +5267,7 @@ def vk_rope_kv_store(var qkv : array; var qout : array; var krow : mp[6] = uint(npairs) mp[7] = uint(pos * kv_dim) // K mirror row offset (f32 elements) mp[8] = uint(pos * kv_dim) // V mirror row offset - memcpy(g_gpu.rks_scal.mapped, addr(cossin[0]), head_size * 4l) // cos[half] + sin[half] + memcpy(g_gpu.rks_scal.mapped, addr(cossin[0]), head_size * 4l) // cos[half] + sin[half] upload_region_at(g_gpu.rks_qkv_dev, 0l, addr(qkv[0]), (qd + 2l * kv_dim) * 4l) // q enters the kernel via vk_y (binding 5) = rks_q_dev — seed it with the raw q rows upload_region_at(g_gpu.rks_q_dev, 0l, addr(qkv[0]), qd * 4l) @@ -4781,18 +5302,21 @@ struct private RLayer { s_kv : VkDescriptorSet // merged k+v two-region GEMV (fk == fv: same arena planes) kv_merged : bool s_qkn : VkDescriptorSet // per-head q/k rmsnorm (qk_norm models only) + s_qkn_rope : VkDescriptorSet // the fused qk-rmsnorm + rope + KV store (qk_norm models only) s_rope : VkDescriptorSet s_attn : VkDescriptorSet s_quant_at : VkDescriptorSet s_wo : VkDescriptorSet s_addr_ffn : VkDescriptorSet - s_quant_xb2 : VkDescriptorSet + s_addr_ffn_rq : VkDescriptorSet // the fused add+rms+requant twins of addr_ffn / addr_next: + s_quant_xb2 : VkDescriptorSet // same inputs, but xq/xs at bindings 4-5 instead of xb s_gate, s_up : VkDescriptorSet s_actrq : VkDescriptorSet s_down : VkDescriptorSet s_addr_next : VkDescriptorSet + s_addr_next_rq : VkDescriptorSet // per-dispatch metas (host-visible; contents rewritten per token) - m_quant_xb, m_q, m_k, m_v, m_qkn, m_rope, m_attn, m_quant_at, m_wo, m_addr_ffn : HostBuf + m_quant_xb, m_q, m_k, m_v, m_qkn, m_qkr, m_rope, m_attn, m_quant_at, m_wo, m_addr_ffn : HostBuf m_quant_xb2, m_gate, m_up, m_actrq, m_down, m_addr_next : HostBuf next_norm_off : int64 // rms_ffn is w_off for addr_ffn; addr_next uses rms_att[l+1] or final made : bool @@ -4804,6 +5328,11 @@ struct private RDec { neox : bool qk_norm : bool // per-head q/k rmsnorm role armed (norms buffer carries the rows) norms_bytes : int64 // full norms buffer size (grows past the fixed layout under qk_norm) + ar_scal_off : int64 // eps/ascale pair, appended to the norms buffer (float units) — + // the fused ar+requant needs binding 4 for its scale output + cos_elems : int64 // cos/sin row + (under qk_norm) the rms_q/rms_k rows + eps + qkn_row_off : int64 // rms_q[0] in the cos buffer (float units); rms_k[l] follows each + qkn_eps_off : int64 // eps, past the rms rows in the cos buffer eps, scale : float // per-token device buffers x_dev, xb_dev : uint64 // residual, normed activation @@ -4825,6 +5354,7 @@ struct private RDec { cls_meta_dev : uint64 cls_set : VkDescriptorSet s_prologue : VkDescriptorSet // xb = rms_att[0](x) + s_prologue_rq : VkDescriptorSet // the fused twin (also does layer 0's activation quant) m_prologue : HostBuf s_quant_final : VkDescriptorSet m_quant_final : HostBuf @@ -4842,6 +5372,7 @@ struct private RDec { pf_q, pf_kv, pf_v, pf_attn, pf_aq, pf_as : uint64 pf_gate, pf_up, pf_ffnout, pf_xb2 : uint64 pf_cos : uint64 // [MAX_NPOS x hs] rope rows + pf_xf, pf_af, pf_hf : uint64 // cm2 (fmt-6) f16 twin feeds: xb rows, attn rows, hidden rows pf_logits_host : HostBuf pf_meta : array // one host meta per (layer, role) + prologue/final/cls pf_sets : array @@ -4891,6 +5422,12 @@ def vk_rdec_prepare(n_layers, dim, qd, kv_dim, head_size, n_heads, hidden, vocab to_log(LOG_WARNING, "dasLLAMA vulkan resident: KV mirror ({n_layers} x {seq_cap} x {kv_dim}) exceeds 32-bit index / 2GiB shard\n") return false } + // the residual-stream kernels stage a whole row in ar_row, and the per-head ones a whole head + // in qr_row/da_q — over either slab they would write past the workgroup array with no diagnostic + if (dim > AR_MAX_DIM || head_size > 256l) { + to_log(LOG_WARNING, "dasLLAMA vulkan resident: dim {dim} > {AR_MAX_DIM} or head_size {head_size} > 256 exceeds the workgroup row slabs\n") + return false + } g_rd = new RDec(ready = false, n_layers = n_layers, dim = dim, qd = qd, kv_dim = kv_dim, head_size = head_size, n_heads = n_heads, kv_mul = n_heads / (kv_dim / head_size), hidden = hidden, vocab = vocab, seq_cap = seq_cap, neox = neox, eps = eps, scale = scale, @@ -4913,10 +5450,17 @@ def vk_rdec_prepare(n_layers, dim, qd, kv_dim, head_size, n_heads, hidden, vocab r.k_mirror = make_device_buf(n_layers * seq_cap * kv_dim * 4l) r.v_mirror = make_device_buf(n_layers * seq_cap * kv_dim * 4l) // norms layout: [rms_att[l], rms_ffn[l]] x L + rms_final (dim each), then under qk_norm the - // per-head rows [rms_q[l], rms_k[l]] x L (head_size each) appended past the fixed layout - r.norms_bytes = (n_layers * 2l * dim + dim + (qk_norm ? n_layers * 2l * head_size : 0l)) * 4l + // per-head rows [rms_q[l], rms_k[l]] x L (head_size each) appended past the fixed layout, + // then the eps/ascale pair the fused ar+requant reads (its binding 4 is the scale output) + r.ar_scal_off = n_layers * 2l * dim + dim + (qk_norm ? n_layers * 2l * head_size : 0l) + r.norms_bytes = (r.ar_scal_off + 2l) * 4l r.norms_dev = make_device_buf(r.norms_bytes) - r.cos_dev = make_device_buf(head_size * 4l) + // cos layout: this token's cos[half]+sin[half], then under qk_norm the rms_q/rms_k rows and + // eps — the fused qk-norm+rope kernel's binding 3 is the V mirror, so they cannot ride norms + r.qkn_row_off = head_size + r.qkn_eps_off = head_size + (qk_norm ? n_layers * 2l * head_size : 0l) + r.cos_elems = r.qkn_eps_off + (qk_norm ? 1l : 0l) + r.cos_dev = make_device_buf(r.cos_elems * 4l) r.scal_dev = make_device_buf(64l) r.ar_scal_dev = make_device_buf(64l) r.logits_dev = make_device_buf(vocab * 4l) @@ -4937,6 +5481,19 @@ def vk_rdec_prepare(n_layers, dim, qd, kv_dim, head_size, n_heads, hidden, vocab def vk_rdec_upload_norms(norms : array) { assert(g_rd != null, "vk_rdec_upload_norms before prepare") upload_region_at(g_rd.norms_dev, 0l, unsafe(addr(norms[0])), long_length(norms) * 4l) + // the fused-rail constants: eps/ascale past the norm rows, and (under qk_norm) a copy of the + // per-head rms rows + eps in the cos buffer, where the fused qk-norm+rope kernel can reach them + var pair : float[2] + pair[0] = g_rd.eps + pair[1] = 1.0 // ascale — the residual add is unscaled in the decode driver + upload_region_at(g_rd.norms_dev, g_rd.ar_scal_off * 4l, unsafe(addr(pair[0])), 8l) + if (g_rd.qk_norm) { + let rows = g_rd.n_layers * 2l * g_rd.head_size + let src = g_rd.n_layers * 2l * g_rd.dim + g_rd.dim // rms_q[0] within `norms` + assert(long_length(norms) >= src + rows, "vk_rdec_upload_norms: qk_norm rows missing") + upload_region_at(g_rd.cos_dev, g_rd.qkn_row_off * 4l, unsafe(addr(norms[src])), rows * 4l) + upload_region_at(g_rd.cos_dev, g_rd.qkn_eps_off * 4l, unsafe(addr(pair[0])), 4l) + } // norms are the last upload of the resident arm sequence — report the accumulated cost if (g_vk_upload_us > 0l) { let gbps = double(g_vk_upload_bytes) / (double(g_vk_upload_us) * 1000.0lf) @@ -4994,14 +5551,19 @@ def vk_rdec_set_layer(l : int64; bq, bk, bv, bo, b1, b3, b2 : int64; fq, fk, fv, fixed_array(0u, 0u, 0u, VHZ_XQ, VHZ_XQ, VHZ_KVK | VHZ_KVV)) } // qk rmsnorm (qk_norm models): q@0 (q_dev), meta@2, norm rows@3, eps@4, k@5 (kv_dev, k at qd) + let mirbytes = g_rd.n_layers * g_rd.seq_cap * kvd * 4l if (g_rd.qk_norm) { L.m_qkn = rd_meta() L.s_qkn = rd_set6(g_rd.q_dev, g_rd.dummy, L.m_qkn.buf, g_rd.norms_dev, g_rd.ar_scal_dev, g_rd.kv_dev, qd * 4l, 256l, META_BYTES, g_rd.norms_bytes, 64l, (qd + 2l * kvd) * 4l, fixed_array(VHZ_Q, 0u, 0u, 0u, 0u, VHZ_KVK)) + // the fused twin: rope's own binding plan, with the rms rows riding the cos buffer + L.m_qkr = rd_meta() + L.s_qkn_rope = rd_set6(g_rd.kv_dev, g_rd.k_mirror, L.m_qkr.buf, g_rd.v_mirror, g_rd.cos_dev, g_rd.q_dev, + (qd + 2l * kvd) * 4l, mirbytes, META_BYTES, mirbytes, g_rd.cos_elems * 4l, qd * 4l, + fixed_array(VHZ_KVK | VHZ_KVV, VHZ_MIR, 0u, VHZ_MIR, VHZ_COS, VHZ_Q)) } // rope+store: kv@0, Kmirror@1, meta@2, Vmirror@3, cossin@4, q@5 - let mirbytes = g_rd.n_layers * g_rd.seq_cap * kvd * 4l L.s_rope = rd_set6(g_rd.kv_dev, g_rd.k_mirror, L.m_rope.buf, g_rd.v_mirror, g_rd.cos_dev, g_rd.q_dev, (qd + 2l * kvd) * 4l, mirbytes, META_BYTES, mirbytes, g_rd.head_size * 4l, qd * 4l, fixed_array(VHZ_KVK | VHZ_KVV, VHZ_MIR, 0u, VHZ_MIR, VHZ_COS, VHZ_Q)) @@ -5021,6 +5583,10 @@ def vk_rdec_set_layer(l : int64; bq, bk, bv, bo, b1, b3, b2 : int64; fq, fk, fv, L.s_addr_ffn = rd_set6(g_rd.x_dev, g_rd.xb2_dev, L.m_addr_ffn.buf, g_rd.norms_dev, g_rd.ar_scal_dev, g_rd.xb_dev, dim * 4l, dim * 4l, META_BYTES, g_rd.norms_bytes, 64l, dim * 4l, fixed_array(VHZ_X, VHZ_XB2, 0u, 0u, 0u, VHZ_XB)) + // fused twin: the normed row quantizes in place of the xb store (xs@4, xq@5) + L.s_addr_ffn_rq = rd_set6(g_rd.x_dev, g_rd.xb2_dev, L.m_addr_ffn.buf, g_rd.norms_dev, g_rd.xs_dev, g_rd.xq_dev, + dim * 4l, dim * 4l, META_BYTES, g_rd.norms_bytes, (dim / 32l) * 4l, dim, + fixed_array(VHZ_X, VHZ_XB2, 0u, 0u, VHZ_XQ, VHZ_XQ)) // quantize xb (post-ffn-norm) -> xq/xs L.s_quant_xb2 = rd_set6(g_rd.xb_dev, g_rd.dummy, L.m_quant_xb2.buf, g_rd.xq_dev, g_rd.xs_dev, g_rd.dummy, dim * 4l, 256l, META_BYTES, dim, (dim / 32l) * 4l, 256l, @@ -5044,6 +5610,11 @@ def vk_rdec_set_layer(l : int64; bq, bk, bv, bo, b1, b3, b2 : int64; fq, fk, fv, L.s_addr_next = rd_set6(g_rd.x_dev, g_rd.ffnout_dev, L.m_addr_next.buf, g_rd.norms_dev, g_rd.ar_scal_dev, g_rd.xb_dev, dim * 4l, dim * 4l, META_BYTES, g_rd.norms_bytes, 64l, dim * 4l, fixed_array(VHZ_X, VHZ_FFO, 0u, 0u, 0u, VHZ_XB)) + // fused twin — on the last layer this also serves the final norm, so the split rail's + // separate quant_final dispatch has no fused counterpart + L.s_addr_next_rq = rd_set6(g_rd.x_dev, g_rd.ffnout_dev, L.m_addr_next.buf, g_rd.norms_dev, g_rd.xs_dev, g_rd.xq_dev, + dim * 4l, dim * 4l, META_BYTES, g_rd.norms_bytes, (dim / 32l) * 4l, dim, + fixed_array(VHZ_X, VHZ_FFO, 0u, 0u, VHZ_XQ, VHZ_XQ)) L.made = true } g_rd.tok_recorded = false // the recorded chain references the replaced sets @@ -5064,6 +5635,10 @@ def vk_rdec_set_cls(cls_block : int64; cls_fmt : int) { g_rd.s_prologue = rd_set6(g_rd.x_dev, g_rd.dummy, g_rd.m_prologue.buf, g_rd.norms_dev, g_rd.ar_scal_dev, g_rd.xb_dev, dim * 4l, 256l, META_BYTES, g_rd.norms_bytes, 64l, dim * 4l, fixed_array(VHZ_X, 0u, 0u, 0u, 0u, VHZ_XB)) + // fused twin — subsumes layer 0's quant_xb + g_rd.s_prologue_rq = rd_set6(g_rd.x_dev, g_rd.dummy, g_rd.m_prologue.buf, g_rd.norms_dev, g_rd.xs_dev, g_rd.xq_dev, + dim * 4l, 256l, META_BYTES, g_rd.norms_bytes, (dim / 32l) * 4l, dim, + fixed_array(VHZ_X, 0u, 0u, 0u, VHZ_XQ, VHZ_XQ)) // quantize final xb g_rd.s_quant_final = rd_set6(g_rd.xb_dev, g_rd.dummy, g_rd.m_quant_final.buf, g_rd.xq_dev, g_rd.xs_dev, g_rd.dummy, dim * 4l, 256l, META_BYTES, dim, (dim / 32l) * 4l, 256l, @@ -5125,12 +5700,39 @@ def private fill_quant_meta(hb : HostBuf; nblocks : int64) { } } +// the f32->f16 convert's meta (arena cm2 chain): word 0 = element count +def private fill_cvt_meta(hb : HostBuf; elems : int64) { + unsafe { + var mp = reinterpret(hb.mapped) + mp[0] = uint(elems) + } +} + +// word 3 is the fused ar+requant's eps/ascale offset; the split ar kernel reads only 0-2 def private fill_addrms_meta(hb : HostBuf; dim, woff : int64; add : bool) { unsafe { var mp = reinterpret(hb.mapped) mp[0] = uint(dim) mp[1] = add ? 1u : 0u mp[2] = uint(woff) + mp[3] = uint(g_rd.ar_scal_off) + } +} + +// the fused qk-rmsnorm+rope meta; words 9-11 index the COS buffer, not the norms buffer +def private fill_qkn_rope_meta(hb : HostBuf; l : int64) { + unsafe { + let hs = g_rd.head_size + var mp = reinterpret(hb.mapped) + mp[0] = uint(g_rd.qd) + mp[1] = uint(g_rd.kv_dim) + mp[2] = uint(hs) + mp[3] = uint(hs / 2l) + mp[4] = g_rd.neox ? 1u : 0u + mp[5] = uint(g_rd.n_heads) + mp[9] = uint(g_rd.qkn_row_off + l * 2l * hs) + mp[10] = uint(g_rd.qkn_row_off + (l * 2l + 1l) * hs) + mp[11] = uint(g_rd.qkn_eps_off) } } @@ -5394,6 +5996,15 @@ def private vk_fa_on : bool { return g_vk_fa != 0 } +var private g_vk_fuse = -1 // DASLLAMA_VK_FUSE=0 pins the split decode-tail dispatches (same-build A/B) + +def private vk_fuse_on : bool { + if (g_vk_fuse < 0) { + g_vk_fuse = env_flag("DASLLAMA_VK_FUSE", true) ? 1 : 0 + } + return g_vk_fuse != 0 +} + // the per-node gate: barrier iff this node's declared masks conflict with the pending set def private vhz_dep(raw : VkCommandBuffer; var h : VkHaz; rmask, wmask : uint; transfer : bool = false) { h.disp++ @@ -5491,6 +6102,7 @@ def private rd_record_token { mn[3] = uint(qd); mn[4] = 0u; mn[5] = uint(qd) // one row: k lives at qd in kv_dev mn[6] = uint(nlfin + dim + l * 2l * hs) // rms_q[l] past the fixed norms layout mn[7] = uint(nlfin + dim + (l * 2l + 1l) * hs) // rms_k[l] + fill_qkn_rope_meta(L.m_qkr, l) } var mr = reinterpret(L.m_rope.mapped) mr[0] = uint(qd); mr[1] = uint(kvd); mr[2] = uint(g_rd.head_size); mr[3] = uint(g_rd.head_size / 2l) @@ -5523,10 +6135,19 @@ def private rd_record_token { cmd_copy_whole(raw, g_rd.cos_host.buf, g_rd.cos_dev, g_rd.head_size * 4l) vhz_dep(raw, h, 0u, VHZ_META, true) cmd_copy_whole(raw, g_rd.ffn_meta_h.buf, g_gpu.ffn_meta_dev, FFN_META_BYTES) - rd_dispatch(raw, g_gpu.ar_pipeline, g_rd.s_prologue, 1u, h) // xb = rms_att[0](x) + // fused rail: a layer's activation quant rides the PREVIOUS addr_next, and qkn rides rope + let fuse = vk_fuse_on() + let fuse_qkr = fuse && g_rd.qk_norm + if (fuse) { + rd_dispatch(raw, g_gpu.ar_rq_pipeline, g_rd.s_prologue_rq, 1u, h) // rms_att[0](x) -> xq/xs + } else { + rd_dispatch(raw, g_gpu.ar_pipeline, g_rd.s_prologue, 1u, h) // xb = rms_att[0](x) + } for (l in range64(g_rd.n_layers)) { var L & = g_rd.layers[l] - rd_dispatch(raw, g_gpu.dn_rq_pipeline, L.s_quant_xb, uint((dim / 4l + 255l) / 256l), h) + if (!fuse) { + rd_dispatch(raw, g_gpu.dn_rq_pipeline, L.s_quant_xb, uint((dim / 4l + 255l) / 256l), h) + } rd_dispatch_gemv(raw, L.fq, L.s_q, rd_gemv_wg(qd), h) if (L.kv_merged) { rd_dispatch_gemv(raw, L.fk, L.s_kv, rd_gemv_wg(2l * kvd), h) @@ -5534,22 +6155,36 @@ def private rd_record_token { rd_dispatch_gemv(raw, L.fk, L.s_k, rd_gemv_wg(kvd), h) rd_dispatch_gemv(raw, L.fv, L.s_v, rd_gemv_wg(kvd), h) } - if (g_rd.qk_norm) { - rd_dispatch(raw, g_gpu.qkn_pipeline, L.s_qkn, uint(g_rd.n_heads + kvd / g_rd.head_size), h) + if (fuse_qkr) { + rd_dispatch(raw, g_gpu.qkn_rope_pipeline, L.s_qkn_rope, uint(g_rd.n_heads + kvd / g_rd.head_size), h) + } else { + if (g_rd.qk_norm) { + rd_dispatch(raw, g_gpu.qkn_pipeline, L.s_qkn, uint(g_rd.n_heads + kvd / g_rd.head_size), h) + } + rd_dispatch(raw, g_gpu.rks_pipeline, L.s_rope, uint((qd / 2l + kvd / 2l + int64(WG_X) - 1l) / int64(WG_X)), h) } - rd_dispatch(raw, g_gpu.rks_pipeline, L.s_rope, uint((qd / 2l + kvd / 2l + int64(WG_X) - 1l) / int64(WG_X)), h) rd_dispatch(raw, g_gpu.da_pipeline, L.s_attn, uint(g_rd.n_heads), h) rd_dispatch(raw, g_gpu.dn_rq_pipeline, L.s_quant_at, uint((qd / 4l + 255l) / 256l), h) rd_dispatch_gemv(raw, L.fo, L.s_wo, rd_gemv_wg(dim), h) - rd_dispatch(raw, g_gpu.ar_pipeline, L.s_addr_ffn, 1u, h) - rd_dispatch(raw, g_gpu.dn_rq_pipeline, L.s_quant_xb2, uint((dim / 4l + 255l) / 256l), h) + if (fuse) { + rd_dispatch(raw, g_gpu.ar_rq_pipeline, L.s_addr_ffn_rq, 1u, h) + } else { + rd_dispatch(raw, g_gpu.ar_pipeline, L.s_addr_ffn, 1u, h) + rd_dispatch(raw, g_gpu.dn_rq_pipeline, L.s_quant_xb2, uint((dim / 4l + 255l) / 256l), h) + } rd_dispatch_gemv(raw, L.f1, L.s_gate, rd_gemv_wg(hid), h) rd_dispatch_gemv(raw, L.f3, L.s_up, rd_gemv_wg(hid), h) rd_dispatch(raw, g_gpu.actrq_pipeline, L.s_actrq, uint((hid / 4l + 255l) / 256l), h) rd_dispatch_gemv(raw, L.f2, L.s_down, rd_gemv_wg(dim), h) - rd_dispatch(raw, g_gpu.ar_pipeline, L.s_addr_next, 1u, h) + if (fuse) { + rd_dispatch(raw, g_gpu.ar_rq_pipeline, L.s_addr_next_rq, 1u, h) + } else { + rd_dispatch(raw, g_gpu.ar_pipeline, L.s_addr_next, 1u, h) + } + } + if (!fuse) { + rd_dispatch(raw, g_gpu.dn_rq_pipeline, g_rd.s_quant_final, uint((dim / 4l + 255l) / 256l), h) } - rd_dispatch(raw, g_gpu.dn_rq_pipeline, g_rd.s_quant_final, uint((dim / 4l + 255l) / 256l), h) rd_dispatch_gemv(raw, g_rd.cls_fmt_marker, g_rd.cls_set, rd_gemv_wg(g_rd.vocab), h) vhz_dep(raw, h, VHZ_LOG, 0u, true) // compute -> transfer for the logits DMA cmd_copy_whole(raw, g_rd.logits_dev, g_rd.logits_host.buf, g_rd.vocab * 4l) @@ -5570,8 +6205,11 @@ var private g_rdq_role : double[16] def private rdq_sample { // stamp count follows each layer's own kv_merged (K-quant ggufs mix formats per layer) - var want = 4 // anchor + prologue + fin_rq + cls - let base_rpl = g_rd.qk_norm ? 16 : 15 + let fuse = vk_fuse_on() + let fuse_qkr = fuse && g_rd.qk_norm + var want = fuse ? 3 : 4 // anchor + prologue + [fin_rq — split rail only] + cls + // fused is 13 either way: merging qkn into rope exactly offsets keeping a separate rope + let base_rpl = fuse ? 13 : (g_rd.qk_norm ? 16 : 15) for (L in g_rd.layers) { want += base_rpl - (L.kv_merged ? 1 : 0) } @@ -5579,27 +6217,24 @@ def private rdq_sample { return // deep graphs overflow PFQ_CAP — no aggregation } g_rdq_pro += pfq_us(1u) - // k+v aggregate into ONE "kv" slot (merged = 1 stamp, split = 2) — uniform incl. mixed models + // k+v aggregate into ONE "kv" slot (split = 2 stamps); fused drops rq_x, so the kv slot shifts down + let nroles = base_rpl - 1 + let kv_at = fuse ? 1 : 2 var qi = 2u for (L in g_rd.layers) { - g_rdq_role[0] += pfq_us(qi) - g_rdq_role[1] += pfq_us(qi + 1u) - g_rdq_role[2] += pfq_us(qi + 2u) - qi += 3u - if (!L.kv_merged) { - g_rdq_role[2] += pfq_us(qi) - qi++ - } - var s = 3 - var rest = base_rpl - 4 // qkn? rope attn rq_a wo ar1 rq_f gate up actrq down ar2 - while (rest > 0) { + var s = 0 + while (s < nroles) { g_rdq_role[s] += pfq_us(qi) - s++ qi++ - rest-- + if (s == kv_at && !L.kv_merged) { + g_rdq_role[s] += pfq_us(qi) // the split v stamp folds into the kv slot + qi++ + } + s++ } } - g_rdq_tail += pfq_us(g_rdq_n - 2u) + pfq_us(g_rdq_n - 1u) + // split rail: fin_rq + cls; fused: cls alone (the last addr_next already did the final norm) + g_rdq_tail += (fuse ? 0.0lf : pfq_us(g_rdq_n - 2u)) + pfq_us(g_rdq_n - 1u) unsafe { var tp = reinterpret(g_pfq_buf.mapped) g_rdq_total += double(tp[g_rdq_n - 1u] - tp[0u]) * double(g_pfq_period) / 1000.0lf @@ -5608,7 +6243,9 @@ def private rdq_sample { if (g_rdq_count % 32l == 0l) { let n = double(g_rdq_count) var names : array - if (g_rd.qk_norm) { + if (fuse) { + names <- ["q", "kv", fuse_qkr ? "qknrope" : "rope", "attn", "rq_a", "wo", "arrq_f", "gate", "up", "actrq", "down", "arrq_n"] + } elif (g_rd.qk_norm) { names <- ["rq_x", "q", "kv", "qkn", "rope", "attn", "rq_a", "wo", "ar1", "rq_f", "gate", "up", "actrq", "down", "ar2"] } else { names <- ["rq_x", "q", "kv", "rope", "attn", "rq_a", "wo", "ar1", "rq_f", "gate", "up", "actrq", "down", "ar2"] @@ -5633,13 +6270,19 @@ def vk_rdec_token(x : array; cossin : array; pos, cnt : int64; var } let kvd = g_rd.kv_dim unsafe { - memcpy(g_rd.x_host.mapped, addr(x[0]), g_rd.dim * 4l) - memcpy(g_rd.cos_host.mapped, addr(cossin[0]), g_rd.head_size * 4l) + memcpy(g_rd.x_host.mapped, addr(x[0]), g_rd.dim * 4l) + memcpy(g_rd.cos_host.mapped, addr(cossin[0]), g_rd.head_size * 4l) for (l in range64(g_rd.n_layers)) { var L & = g_rd.layers[l] + let row = uint(l * g_rd.seq_cap * kvd + pos * kvd) var mr = reinterpret(L.m_rope.mapped) - mr[7] = uint(l * g_rd.seq_cap * kvd + pos * kvd) - mr[8] = uint(l * g_rd.seq_cap * kvd + pos * kvd) + mr[7] = row + mr[8] = row + if (g_rd.qk_norm) { // the fused twin keeps the mirror-row words at the same indices + var mq = reinterpret(L.m_qkr.mapped) + mq[7] = row + mq[8] = row + } var ma = reinterpret(L.m_attn.mapped) ma[0] = uint(cnt) } @@ -5690,6 +6333,16 @@ def private pf_setup { g_rd.pf_logits_host = make_host_buf(g_rd.vocab * 4l, false, [cached = true]) g_rd.pf_cmd = alloc_cmd() ensure_batch_state() // hq_dev/hs_dev for the FFN act + // fmt-6 (cm2) groups feed the twin f16 rows instead of q8 images + var any6 = false + for (L in g_rd.layers) { + any6 = any6 || L.fq == 6 || L.fo == 6 || L.f1 == 6 || L.f2 == 6 + } + if (any6) { + g_rd.pf_xf = make_device_buf(np * dim * 2l) + g_rd.pf_af = make_device_buf(np * qd * 2l) + g_rd.pf_hf = make_device_buf(np * hid * 2l) + } let nsets = int(g_rd.n_layers * PF_ROLES + 2l) // + prologue, final quantize of the last row g_rd.pf_sets |> resize(nsets) g_rd.pf_meta |> resize(nsets) @@ -5703,19 +6356,40 @@ def private pf_setup { let b = int(l * PF_ROLES) let pq = arena_planes(L.fq); let pk = arena_planes(L.fk); let pv = arena_planes(L.fv) let po = arena_planes(L.fo); let p1 = arena_planes(L.f1); let p3 = arena_planes(L.f3); let p2 = arena_planes(L.f2) - // 0 quant_xb: pf_xb -> pf_xq/pf_xs - pf_bind(b + 0, g_rd.pf_xb, g_rd.dummy, g_rd.pf_meta[b + 0].buf, g_rd.pf_xq, g_rd.pf_xs, g_rd.dummy, - np * dim * 4l, 256l, BATCH_META_BYTES, np * dim, np * (dim / 32l) * 4l, 256l, + // fmt-6 groups run on the f16 twin plane at binding 3 (idle binding-4 bits are lens-ignored) + let qkv6 = L.fq == 6 + let xqb = qkv6 ? g_rd.pf_xf : g_rd.pf_xq + let xsb = qkv6 ? g_rd.dummy : g_rd.pf_xs + let xqn = qkv6 ? np * dim * 2l : np * dim + let xsn = qkv6 ? 256l : np * (dim / 32l) * 4l + let wo6 = L.fo == 6 + let aqb = wo6 ? g_rd.pf_af : g_rd.pf_aq + let asb = wo6 ? g_rd.dummy : g_rd.pf_as + let aqn = wo6 ? np * qd * 2l : np * qd + let asn = wo6 ? 256l : np * (qd / 32l) * 4l + let gu6 = L.f1 == 6 + let gqb = gu6 ? g_rd.pf_xf : g_rd.pf_xq + let gsb = gu6 ? g_rd.dummy : g_rd.pf_xs + let gqn = gu6 ? np * dim * 2l : np * dim + let gsn = gu6 ? 256l : np * (dim / 32l) * 4l + let dn6 = L.f2 == 6 + let hqb = dn6 ? g_rd.pf_hf : g_gpu.hq_dev + let hsb = dn6 ? g_rd.dummy : g_gpu.hs_dev + let hqn = dn6 ? np * hid * 2l : np * hid + let hsn = dn6 ? 256l : np * (hid / 32l) * 4l + // 0 quant_xb: pf_xb -> pf_xq/pf_xs (fmt 6: f16 convert -> pf_xf) + pf_bind(b + 0, g_rd.pf_xb, g_rd.dummy, g_rd.pf_meta[b + 0].buf, xqb, xsb, g_rd.dummy, + np * dim * 4l, 256l, BATCH_META_BYTES, xqn, xsn, 256l, fixed_array(VHZ_XB, 0u, 0u, VHZ_XQ, VHZ_XQ, 0u)) // 1/2/3 q/k/v batch GEMM: arena planes, pf_xq/pf_xs act, -> pf_q / pf_kv(k@0) / pf_kv(v@np*kvd) - pf_bind(b + 1, pq.wq, pq.ws, g_rd.pf_meta[b + 1].buf, g_rd.pf_xq, g_rd.pf_xs, g_rd.pf_q, - pq.wqb, pq.wsb, BATCH_META_BYTES, np * dim, np * (dim / 32l) * 4l, np * qd * 4l, + pf_bind(b + 1, pq.wq, pq.ws, g_rd.pf_meta[b + 1].buf, xqb, xsb, g_rd.pf_q, + pq.wqb, pq.wsb, BATCH_META_BYTES, xqn, xsn, np * qd * 4l, fixed_array(0u, 0u, 0u, VHZ_XQ, VHZ_XQ, VHZ_Q)) - pf_bind(b + 2, pk.wq, pk.ws, g_rd.pf_meta[b + 2].buf, g_rd.pf_xq, g_rd.pf_xs, g_rd.pf_kv, - pk.wqb, pk.wsb, BATCH_META_BYTES, np * dim, np * (dim / 32l) * 4l, np * 2l * kvd * 4l, + pf_bind(b + 2, pk.wq, pk.ws, g_rd.pf_meta[b + 2].buf, xqb, xsb, g_rd.pf_kv, + pk.wqb, pk.wsb, BATCH_META_BYTES, xqn, xsn, np * 2l * kvd * 4l, fixed_array(0u, 0u, 0u, VHZ_XQ, VHZ_XQ, VHZ_KVK)) - pf_bind(b + 3, pv.wq, pv.ws, g_rd.pf_meta[b + 3].buf, g_rd.pf_xq, g_rd.pf_xs, g_rd.pf_v, - pv.wqb, pv.wsb, BATCH_META_BYTES, np * dim, np * (dim / 32l) * 4l, np * kvd * 4l, + pf_bind(b + 3, pv.wq, pv.ws, g_rd.pf_meta[b + 3].buf, xqb, xsb, g_rd.pf_v, + pv.wqb, pv.wsb, BATCH_META_BYTES, xqn, xsn, np * kvd * 4l, fixed_array(0u, 0u, 0u, VHZ_XQ, VHZ_XQ, VHZ_VST)) // 4 rope+store: q@0 in place, Kmir@1, Vmir@3, cos@4, kv@5 pf_bind(b + 4, g_rd.pf_q, g_rd.k_mirror, g_rd.pf_meta[b + 4].buf, g_rd.v_mirror, g_rd.pf_cos, g_rd.pf_kv, @@ -5725,36 +6399,36 @@ def private pf_setup { pf_bind(b + 5, g_rd.pf_q, g_rd.k_mirror, g_rd.pf_meta[b + 5].buf, g_rd.v_mirror, g_rd.scal_dev, g_rd.pf_attn, np * qd * 4l, mirbytes, BATCH_META_BYTES, mirbytes, 64l, np * qd * 4l, fixed_array(VHZ_Q, VHZ_MIR, 0u, VHZ_MIR, 0u, VHZ_ATT)) - // 6 quant_at: pf_attn -> pf_aq/pf_as - pf_bind(b + 6, g_rd.pf_attn, g_rd.dummy, g_rd.pf_meta[b + 6].buf, g_rd.pf_aq, g_rd.pf_as, g_rd.dummy, - np * qd * 4l, 256l, BATCH_META_BYTES, np * qd, np * (qd / 32l) * 4l, 256l, + // 6 quant_at: pf_attn -> pf_aq/pf_as (fmt 6: f16 convert -> pf_af) + pf_bind(b + 6, g_rd.pf_attn, g_rd.dummy, g_rd.pf_meta[b + 6].buf, aqb, asb, g_rd.dummy, + np * qd * 4l, 256l, BATCH_META_BYTES, aqn, asn, 256l, fixed_array(VHZ_ATT, 0u, 0u, VHZ_AQ, VHZ_AQ, 0u)) // 7 wo GEMM -> pf_xb2 - pf_bind(b + 7, po.wq, po.ws, g_rd.pf_meta[b + 7].buf, g_rd.pf_aq, g_rd.pf_as, g_rd.pf_xb2, - po.wqb, po.wsb, BATCH_META_BYTES, np * qd, np * (qd / 32l) * 4l, np * dim * 4l, + pf_bind(b + 7, po.wq, po.ws, g_rd.pf_meta[b + 7].buf, aqb, asb, g_rd.pf_xb2, + po.wqb, po.wsb, BATCH_META_BYTES, aqn, asn, np * dim * 4l, fixed_array(0u, 0u, 0u, VHZ_AQ, VHZ_AQ, VHZ_XB2)) // 8 addr_ffn: x += xb2, xb = rms_ffn pf_bind(b + 8, g_rd.pf_x, g_rd.pf_xb2, g_rd.pf_meta[b + 8].buf, g_rd.norms_dev, g_rd.ar_scal_dev, g_rd.pf_xb, np * dim * 4l, np * dim * 4l, BATCH_META_BYTES, normbytes, 64l, np * dim * 4l, fixed_array(VHZ_X, VHZ_XB2, 0u, 0u, 0u, VHZ_XB)) - // 9 quant_xb2: pf_xb -> pf_xq - pf_bind(b + 9, g_rd.pf_xb, g_rd.dummy, g_rd.pf_meta[b + 9].buf, g_rd.pf_xq, g_rd.pf_xs, g_rd.dummy, - np * dim * 4l, 256l, BATCH_META_BYTES, np * dim, np * (dim / 32l) * 4l, 256l, + // 9 quant_xb2: pf_xb -> pf_xq (fmt 6: f16 convert -> pf_xf) + pf_bind(b + 9, g_rd.pf_xb, g_rd.dummy, g_rd.pf_meta[b + 9].buf, gqb, gsb, g_rd.dummy, + np * dim * 4l, 256l, BATCH_META_BYTES, gqn, gsn, 256l, fixed_array(VHZ_XB, 0u, 0u, VHZ_XQ, VHZ_XQ, 0u)) // 10/11 gate/up GEMM - pf_bind(b + 10, p1.wq, p1.ws, g_rd.pf_meta[b + 10].buf, g_rd.pf_xq, g_rd.pf_xs, g_rd.pf_gate, - p1.wqb, p1.wsb, BATCH_META_BYTES, np * dim, np * (dim / 32l) * 4l, np * hid * 4l, + pf_bind(b + 10, p1.wq, p1.ws, g_rd.pf_meta[b + 10].buf, gqb, gsb, g_rd.pf_gate, + p1.wqb, p1.wsb, BATCH_META_BYTES, gqn, gsn, np * hid * 4l, fixed_array(0u, 0u, 0u, VHZ_XQ, VHZ_XQ, VHZ_GATE)) - pf_bind(b + 11, p3.wq, p3.ws, g_rd.pf_meta[b + 11].buf, g_rd.pf_xq, g_rd.pf_xs, g_rd.pf_up, - p3.wqb, p3.wsb, BATCH_META_BYTES, np * dim, np * (dim / 32l) * 4l, np * hid * 4l, + pf_bind(b + 11, p3.wq, p3.ws, g_rd.pf_meta[b + 11].buf, gqb, gsb, g_rd.pf_up, + p3.wqb, p3.wsb, BATCH_META_BYTES, gqn, gsn, np * hid * 4l, fixed_array(0u, 0u, 0u, VHZ_XQ, VHZ_XQ, VHZ_UP)) - // 12 actrq: up@0, ffn_meta@2, hq@3, hs@4, gate@5 - pf_bind(b + 12, g_rd.pf_up, g_rd.dummy, g_gpu.ffn_meta_dev, g_gpu.hq_dev, g_gpu.hs_dev, g_rd.pf_gate, - np * hid * 4l, 256l, FFN_META_BYTES, BATCH_HQ_BYTES, BATCH_HS_BYTES, np * hid * 4l, + // 12 actrq: up@0, ffn_meta@2, hq@3, hs@4, gate@5 (fmt 6: actf16 -> pf_hf@3) + pf_bind(b + 12, g_rd.pf_up, g_rd.dummy, g_gpu.ffn_meta_dev, hqb, hsb, g_rd.pf_gate, + np * hid * 4l, 256l, FFN_META_BYTES, dn6 ? hqn : BATCH_HQ_BYTES, dn6 ? hsn : BATCH_HS_BYTES, np * hid * 4l, fixed_array(VHZ_UP, 0u, VHZ_META, VHZ_HQ, VHZ_HQ, VHZ_GATE)) // 13 down GEMM: hq/hs act -> pf_ffnout - pf_bind(b + 13, p2.wq, p2.ws, g_rd.pf_meta[b + 13].buf, g_gpu.hq_dev, g_gpu.hs_dev, g_rd.pf_ffnout, - p2.wqb, p2.wsb, BATCH_META_BYTES, np * hid, np * (hid / 32l) * 4l, np * dim * 4l, + pf_bind(b + 13, p2.wq, p2.ws, g_rd.pf_meta[b + 13].buf, hqb, hsb, g_rd.pf_ffnout, + p2.wqb, p2.wsb, BATCH_META_BYTES, hqn, hsn, np * dim * 4l, fixed_array(0u, 0u, 0u, VHZ_HQ, VHZ_HQ, VHZ_FFO)) // 14 addr_next: x += ffnout, xb = rms_att[l+1]|final pf_bind(b + 14, g_rd.pf_x, g_rd.pf_ffnout, g_rd.pf_meta[b + 14].buf, g_rd.norms_dev, g_rd.ar_scal_dev, g_rd.pf_xb, @@ -5868,17 +6542,22 @@ def vk_rdec_prefill(x_batch : array; cos_batch : array; npos : int let last = w0 + wlen >= npos upload_region_at(g_rd.pf_x, 0l, addr(x_batch[int(w0 * dim)]), wlen * dim * 4l) upload_region_at(g_rd.pf_cos, 0l, addr(cos_batch[int(w0 * g_rd.head_size)]), wlen * g_rd.head_size * 4l) - fill_ffn_meta(hid, wlen * hid / 32l, false) // actrq over the window's rows + fill_ffn_meta(wlen * hid, wlen * hid / 32l, false) // act(rq|f16) over the window's rows // fill metas let mA = @(idx : int; n, d, blk : int64; fmt : int) { fill_arena_batch_meta(g_rd.pf_meta[idx], n, d, blk, wlen, fmt) } let mQ = @(idx : int; blocks : int64) { fill_quant_meta(g_rd.pf_meta[idx], blocks) } + let mC = @(idx : int; elems : int64) { fill_cvt_meta(g_rd.pf_meta[idx], elems) } let mR = @(idx : int; woff : int64; add : bool) { fill_addrms_meta(g_rd.pf_meta[idx], dim, woff, add) } for (l in range64(g_rd.n_layers)) { var L & = g_rd.layers[l] let b = int(l * PF_ROLES) - invoke(mQ, b + 0, wlen * dim / 32l) + if (L.fq == 6) { + invoke(mC, b + 0, wlen * dim) + } else { + invoke(mQ, b + 0, wlen * dim / 32l) + } invoke(mA, b + 1, dim, qd, L.bq, L.fq) invoke(mA, b + 2, dim, kvd, L.bk, L.fk) invoke(mA, b + 3, dim, kvd, L.bv, L.fv) @@ -5897,10 +6576,18 @@ def vk_rdec_prefill(x_batch : array; cos_batch : array; npos : int var ma = reinterpret(g_rd.pf_meta[b + 5].mapped) ma[0] = uint(wlen); ma[1] = uint(g_rd.n_heads); ma[2] = uint(kvd); ma[3] = uint(g_rd.head_size); ma[4] = uint(g_rd.kv_mul) ma[5] = uint(l * g_rd.seq_cap * kvd); ma[6] = uint(l * g_rd.seq_cap * kvd); ma[7] = uint(w0) - invoke(mQ, b + 6, wlen * qd / 32l) + if (L.fo == 6) { + invoke(mC, b + 6, wlen * qd) + } else { + invoke(mQ, b + 6, wlen * qd / 32l) + } invoke(mA, b + 7, qd, dim, L.bo, L.fo) invoke(mR, b + 8, (l * 2l + 1l) * dim, true) - invoke(mQ, b + 9, wlen * dim / 32l) + if (L.f1 == 6) { + invoke(mC, b + 9, wlen * dim) + } else { + invoke(mQ, b + 9, wlen * dim / 32l) + } invoke(mA, b + 10, dim, hid, L.b1, L.f1) invoke(mA, b + 11, dim, hid, L.b3, L.f3) invoke(mA, b + 13, hid, dim, L.b2, L.f2) @@ -5933,10 +6620,16 @@ def vk_rdec_prefill(x_batch : array; cos_batch : array; npos : int rd_dispatch(raw, g_gpu.ar_pipeline_b, g_rd.pf_sets[bp + 0], uint(wlen), h) // prologue norm for (l in range64(g_rd.n_layers)) { let b = int(l * PF_ROLES) - rd_dispatch(raw, g_gpu.dn_rq_pipeline, g_rd.pf_sets[b + 0], uint((wlen * dim / 4l + 255l) / 256l), h) - rd_dispatch(raw, q8_batch_pipe_for(qd, wlen), g_rd.pf_sets[b + 1], fill_arena_batch_meta(g_rd.pf_meta[b + 1], dim, qd, g_rd.layers[l].bq, wlen, 0), h) - rd_dispatch(raw, q8_batch_pipe_for(kvd, wlen), g_rd.pf_sets[b + 2], fill_arena_batch_meta(g_rd.pf_meta[b + 2], dim, kvd, g_rd.layers[l].bk, wlen, 0), h) - rd_dispatch(raw, q8_batch_pipe_for(kvd, wlen), g_rd.pf_sets[b + 3], fill_arena_batch_meta(g_rd.pf_meta[b + 3], dim, kvd, g_rd.layers[l].bv, wlen, 0), h) + // fmt-6 (cm2) roles: f32->f16 convert feeds the decode-in-load twin; the fill's fmt + // must match the dispatched pipe's tile rule, so both key off the same group flag + let fq6 = g_rd.layers[l].fq == 6 + let fo6 = g_rd.layers[l].fo == 6 + let gu6 = g_rd.layers[l].f1 == 6 + let dn6 = g_rd.layers[l].f2 == 6 + rd_dispatch(raw, weak_copy(fq6 ? g_gpu.f16cvt_pipeline : g_gpu.dn_rq_pipeline), g_rd.pf_sets[b + 0], uint((wlen * dim / 4l + 255l) / 256l), h) + rd_dispatch(raw, pf_gemm_pipe(fq6, qd, wlen), g_rd.pf_sets[b + 1], fill_arena_batch_meta(g_rd.pf_meta[b + 1], dim, qd, g_rd.layers[l].bq, wlen, fq6 ? 6 : 0), h) + rd_dispatch(raw, pf_gemm_pipe(fq6, kvd, wlen), g_rd.pf_sets[b + 2], fill_arena_batch_meta(g_rd.pf_meta[b + 2], dim, kvd, g_rd.layers[l].bk, wlen, fq6 ? 6 : 0), h) + rd_dispatch(raw, pf_gemm_pipe(fq6, kvd, wlen), g_rd.pf_sets[b + 3], fill_arena_batch_meta(g_rd.pf_meta[b + 3], dim, kvd, g_rd.layers[l].bv, wlen, fq6 ? 6 : 0), h) vhz_dep(raw, h, VHZ_VST, VHZ_KVV, true) cmd_copy_range(raw, g_rd.pf_v, 0l, g_rd.pf_kv, wlen * kvd * 4l, wlen * kvd * 4l) // v into pf_kv's v half if (g_rd.qk_norm) { @@ -5945,14 +6638,14 @@ def vk_rdec_prefill(x_batch : array; cos_batch : array; npos : int } rd_dispatch(raw, g_gpu.rks_b_pipeline, g_rd.pf_sets[b + 4], ropewg, h) rd_dispatch(raw, weak_copy(fa128 ? g_gpu.da_b128_pipeline : g_gpu.da_b_pipeline), g_rd.pf_sets[b + 5], attnwg, h) - rd_dispatch(raw, g_gpu.dn_rq_pipeline, g_rd.pf_sets[b + 6], uint((wlen * qd / 4l + 255l) / 256l), h) - rd_dispatch(raw, q8_batch_pipe_for(dim, wlen), g_rd.pf_sets[b + 7], fill_arena_batch_meta(g_rd.pf_meta[b + 7], qd, dim, g_rd.layers[l].bo, wlen, 0), h) + rd_dispatch(raw, weak_copy(fo6 ? g_gpu.f16cvt_pipeline : g_gpu.dn_rq_pipeline), g_rd.pf_sets[b + 6], uint((wlen * qd / 4l + 255l) / 256l), h) + rd_dispatch(raw, pf_gemm_pipe(fo6, dim, wlen), g_rd.pf_sets[b + 7], fill_arena_batch_meta(g_rd.pf_meta[b + 7], qd, dim, g_rd.layers[l].bo, wlen, fo6 ? 6 : 0), h) rd_dispatch(raw, g_gpu.ar_pipeline_b, g_rd.pf_sets[b + 8], uint(wlen), h) - rd_dispatch(raw, g_gpu.dn_rq_pipeline, g_rd.pf_sets[b + 9], uint((wlen * dim / 4l + 255l) / 256l), h) - rd_dispatch(raw, q8_batch_pipe_for(hid, wlen), g_rd.pf_sets[b + 10], fill_arena_batch_meta(g_rd.pf_meta[b + 10], dim, hid, g_rd.layers[l].b1, wlen, 0), h) - rd_dispatch(raw, q8_batch_pipe_for(hid, wlen), g_rd.pf_sets[b + 11], fill_arena_batch_meta(g_rd.pf_meta[b + 11], dim, hid, g_rd.layers[l].b3, wlen, 0), h) - rd_dispatch(raw, g_gpu.actrq_pipeline, g_rd.pf_sets[b + 12], uint((wlen * hid / 4l + 255l) / 256l), h) - rd_dispatch(raw, q8_batch_pipe_for(dim, wlen), g_rd.pf_sets[b + 13], fill_arena_batch_meta(g_rd.pf_meta[b + 13], hid, dim, g_rd.layers[l].b2, wlen, 0), h) + rd_dispatch(raw, weak_copy(gu6 ? g_gpu.f16cvt_pipeline : g_gpu.dn_rq_pipeline), g_rd.pf_sets[b + 9], uint((wlen * dim / 4l + 255l) / 256l), h) + rd_dispatch(raw, pf_gemm_pipe(gu6, hid, wlen), g_rd.pf_sets[b + 10], fill_arena_batch_meta(g_rd.pf_meta[b + 10], dim, hid, g_rd.layers[l].b1, wlen, gu6 ? 6 : 0), h) + rd_dispatch(raw, pf_gemm_pipe(gu6, hid, wlen), g_rd.pf_sets[b + 11], fill_arena_batch_meta(g_rd.pf_meta[b + 11], dim, hid, g_rd.layers[l].b3, wlen, gu6 ? 6 : 0), h) + rd_dispatch(raw, weak_copy(dn6 ? g_gpu.actf16_pipeline : g_gpu.actrq_pipeline), g_rd.pf_sets[b + 12], uint((wlen * hid / 4l + 255l) / 256l), h) + rd_dispatch(raw, pf_gemm_pipe(dn6, dim, wlen), g_rd.pf_sets[b + 13], fill_arena_batch_meta(g_rd.pf_meta[b + 13], hid, dim, g_rd.layers[l].b2, wlen, dn6 ? 6 : 0), h) rd_dispatch(raw, g_gpu.ar_pipeline_b, g_rd.pf_sets[b + 14], uint(wlen), h) } if (last) { @@ -6007,18 +6700,18 @@ def vk_rdec_prefill(x_batch : array; cos_batch : array; npos : int def vk_arena_ffn(var y : array; blk1, blk3, blk2 : int64; n, nfe : int64; f1, f3, f2 : int; xq : array; xs : array; is_gelu : bool) { verify(vk_moe_init()) - assert((f3 != 0) == (f1 != 0), "dasLLAMA vulkan arena: gate/up form mismatch — one activation image") + assert(vk_fmt_b32(f3) == vk_fmt_b32(f1), "dasLLAMA vulkan arena: gate/up form mismatch — one activation image") assert(nfe * 4l <= Y_BYTES && n * 4l <= Y_BYTES, "dasLLAMA vulkan arena: ffn planes exceed y capacity") ensure_af_state(f1, f3, f2) - let xunit = f1 != 0 ? 256l : 32l - let dkq = f2 != 0 + let xunit = fmt_unit(f1) + let dkq = !vk_fmt_b32(f2) fill_af_meta(g_gpu.af_meta_g, n, nfe, blk1) fill_af_meta(g_gpu.af_meta_u, n, nfe, blk3) fill_af_meta(g_gpu.af_meta_d, nfe, n, blk2) fill_ffn_meta(nfe, nfe / (dkq ? 256l : 32l), is_gelu) unsafe { - memcpy(g_gpu.af_xq.mapped, addr(xq[0]), n) - memcpy(g_gpu.af_xs.mapped, addr(xs[0]), (n / xunit) * 4l) + memcpy(g_gpu.af_xq.mapped, addr(xq[0]), n) + memcpy(g_gpu.af_xs.mapped, addr(xs[0]), (n / xunit) * 4l) var raw = alloc_cmd() let begin = VkCommandBufferBeginInfo() vk_check(vkBeginCommandBuffer(raw, begin), null) @@ -6044,9 +6737,13 @@ def vk_arena_ffn(var y : array; blk1, blk3, blk2 : int64; n, nfe : int64; // dispatch workgroup count. Mirrors fill_batch_meta for one region [0, npos). def private fill_arena_batch_meta(hb : HostBuf; n, d, blk, npos : int64; fmt : int) : uint { let tile = fmt == 0 ? q8_batch_tile_for(d, npos) : BATCH_TILE - let wtiles = (d + tile - 1l) / tile - let rtiles = (npos + tile - 1l) / tile + // the cm2 twin tiles 64 weight-rows x 128 tokens per workgroup (fill_batch_meta's rule) + let tile_d = fmt == 6 ? 64l : tile + let tile_c = fmt == 6 ? 128l : tile + let wtiles = (d + tile_d - 1l) / tile_d + let rtiles = (npos + tile_c - 1l) / tile_c let total = rtiles * wtiles + assert((8l + total) * 4l <= BATCH_META_BYTES, "dasLLAMA vulkan arena: batch meta exceeds capacity") unsafe { var mp = reinterpret(hb.mapped) mp[0] = uint(n) @@ -6119,7 +6816,7 @@ def vk_arena_gemv(var y : array; blk : int64; n, d : int64; fmt : int; xq let xqp = unsafe(addr(xq[0])) let xsp = unsafe(addr(xs[0])) arena_ensure_scratch(fmt) - let unit = fmt != 0 ? 256l : 32l + let unit = fmt_unit(fmt) // q8 AND q8n take 32-block Q8_0 activations let nb = n / unit assert(n <= XQ_BYTES && nb * 4l <= XS_BYTES, "dasLLAMA vulkan arena: activations exceed scratch capacity") assert(d * 4l <= Y_BYTES, "dasLLAMA vulkan arena: gemv output exceeds y capacity") @@ -6132,8 +6829,8 @@ def vk_arena_gemv(var y : array; blk : int64; n, d : int64; fmt : int; xq mp[3] = 0u // y base row mp[4] = uint(blk) // THE arena base — a stack-relative offset in the per-op tier mp[5] = 0u // activation block base - memcpy(a.xq.mapped, reinterpret(xqp), n) // one int8 quant per weight - memcpy(a.xs.mapped, reinterpret(xsp), nb * 4l) + memcpy(a.xq.mapped, reinterpret(xqp), n) // one int8 quant per weight + memcpy(a.xs.mapped, reinterpret(xsp), nb * 4l) var raw = alloc_cmd() let begin = VkCommandBufferBeginInfo() vk_check(vkBeginCommandBuffer(raw, begin), null) @@ -6193,6 +6890,8 @@ def private gemv_pipe(fmt : int) : Pipeline { return weak_copy(g_gpu.gemv_k6) } elif (fmt == 4) { return weak_copy(g_gpu.gemv_q40) + } elif (fmt == 6) { + return weak_copy(g_gpu.gemv_q8n) } panic("dasLLAMA vulkan tier: no GEMV kernel for stack format {fmt}") return weak_copy(g_gpu.pipeline) @@ -6205,7 +6904,7 @@ def private q8_batch_pipe(tile : int64; aligned : bool = false) : Pipeline { return weak_copy(g_gpu.batch_coopmat_f16) } elif (g_gpu.coopmat_mode == 2) { return weak_copy(g_gpu.batch_coopmat_i8) - } elif (g_gpu.coopmat_mode == 3) { + } elif (g_gpu.coopmat_mode >= 3) { if (tile == BATCH_TILE) { // the small tier IS the sdot4 kernel — measured, not a fallback return weak_copy(g_gpu.batch_pipeline) } @@ -6233,6 +6932,8 @@ def private batch_pipe(fmt : int; tile : int64; aligned : bool = false) : Pipeli } elif (fmt == 4) { // Q4_0 rides the f16 coopmat arm too (no s8-native for nibbles — mode 2 falls back to sdot4) return weak_copy(g_gpu.coopmat_mode == 1 ? g_gpu.batch_coopmat_q40 : g_gpu.batch_q40) + } elif (fmt == 6) { + return weak_copy(g_gpu.batch_cm2) } panic("dasLLAMA vulkan tier: no batch tile for stack format {fmt}") return weak_copy(g_gpu.batch_pipeline) @@ -6243,8 +6944,24 @@ def private submit_nowait(var raw : VkCommandBuffer) { var submit = VkSubmitInfo() submit.commandBufferCount = 1u let fence = g_gpu.fence + // a pending transfer-queue handoff attaches here — later in-order submits inherit the ordering + var tss = VkTimelineSemaphoreSubmitInfo() + var wval = g_gpu.pend_wait + var sem = g_gpu.xfer_sem + var wstage : VkPipelineStageFlags + wstage.transfer = true + wstage.compute_shader = true unsafe { submit.pCommandBuffers = addr(raw) + if (wval > 0ul) { + tss.waitSemaphoreValueCount = 1u + tss.pWaitSemaphoreValues = addr(wval) + submit.pNext = addr(tss) + submit.waitSemaphoreCount = 1u + submit.pWaitSemaphores = addr(sem) + submit.pWaitDstStageMask = addr(wstage) + g_gpu.pend_wait = 0ul + } vk_check(vkQueueSubmit(g_gpu.queue, 1u, addr(submit), fence), null) } } @@ -6267,7 +6984,7 @@ def private submit_wait(var raw : VkCommandBuffer) { // row-major): [n, d, nrows, ybase, (rel_wblock, xblock)*], block units per format (32 q8 / 256 kq). def private fill_stack_meta_rows(si : int; offs : int64 const?; nregions : int64; n, d, nrows, xnb : int64; ybase : int64 = 0l) { assert((4l + nrows * 2l) * 4l <= META_BYTES, "dasLLAMA vulkan tier: FFN meta exceeds capacity") - let unit = g_gpu.stacks[si].fmt != 0 ? 256l : 32l + let unit = fmt_unit(g_gpu.stacks[si].fmt) unsafe { var mp = reinterpret(g_gpu.stacks[si].meta.mapped) mp[0] = uint(n) @@ -6289,10 +7006,10 @@ def private fill_stack_meta_rows(si : int; offs : int64 const?; nregions : int64 def private fill_stack_acts(si : int; xqp : int8 const?; xsp : float const?; xq_bytes : int64) { assert(xq_bytes <= XQ_BYTES, "dasLLAMA vulkan tier: dispatch exceeds scratch capacity") - let unit = g_gpu.stacks[si].fmt != 0 ? 256l : 32l + let unit = fmt_unit(g_gpu.stacks[si].fmt) unsafe { - memcpy(g_gpu.stacks[si].xq.mapped, reinterpret(xqp), xq_bytes) - memcpy(g_gpu.stacks[si].xs.mapped, reinterpret(xsp), (xq_bytes / unit) * 4l) + memcpy(g_gpu.stacks[si].xq.mapped, reinterpret(xqp), xq_bytes) + memcpy(g_gpu.stacks[si].xs.mapped, reinterpret(xsp), (xq_bytes / unit) * 4l) } } @@ -6309,13 +7026,13 @@ def private ensure_ffn_state { g_gpu.ffn_ready = true } -def private fill_ffn_meta(elems, nblocks : int64; is_gelu : bool; comb_n : int64 = 0l; comb_k : int64 = 0l; comb_r0 : int64 = 0l; comb_r1 : int64 = 0l) { +def private fill_ffn_meta(elems, nblocks : int64; is_gelu : bool; comb_n : int64 = 0l; comb_k : int64 = 0l; comb_r0 : int64 = 0l; comb_r1 : int64 = 0l; xf_words : int64 = 0l) { unsafe { var mp = reinterpret(g_gpu.ffn_meta.mapped) mp[0] = uint(elems) mp[1] = is_gelu ? 1u : 0u mp[2] = uint(nblocks) - mp[3] = 0u + mp[3] = uint(xf_words) // cm2 chain: the dequant kernel's word count (q8 image / 4) mp[4] = uint(comb_n) mp[5] = uint(comb_k) mp[6] = uint(comb_r0) @@ -6354,6 +7071,45 @@ def private ensure_batch_state { to_log(LOG_INFO, "dasLLAMA vulkan tier: prefill batch tier engaged\n") } +// hazard bits for the cm2 chain's two f16 planes +let private VHB_XF = 0x8000u // f16 gathered-activation plane (dequant out, gate/up twin B) +let private VHB_HF = 0x10000u // f16 hidden plane (actf16 out, down twin B) + +// cm2 batch state: the two f16 planes (2 bytes per element — 2x the q8 image caps bound the +// same chunk rows) and the dequant/act convert sets. Mode 4 + fmt-6 stacks only. +def private ensure_cm2_batch { + if (g_gpu.cm2_batch_ready) { + return + } + ensure_batch_state() + g_gpu.batch_xf_dev = make_device_buf(2l * BATCH_XQ_BYTES) + g_gpu.batch_hf_dev = make_device_buf(2l * BATCH_HQ_BYTES) + g_gpu.batch_xf_set = alloc_desc_set() + g_gpu.batch_hf_set = alloc_desc_set() + var writes : array + // dequant bindings: f16 out@0, params@2, xq/xs@3/4; 1/5 unused filler + write_buf_desc(writes, g_gpu.batch_xf_set, 0u, g_gpu.batch_xf_dev, 2l * BATCH_XQ_BYTES) + write_buf_desc(writes, g_gpu.batch_xf_set, 1u, g_gpu.batch_xs_dev, BATCH_XS_BYTES) + write_buf_desc(writes, g_gpu.batch_xf_set, 2u, g_gpu.ffn_meta_dev, FFN_META_BYTES) + write_buf_desc(writes, g_gpu.batch_xf_set, 3u, g_gpu.batch_xq_dev, BATCH_XQ_BYTES) + write_buf_desc(writes, g_gpu.batch_xf_set, 4u, g_gpu.batch_xs_dev, BATCH_XS_BYTES) + write_buf_desc(writes, g_gpu.batch_xf_set, 5u, g_gpu.batch_y1_dev, BATCH_Y_BYTES) + // act-f16 bindings mirror actrq: up@0, params@2, f16 hidden out@3, gate@5; 1/4 unused filler + write_buf_desc(writes, g_gpu.batch_hf_set, 0u, g_gpu.batch_y2_dev, BATCH_Y_BYTES) + write_buf_desc(writes, g_gpu.batch_hf_set, 1u, g_gpu.batch_xs_dev, BATCH_XS_BYTES) + write_buf_desc(writes, g_gpu.batch_hf_set, 2u, g_gpu.ffn_meta_dev, FFN_META_BYTES) + write_buf_desc(writes, g_gpu.batch_hf_set, 3u, g_gpu.batch_hf_dev, 2l * BATCH_HQ_BYTES) + write_buf_desc(writes, g_gpu.batch_hf_set, 4u, g_gpu.hs_dev, BATCH_HS_BYTES) + write_buf_desc(writes, g_gpu.batch_hf_set, 5u, g_gpu.batch_y1_dev, BATCH_Y_BYTES) + let no_copies : array + update_descriptor_sets(g_gpu.device, writes, no_copies) + hz_set_bits(g_gpu.batch_xf_set, VHB_XF, 0u, VHB_META, VHB_ACT, VHB_ACT, 0u) + hz_set_bits(g_gpu.batch_hf_set, VHB_Y2, 0u, VHB_META, VHB_HF, 0u, VHB_Y1) + g_gpu.resident_bytes += 2l * BATCH_XQ_BYTES + 2l * BATCH_HQ_BYTES + g_gpu.cm2_batch_ready = true + to_log(LOG_INFO, "dasLLAMA vulkan tier: cm2 prefill chain engaged (decode-in-load, f16 activations)\n") +} + def private ensure_comb_state { if (g_gpu.comb_ready) { return @@ -6394,7 +7150,7 @@ def private ensure_batch_set(si : int; ybuf, xqbuf, xsbuf : uint64; xqbytes, xsb } var writes : array write_buf_desc(writes, g_gpu.stacks[si].batch_set, 0u, g_gpu.stacks[si].wqbuf, g_gpu.stacks[si].wqbytes) - write_buf_desc(writes, g_gpu.stacks[si].batch_set, 1u, g_gpu.stacks[si].wsbuf, g_gpu.stacks[si].wsbytes) + write_buf_desc(writes, g_gpu.stacks[si].batch_set, 1u, g_gpu.stacks[si].wsbuf, ws_range(si)) write_buf_desc(writes, g_gpu.stacks[si].batch_set, 2u, g_gpu.stacks[si].batch_meta_dev, BATCH_META_BYTES) write_buf_desc(writes, g_gpu.stacks[si].batch_set, 3u, xqbuf, xqbytes) write_buf_desc(writes, g_gpu.stacks[si].batch_set, 4u, xsbuf, xsbytes) @@ -6449,8 +7205,8 @@ def private upload_batch_acts(xqp : int8 const?; xsp : float const?; r0, rows, n let xs_bytes = rows * (n / xunit) * 4l unsafe { var sp = reinterpret(g_gpu.staging.mapped) - par_memcpy(g_gpu.staging.mapped, reinterpret(xqp + r0 * n), xq_bytes) - par_memcpy(reinterpret(sp + BATCH_XQ_BYTES), reinterpret(xsp + r0 * (n / xunit)), xs_bytes) + par_memcpy(g_gpu.staging.mapped, reinterpret(xqp + r0 * n), xq_bytes) + par_memcpy(reinterpret(sp + BATCH_XQ_BYTES), reinterpret(xsp + r0 * (n / xunit)), xs_bytes) } } @@ -6468,7 +7224,8 @@ def private cmd_copy_batch_acts(raw : VkCommandBuffer; xq_bytes, xs_bytes : int6 // window), then the per-workgroup region map. Weight offsets follow the stack's format units — // 32-weight blocks for q8, superblocks for kq. Returns the workgroup count. def private fill_batch_meta(si : int; offs : int64 const?; nregions : int64; n, d : int64; r0, r1 : int64) : uint { - let unit = g_gpu.stacks[si].fmt != 0 ? 256l : 32l + let fmt = g_gpu.stacks[si].fmt + let unit = fmt_unit(fmt) unsafe { var mp = reinterpret(g_gpu.stacks[si].batch_meta.mapped) var nrc = 0l @@ -6483,12 +7240,14 @@ def private fill_batch_meta(si : int; offs : int64 const?; nregions : int64; n, all128 = all128 && (b1 - b0) % 128l == 0l } } - // route the tile on the AVERAGE rows per active region — one pipeline serves the dispatch - let tile = g_gpu.stacks[si].fmt == 0 ? q8_batch_tile_for(d, nrc > 0l ? nrows / nrc : r1 - r0) : BATCH_TILE + // route the tile on the AVERAGE rows per active region; the cm2 twin tiles 64w x 128t + let tile = fmt == 0 ? q8_batch_tile_for(d, nrc > 0l ? nrows / nrc : r1 - r0) : BATCH_TILE + let tile_d = fmt == 6 ? 64l : tile + let tile_c = fmt == 6 ? 128l : tile g_gpu.stacks[si].batch_tile = tile - g_gpu.stacks[si].batch_aligned = g_gpu.stacks[si].fmt == 0 && tile == 128l && d % 128l == 0l && all128 - let wtiles = (d + tile - 1l) / tile - let wg_bound = ((r1 - r0 + tile - 1l) / tile + nrc) * wtiles + g_gpu.stacks[si].batch_aligned = fmt == 0 && tile == 128l && d % 128l == 0l && all128 + let wtiles = (d + tile_d - 1l) / tile_d + let wg_bound = ((r1 - r0 + tile_c - 1l) / tile_c + nrc) * wtiles assert((4l + nrc * 4l + wg_bound) * 4l <= BATCH_META_BYTES, "dasLLAMA vulkan tier: batch meta exceeds capacity") let map_off = 4l + nrc * 4l mp[0] = uint(n) @@ -6510,7 +7269,7 @@ def private fill_batch_meta(si : int; offs : int64 const?; nregions : int64; n, mp[mb + 1l] = uint(rb0 - r0) mp[mb + 2l] = uint(rb1 - rb0) mp[mb + 3l] = uint(wg) - let rwg = ((rb1 - rb0 + tile - 1l) / tile) * wtiles + let rwg = ((rb1 - rb0 + tile_c - 1l) / tile_c) * wtiles for (g in range64(rwg)) { mp[map_off + wg + g] = uint(ri) } @@ -6565,14 +7324,23 @@ def private record_ffn_batch_cmd(s1, s3, s2 : int; nwg_gu, nwg_dn : uint; rows, cmd_copy_whole(raw, g_gpu.comb_stage.buf, g_gpu.comb_w_dev, comb_wi_bytes) cmd_copy_at(raw, g_gpu.comb_stage.buf, COMB_WI_BYTES, g_gpu.comb_inv_dev, comb_wi_bytes) } + if (g_gpu.stacks[s1].fmt == 6) { // the cm2 chain: dequant feed, twin GEMMs, f16 mid-step + hz_dispatch(raw, h, weak_copy(g_gpu.xf16_pipeline), g_gpu.batch_xf_set, + uint((axq_bytes / 4l + 255l) / 256l)) + } hz_dispatch(raw, h, batch_pipe(g_gpu.stacks[s1].fmt, g_gpu.stacks[s1].batch_tile, g_gpu.stacks[s1].batch_aligned), g_gpu.stacks[s1].batch_set, nwg_gu) hz_dispatch(raw, h, batch_pipe(g_gpu.stacks[s3].fmt, g_gpu.stacks[s3].batch_tile, g_gpu.stacks[s3].batch_aligned), g_gpu.stacks[s3].batch_set, nwg_gu) - // the requant form follows the DOWN stack — its GEMM is the consumer of hq/hs - let dkq = g_gpu.stacks[s2].fmt != 0 - hz_dispatch(raw, h, weak_copy(dkq ? g_gpu.actrq_k_pipeline : g_gpu.actrq_pipeline), - g_gpu.batch_actrq_set, uint((rows * nfe / (dkq ? 8l : 4l) + 255l) / 256l)) + // the mid-step form follows the DOWN stack — its GEMM is the consumer of the hidden plane + if (g_gpu.stacks[s2].fmt == 6) { + hz_dispatch(raw, h, weak_copy(g_gpu.actf16_pipeline), g_gpu.batch_hf_set, + uint((rows * nfe / 4l + 255l) / 256l)) + } else { + let dkq = g_gpu.stacks[s2].fmt != 0 + hz_dispatch(raw, h, weak_copy(dkq ? g_gpu.actrq_k_pipeline : g_gpu.actrq_pipeline), + g_gpu.batch_actrq_set, uint((rows * nfe / (dkq ? 8l : 4l) + 255l) / 256l)) + } hz_dispatch(raw, h, batch_pipe(g_gpu.stacks[s2].fmt, g_gpu.stacks[s2].batch_tile, g_gpu.stacks[s2].batch_aligned), g_gpu.stacks[s2].batch_set, nwg_dn) if (comb_npos > 0u) { @@ -6598,15 +7366,28 @@ def private vk_moe_ffn_batch(var goutp : float?; offs1 : int64 const?; offs3 : i let s3 = find_stack(unsafe(offs3[0]), f3) let s2 = find_stack(unsafe(offs2[0]), f2) // gate/up share one activation image (forms must agree); down's intermediate is its own unit - let kq_in = f1 != 0 - assert((f3 != 0) == kq_in, + let kq_in = !vk_fmt_b32(f1) + assert(!vk_fmt_b32(f3) == kq_in, "dasLLAMA vulkan tier: gate/up form mismatch — they share one activation image") - let xunit_in = kq_in ? 256l : 32l - let xunit_mid = f2 != 0 ? 256l : 32l + let xunit_in = fmt_unit(f1) + let xunit_mid = fmt_unit(f2) ensure_batch_state() - ensure_batch_set(s1, g_gpu.batch_y1_dev, g_gpu.batch_xq_dev, g_gpu.batch_xs_dev, BATCH_XQ_BYTES, BATCH_XS_BYTES, VHB_ACT, VHB_Y1) - ensure_batch_set(s3, g_gpu.batch_y2_dev, g_gpu.batch_xq_dev, g_gpu.batch_xs_dev, BATCH_XQ_BYTES, BATCH_XS_BYTES, VHB_ACT, VHB_Y2) - ensure_batch_set(s2, g_gpu.batch_y1_dev, g_gpu.hq_dev, g_gpu.hs_dev, BATCH_HQ_BYTES, BATCH_HS_BYTES, VHB_HQ, VHB_Y1) + if (f1 == 6 || f2 == 6) { + ensure_cm2_batch() // any twin in the chain needs the f16 planes + convert sets + } + // a q8n gate/up pair loads the dequantized f16 plane as B; a q8n down loads the f16 hidden + if (f1 == 6) { + ensure_batch_set(s1, g_gpu.batch_y1_dev, g_gpu.batch_xf_dev, g_gpu.batch_xs_dev, 2l * BATCH_XQ_BYTES, BATCH_XS_BYTES, VHB_XF, VHB_Y1) + ensure_batch_set(s3, g_gpu.batch_y2_dev, g_gpu.batch_xf_dev, g_gpu.batch_xs_dev, 2l * BATCH_XQ_BYTES, BATCH_XS_BYTES, VHB_XF, VHB_Y2) + } else { + ensure_batch_set(s1, g_gpu.batch_y1_dev, g_gpu.batch_xq_dev, g_gpu.batch_xs_dev, BATCH_XQ_BYTES, BATCH_XS_BYTES, VHB_ACT, VHB_Y1) + ensure_batch_set(s3, g_gpu.batch_y2_dev, g_gpu.batch_xq_dev, g_gpu.batch_xs_dev, BATCH_XQ_BYTES, BATCH_XS_BYTES, VHB_ACT, VHB_Y2) + } + if (f2 == 6) { + ensure_batch_set(s2, g_gpu.batch_y1_dev, g_gpu.batch_hf_dev, g_gpu.hs_dev, 2l * BATCH_HQ_BYTES, BATCH_HS_BYTES, VHB_HF, VHB_Y1) + } else { + ensure_batch_set(s2, g_gpu.batch_y1_dev, g_gpu.hq_dev, g_gpu.hs_dev, BATCH_HQ_BYTES, BATCH_HS_BYTES, VHB_HQ, VHB_Y1) + } let comb = npos > 0l var comb_k = 0l if (comb) { @@ -6617,8 +7398,8 @@ def private vk_moe_ffn_batch(var goutp : float?; offs1 : int64 const?; offs3 : i "dasLLAMA vulkan tier: combined output exceeds capacity") unsafe { var cp = reinterpret(g_gpu.comb_stage.mapped) - par_memcpy(g_gpu.comb_stage.mapped, reinterpret(wp), nrows * 4l) - par_memcpy(reinterpret(cp + COMB_WI_BYTES), reinterpret(invp), nrows * 4l) + par_memcpy(g_gpu.comb_stage.mapped, reinterpret(wp), nrows * 4l) + par_memcpy(reinterpret(cp + COMB_WI_BYTES), reinterpret(invp), nrows * 4l) } } let chunk = ffn_chunk_rows(n, nfe, xunit_in, xunit_mid) @@ -6634,7 +7415,8 @@ def private vk_moe_ffn_batch(var goutp : float?; offs1 : int64 const?; offs3 : i let ts1 = ref_time_ticks() upload_batch_acts(xqp, xsp, r0, rows, n, xunit_in) let ts2 = ref_time_ticks() - fill_ffn_meta(rows * nfe, rows * nfe / xunit_mid, is_gelu, comb ? n : 0l, comb_k, r0, r1) + fill_ffn_meta(rows * nfe, rows * nfe / xunit_mid, is_gelu, comb ? n : 0l, comb_k, r0, r1, + f1 == 6 ? rows * n / 4l : 0l) record_ffn_batch_cmd(s1, s3, s2, nwg1, nwg2, rows, nfe, rows * n * 4l, rows * n, rows * (n / xunit_in) * 4l, [comb_npos = comb ? uint(npos) : 0u, comb_first = r0 == 0l, comb_last = r1 == nrows, comb_wi_bytes = nrows * 4l, comb_y_bytes = npos * n * 4l]) @@ -6683,7 +7465,7 @@ def private record_dense_cmd(si : int; nwg : uint; y_bytes : int64; axq_bytes, a //! the FFN arm; activations arrive pre-quantized in the plane's own form. def vk_moe_dense(var yp : float?; woff : int64; xqp : int8 const?; xsp : float const?; n, d, nrows : int64; fmt : int) { let si = find_stack(woff, fmt) - let xunit = fmt != 0 ? 256l : 32l + let xunit = fmt_unit(fmt) ensure_batch_state() ensure_batch_set(si, g_gpu.batch_y1_dev, g_gpu.batch_xq_dev, g_gpu.batch_xs_dev, BATCH_XQ_BYTES, BATCH_XS_BYTES, VHB_ACT, VHB_Y1) var chunk = min(BATCH_XQ_BYTES / n, BATCH_XS_BYTES / ((n / xunit) * 4l)) @@ -6892,12 +7674,12 @@ def vk_moe_dn(var yp : float?; woq, woz, woo : int64; var sp = reinterpret(g_gpu.dn_smalls.mapped) sp[0] = eps sp[1] = 1.0 / sqrt(float(ds)) - memcpy(reinterpret(sp + DN_SM_TAPS), reinterpret(convw), cd * dconv * 4l) - memcpy(reinterpret(sp + DN_SM_WNORM), reinterpret(wnormp), ds * 4l) - memcpy(reinterpret(sp + DN_SM_A), reinterpret(ap), nvh * 4l) - memcpy(reinterpret(sp + DN_SM_DT), reinterpret(dtp), nvh * 4l) - memcpy(reinterpret(sp + DN_SM_HIST), reinterpret(histp), cd * taps * 4l) - memcpy(g_gpu.dn_state_stage.mapped, reinterpret(stp), nvh * ds * ds * 4l) + memcpy(reinterpret(sp + DN_SM_TAPS), reinterpret(convw), cd * dconv * 4l) + memcpy(reinterpret(sp + DN_SM_WNORM), reinterpret(wnormp), ds * 4l) + memcpy(reinterpret(sp + DN_SM_A), reinterpret(ap), nvh * 4l) + memcpy(reinterpret(sp + DN_SM_DT), reinterpret(dtp), nvh * 4l) + memcpy(reinterpret(sp + DN_SM_HIST), reinterpret(histp), cd * taps * 4l) + memcpy(g_gpu.dn_state_stage.mapped, reinterpret(stp), nvh * ds * ds * 4l) } var qkv_region = fixed_array(woq, 0l, 0l) var z_region = fixed_array(woz, 0l, 0l) @@ -6920,8 +7702,8 @@ def vk_moe_dn(var yp : float?; woq, woz, woo : int64; upload_batch_acts(xqp, xsp, r0, rows, dim, 32l) unsafe { var sp = reinterpret(g_gpu.dn_smalls.mapped) - par_memcpy(reinterpret(sp + DN_SM_BETA), reinterpret(bp + r0 * nvh), rows * nvh * 4l) - par_memcpy(reinterpret(sp + DN_SM_G), reinterpret(gp + r0 * nvh), rows * nvh * 4l) + par_memcpy(reinterpret(sp + DN_SM_BETA), reinterpret(bp + r0 * nvh), rows * nvh * 4l) + par_memcpy(reinterpret(sp + DN_SM_G), reinterpret(gp + r0 * nvh), rows * nvh * 4l) } fill_dn_meta(rows, cd, kd, di, nvh, nkh, ds, dconv, r0 == 0l ? 1 : 2, rows * di / 32l) let ts1 = ref_time_ticks() @@ -7070,12 +7852,12 @@ def private dnd_cold_upload(st : DnStep; stp : float const?; histp : float const var sp = reinterpret(g_gpu.dn_smalls.mapped) sp[0] = eps sp[1] = 1.0 / sqrt(float(ds)) - memcpy(reinterpret(sp + DN_SM_TAPS), reinterpret(convw), cd * dconv * 4l) - memcpy(reinterpret(sp + DN_SM_WNORM), reinterpret(wnormp), ds * 4l) - memcpy(reinterpret(sp + DN_SM_A), reinterpret(ap), nvh * 4l) - memcpy(reinterpret(sp + DN_SM_DT), reinterpret(dtp), nvh * 4l) - memcpy(reinterpret(sp + DN_SM_HIST), reinterpret(histp), cd * taps * 4l) - memcpy(g_gpu.dn_state_stage.mapped, reinterpret(stp), st.state_bytes) + memcpy(reinterpret(sp + DN_SM_TAPS), reinterpret(convw), cd * dconv * 4l) + memcpy(reinterpret(sp + DN_SM_WNORM), reinterpret(wnormp), ds * 4l) + memcpy(reinterpret(sp + DN_SM_A), reinterpret(ap), nvh * 4l) + memcpy(reinterpret(sp + DN_SM_DT), reinterpret(dtp), nvh * 4l) + memcpy(reinterpret(sp + DN_SM_HIST), reinterpret(histp), cd * taps * 4l) + memcpy(g_gpu.dn_state_stage.mapped, reinterpret(stp), st.state_bytes) } let raw = g_gpu.batch_cmd let rf : VkCommandBufferResetFlags @@ -7112,8 +7894,8 @@ def private vk_moe_dn_step(var yp : float?; s_qkv, s_z, s_o : int; woq, woz, woo fill_stack_acts(s_qkv, xqp, xsp, dim) // the z GEMV set shares this image unsafe { var bg = reinterpret(g_gpu.dnd_bg.mapped) - memcpy(reinterpret(bg), reinterpret(bp), nvh * 4l) - memcpy(reinterpret(bg + nvh), reinterpret(gp), nvh * 4l) + memcpy(reinterpret(bg), reinterpret(bp), nvh * 4l) + memcpy(reinterpret(bg + nvh), reinterpret(gp), nvh * 4l) bg[2l * nvh] = float(st.parity) } let ts1 = ref_time_ticks() @@ -7352,8 +8134,8 @@ def vk_moe_attn(var yp : float?; var kp : float?; var vp : float?; sp[0] = eps sp[1] = scale if (qknorm) { - memcpy(reinterpret(sp + AT_SM_RMSQ), reinterpret(rmsqp), hs * 4l) - memcpy(reinterpret(sp + AT_SM_RMSK), reinterpret(rmskp), hs * 4l) + memcpy(reinterpret(sp + AT_SM_RMSQ), reinterpret(rmsqp), hs * 4l) + memcpy(reinterpret(sp + AT_SM_RMSK), reinterpret(rmskp), hs * 4l) } } var q_region = fixed_array(woq, 0l, 0l) @@ -7385,8 +8167,8 @@ def vk_moe_attn(var yp : float?; var kp : float?; var vp : float?; upload_batch_acts(xqp, xsp, r0, rows, dim, xunit) unsafe { var sp = reinterpret(g_gpu.at_smalls.mapped) - par_memcpy(reinterpret(sp + AT_SM_COS), reinterpret(cosp + r0 * half), rows * half * 4l) - par_memcpy(reinterpret(sp + AT_SM_COS + rows * half), reinterpret(sinp + r0 * half), rows * half * 4l) + par_memcpy(reinterpret(sp + AT_SM_COS), reinterpret(cosp + r0 * half), rows * half * 4l) + par_memcpy(reinterpret(sp + AT_SM_COS + rows * half), reinterpret(sinp + r0 * half), rows * half * 4l) } fill_at_meta(rows, r0, qd, kv_dim, hs, half, n_heads, n_kv, kv_mul, rows * qd / (fo != 0 ? 256l : 32l), gated, qknorm) @@ -7422,14 +8204,14 @@ def private ffn_cmd_for(s1, s3, s2 : int; nrows, n, nfe : int64) : VkCommandBuff var writes : array // fused act+requant bindings: up@0, params@2, hq/hs@3/4, gate@5; binding 1 is unused filler write_buf_desc(writes, fc.actrq_set, 0u, g_gpu.stacks[s3].y_dev, Y_BYTES) - write_buf_desc(writes, fc.actrq_set, 1u, g_gpu.stacks[s1].wsbuf, g_gpu.stacks[s1].wsbytes) + write_buf_desc(writes, fc.actrq_set, 1u, g_gpu.stacks[s1].wsbuf, ws_range(s1)) write_buf_desc(writes, fc.actrq_set, 2u, g_gpu.ffn_meta_dev, FFN_META_BYTES) write_buf_desc(writes, fc.actrq_set, 3u, g_gpu.hq_dev, BATCH_HQ_BYTES) write_buf_desc(writes, fc.actrq_set, 4u, g_gpu.hs_dev, BATCH_HS_BYTES) write_buf_desc(writes, fc.actrq_set, 5u, g_gpu.stacks[s1].y_dev, Y_BYTES) // down: its own planes + host-visible meta, hq/hs as the activation image, its y_dev write_buf_desc(writes, fc.down_set, 0u, g_gpu.stacks[s2].wqbuf, g_gpu.stacks[s2].wqbytes) - write_buf_desc(writes, fc.down_set, 1u, g_gpu.stacks[s2].wsbuf, g_gpu.stacks[s2].wsbytes) + write_buf_desc(writes, fc.down_set, 1u, g_gpu.stacks[s2].wsbuf, ws_range(s2)) write_buf_desc(writes, fc.down_set, 2u, g_gpu.stacks[s2].meta.buf, META_BYTES) write_buf_desc(writes, fc.down_set, 3u, g_gpu.hq_dev, BATCH_HQ_BYTES) write_buf_desc(writes, fc.down_set, 4u, g_gpu.hs_dev, BATCH_HS_BYTES) @@ -7458,8 +8240,8 @@ def private ffn_cmd_for(s1, s3, s2 : int; nrows, n, nfe : int64) : VkCommandBuff let ggu = uint((nrows * nfe + g_gpu.rows_per_wg - 1l) / g_gpu.rows_per_wg) hz_dispatch(raw, h, gemv_pipe(g_gpu.stacks[s1].fmt), g_gpu.stacks[s1].raw_set, ggu) hz_dispatch(raw, h, gemv_pipe(g_gpu.stacks[s3].fmt), g_gpu.stacks[s3].raw_set, ggu) - // the requant form follows the DOWN stack — its GEMV is the consumer of hq/hs - let dkq = g_gpu.stacks[s2].fmt != 0 + // the requant form follows the DOWN stack (q8n counts as q8 — decode acts stay Q8_0) + let dkq = !vk_fmt_b32(g_gpu.stacks[s2].fmt) hz_dispatch(raw, h, weak_copy(dkq ? g_gpu.actrq_k_pipeline : g_gpu.actrq_pipeline), fc.actrq_set, uint((nrows * nfe / (dkq ? 8l : 4l) + 255l) / 256l)) hz_dispatch(raw, h, gemv_pipe(g_gpu.stacks[s2].fmt), fc.down_set, uint((nrows * n + g_gpu.rows_per_wg - 1l) / g_gpu.rows_per_wg)) @@ -7484,11 +8266,11 @@ def private ffn_gemv_prep(offs1 : int64 const?; offs3 : int64 const?; offs2 : in let s2 = find_stack(unsafe(offs2[0]), f2) assert(nrows * max(nfe, n) * 4l <= Y_BYTES, "dasLLAMA vulkan tier: FFN dispatch exceeds output capacity") // gate/up share one activation image (same xqp below) so forms must agree; down's is its own - let kq_in = f1 != 0 - assert((f3 != 0) == kq_in, + let kq_in = !vk_fmt_b32(f1) + assert(!vk_fmt_b32(f3) == kq_in, "dasLLAMA vulkan tier: gate/up form mismatch — they share one activation image") - let xunit_in = kq_in ? 256l : 32l - let xunit_mid = f2 != 0 ? 256l : 32l + let xunit_in = fmt_unit(f1) + let xunit_mid = fmt_unit(f2) ensure_gemv_scratch(s1) ensure_gemv_scratch(s3) ensure_gemv_scratch(s2) @@ -7621,13 +8403,16 @@ def private heat_stream_src(base : int64; fmt : int) : int { def private heat_pool_make(l : int64; b1, b3, b2 : int64; f1, f3, f2 : int; n, nfe : int64) { ensure_batch_state() // advise records its slice copies into batch_cmd var pool = HeatPool(ege = n * nfe) - let kq = f1 != 0 let src1 = heat_stream_src(b1, f1) let src3 = heat_stream_src(b3, f3) let src2 = heat_stream_src(b2, f2) let want = gpu_want_heat() - if (src1 < 0 || src3 < 0 || src2 < 0 || ((f3 != 0) != kq) || ((f2 != 0) != kq) || want <= 0l) { - to_log(LOG_INFO, "dasLLAMA vulkan tier: heat cache skips layer {l} (no stream mirrors)\n") + // gate/up share one act image; down is free — the mid-step form follows IT (the mixed-triple rail) + if (src1 < 0 || src3 < 0 || src2 < 0 || vk_fmt_b32(f3) != vk_fmt_b32(f1) || want <= 0l) { + let why = (want <= 0l ? "want 0" + : (src1 < 0 || src3 < 0 || src2 < 0 ? "no mirror src {src1}/{src3}/{src2} for b {b1}/{b3}/{b2} fmt {f1}/{f3}/{f2}" + : "gate/up form mismatch fmt {f1}/{f3}")) + to_log(LOG_INFO, "dasLLAMA vulkan tier: heat cache skips layer {l} ({why})\n") g_gpu.heat_pools[l] <- pool return } @@ -7669,8 +8454,10 @@ def private heat_record_slice(raw : VkCommandBuffer; src_si, dst_si : int; e, sl let rbs = g_gpu.stream_stacks[src_si].wsbytes / rows_src cmd_copy_range(raw, g_gpu.stream_stacks[src_si].hwq.buf, e * rpe * rbq, g_gpu.stacks[dst_si].wqbuf, slot * rpe * rbq, rpe * rbq) - cmd_copy_range(raw, g_gpu.stream_stacks[src_si].hws.buf, e * rpe * rbs, - g_gpu.stacks[dst_si].wsbuf, slot * rpe * rbs, rpe * rbs) + if (rbs > 0l) { // q8n mirrors carry no scale plane + cmd_copy_range(raw, g_gpu.stream_stacks[src_si].hws.buf, e * rpe * rbs, + g_gpu.stacks[dst_si].wsbuf, slot * rpe * rbs, rpe * rbs) + } } //! MoeGpuHeatAdviseFn: reconcile layer l's pool toward the desired hot set (ids heat-descending). @@ -7771,15 +8558,16 @@ def private ensure_stream_slots { g_gpu.stream_cmd = alloc_cmd() g_gpu.stream_slots |> reserve(2) for (_slot in range(2)) { - let s1 = make_stack_shell(-1l, 0l, 0, g_gpu.stream_max_gu_wq, g_gpu.stream_max_gu_ws) - let s3 = make_stack_shell(-1l, 0l, 0, g_gpu.stream_max_gu_wq, g_gpu.stream_max_gu_ws) - let s2 = make_stack_shell(-1l, 0l, 0, g_gpu.stream_max_dn_wq, g_gpu.stream_max_dn_ws) + let s1 = make_stack_shell(-1l, 0l, 0, g_gpu.stream_max_gu_wq, g_gpu.stream_max_gu_ws, [xfer_shared = true]) + let s3 = make_stack_shell(-1l, 0l, 0, g_gpu.stream_max_gu_wq, g_gpu.stream_max_gu_ws, [xfer_shared = true]) + let s2 = make_stack_shell(-1l, 0l, 0, g_gpu.stream_max_dn_wq, g_gpu.stream_max_dn_ws, [xfer_shared = true]) g_gpu.stream_slots |> push(StreamSlot(group = -1, s1 = s1, s3 = s3, s2 = s2)) } g_gpu.stream_ready = true let slot_bytes = 2l * (g_gpu.stream_max_gu_wq + g_gpu.stream_max_gu_ws) + g_gpu.stream_max_dn_wq + g_gpu.stream_max_dn_ws // reserve was carved from the weight budget at init — overcommit here silently hits the WDDM paging cliff - assert(2l * slot_bytes <= STREAM_RESERVE, "dasLLAMA vulkan tier: stream slots exceed STREAM_RESERVE") + let reserve = g_gpu.stream_reserve > 0l ? g_gpu.stream_reserve : STREAM_RESERVE + assert(2l * slot_bytes <= reserve, "dasLLAMA vulkan tier: stream slots exceed the carved stream reserve") to_log(LOG_INFO, "dasLLAMA vulkan tier: streamed prefill engaged ({length(g_gpu.stream_stacks) / 3} layers, slot {slot_bytes} bytes x 2)\n") } @@ -7791,12 +8579,30 @@ def private rebase_pseudo(si : int; ss : StreamStack const) { g_gpu.stacks[si].fmt = ss.fmt } -// async is UNFENCED — the prefetch rides the GPU-idle window, ordered by the next chain's -// transfer->compute barrier. One shared cmd buffer is safe: a fenced wait always intervenes -// between copy recordings, so the prior copy has retired off the queue. +// host-wait the transfer timeline up to `v` — the backstop before re-recording the shared xfer +// cmd, and the sync (cold-miss) completion wait +def private xfer_host_wait(v : uint64) { + var wi = VkSemaphoreWaitInfo() + var sem = g_gpu.xfer_sem + var val = v + unsafe { + wi.semaphoreCount = 1u + wi.pSemaphores = addr(sem) + wi.pValues = addr(val) + vk_check(vkWaitSemaphores(dev_raw(), wi, 0xFFFFFFFFFFFFFFFFul), null) + } +} + +// Transfer family armed: copies ride the dedicated queue signaling the timeline (true overlap; +// the slot's consumer attaches the GPU-side wait — see vk_moe_ffn_streamed). Single-queue mode +// keeps the old unfenced rail. Shared cmds are safe: a wait always intervenes between recordings. def private copy_stream_group(si, g : int; sync : bool) { let slot = g_gpu.stream_slots[si] - var raw = g_gpu.stream_cmd + let xfer = g_gpu.xfer_fam >= 0 + if (xfer && g_gpu.xfer_value > 0ul) { + xfer_host_wait(g_gpu.xfer_value) // every prior transfer-queue copy retired + } + var raw = xfer ? g_gpu.xfer_cmd : g_gpu.stream_cmd let rf : VkCommandBufferResetFlags vk_check(vkResetCommandBuffer(raw, rf), null) let begin = VkCommandBufferBeginInfo() @@ -7805,10 +8611,34 @@ def private copy_stream_group(si, g : int; sync : bool) { let sk = g * 3 + k let psi = k == 0 ? slot.s1 : (k == 1 ? slot.s3 : slot.s2) cmd_copy_whole(raw, g_gpu.stream_stacks[sk].hwq.buf, g_gpu.stacks[psi].wqbuf, g_gpu.stream_stacks[sk].wqbytes) - cmd_copy_whole(raw, g_gpu.stream_stacks[sk].hws.buf, g_gpu.stacks[psi].wsbuf, g_gpu.stream_stacks[sk].wsbytes) + if (g_gpu.stream_stacks[sk].wsbytes > 0l) { // q8n mirrors carry no scale plane + cmd_copy_whole(raw, g_gpu.stream_stacks[sk].hws.buf, g_gpu.stacks[psi].wsbuf, g_gpu.stream_stacks[sk].wsbytes) + } } vk_check(vkEndCommandBuffer(raw), null) - if (sync) { + if (xfer) { + g_gpu.xfer_value++ + var sig = g_gpu.xfer_value + var tss = VkTimelineSemaphoreSubmitInfo() + var sem = g_gpu.xfer_sem + var submit = VkSubmitInfo() + submit.commandBufferCount = 1u + submit.signalSemaphoreCount = 1u + let nofence : VkFence + unsafe { + tss.signalSemaphoreValueCount = 1u + tss.pSignalSemaphoreValues = addr(sig) + submit.pNext = addr(tss) + submit.pCommandBuffers = addr(raw) + submit.pSignalSemaphores = addr(sem) + vk_check(vkQueueSubmit(g_gpu.xfer_queue, 1u, addr(submit), nofence), null) + } + if (sync) { + xfer_host_wait(g_gpu.xfer_value) + } + // the consumer still attaches the GPU-side wait — cross-queue visibility by the book + g_gpu.stream_slots[si].wait_value = g_gpu.xfer_value + } elif (sync) { submit_wait(raw) } else { var submit = VkSubmitInfo() @@ -7841,6 +8671,11 @@ def private vk_moe_ffn_streamed(g : int; var goutp : float?; offs1 : int64 const rebase_pseudo(g_gpu.stream_slots[si].s1, g_gpu.stream_stacks[g * 3]) rebase_pseudo(g_gpu.stream_slots[si].s3, g_gpu.stream_stacks[g * 3 + 1]) rebase_pseudo(g_gpu.stream_slots[si].s2, g_gpu.stream_stacks[g * 3 + 2]) + // the slot's transfer-queue fill hands off here: the next compute submit waits the timeline + if (g_gpu.stream_slots[si].wait_value > 0ul) { + g_gpu.pend_wait = g_gpu.stream_slots[si].wait_value + g_gpu.stream_slots[si].wait_value = 0ul + } vk_moe_ffn_batch(goutp, offs1, offs3, offs2, nregions, xqp, xsp, wp, invp, n, nfe, nrows, npos, f1, f3, f2, is_gelu) if (g > 0 && g_gpu.stream_slots[(g - 1) % 2].group != g - 1) { copy_stream_group((g - 1) % 2, g - 1, false) @@ -8000,6 +8835,11 @@ def private vk_plane_bytes(n, rows : int64; fmt : int) : int64 { return pb.wq + pb.ws } +// the loader's upload router asks this before gathering q8 expert stacks: mode 4 (cm2) wants +// them as fmt-6 q8n blocks (decode-in-load). Probes the device — the answer must precede the +// first gather. +def private vk_expert_q8n : bool => vk_moe_init() && g_gpu.coopmat_mode == 4 + def private vk_device_name : string { if (g_gpu == null) { return "" @@ -8046,15 +8886,31 @@ def vulkan_moe_gpu_arm : bool { return true } +// the vulkan flavor's config identity. ENV-OR-AUTO knobs only — resolving them (vk_moe_init) +// would carve the stream reserve BEFORE the loader reports the model's slot need (518ae164a); +// "auto" resolves deterministically per box and the image is local, so the identity is stable. +def private vk_bake_tag : string { + if (!vulkan_moe_gpu_arm()) { + return "" + } + let cm = env_str("DASLLAMA_COOPMAT", "auto") + let vb = env_str("DASLLAMA_GPU_VRAM_MB", "auto") + return ("m{cm} b{vb}" + + " l{gpu_want_moe_layers()} s{gpu_want_moe_stream()}" + + " r{gpu_want_dn() ? 1 : 0}{gpu_want_attn() ? 1 : 0}{gpu_want_dnd() ? 1 : 0}{gpu_want_shexp() ? 1 : 0}{gpu_want_qkv() ? 1 : 0}{gpu_want_cls() ? 1 : 0}") +} + [init] def dasllama_math_vulkan_register() { register_kernel_backend(KernelBackend(name = "vulkan", mm = @@vulkan_no_gemv, batch = @@vulkan_no_batch, group3 = @@vulkan_no_group3, repack = @@vulkan_repack_noop, needs_repack = false, priority = -1, available = @@vulkan_backend_available)) set_moe_gpu_arm("vulkan", @@vulkan_moe_gpu_arm) + set_moe_gpu_bake_tag(@@vk_bake_tag) set_moe_gpu_vram_report(@@vk_resident_bytes) set_moe_gpu_device_report(@@vk_device_name) set_moe_gpu_budget_hooks(@@vk_weight_budget, @@vk_plane_bytes) + set_moe_gpu_expert_q8n_probe(@@vk_expert_q8n) install_moe_gpu_resident(@@vk_arena_reserve, @@vk_arena_place, @@vk_rdec_prepare, @@vk_rdec_upload_norms, @@vk_rdec_set_layer, @@vk_rdec_set_cls, @@vk_rdec_token, @@vk_rdec_prefill, @@vk_rdec_sync_kv, @@vk_rdec_read_kv, @@vk_rdec_read_kv_bulk) diff --git a/modules/dasLLAMA/dasllama/dasllama_qwen3a.das b/modules/dasLLAMA/dasllama/dasllama_qwen3a.das index 6f35c35b9c..e88c4e6042 100644 --- a/modules/dasLLAMA/dasllama/dasllama_qwen3a.das +++ b/modules/dasLLAMA/dasllama/dasllama_qwen3a.das @@ -52,7 +52,7 @@ def load_qwen3a_tower(path : string) : Qwen3aTower { panic("dasLLAMA: cannot open mmproj gguf '{path}'") } fmap(f) $(var bytes : array#) { - let m <- parse_gguf_meta(bytes) + var inscope m <- parse_gguf_meta(bytes) // qwen3a mmprojs spell the key clip.audio.projector_type (not clip.projector_type) let proj = (gguf_has(m, "clip.audio.projector_type") ? gguf_str(m, bytes, "clip.audio.projector_type") : gguf_str(m, bytes, "clip.projector_type")) diff --git a/modules/dasLLAMA/dasllama/dasllama_tokenizer.das b/modules/dasLLAMA/dasllama/dasllama_tokenizer.das index 8d861cb3e8..45ff6eb3ba 100644 --- a/modules/dasLLAMA/dasllama/dasllama_tokenizer.das +++ b/modules/dasLLAMA/dasllama/dasllama_tokenizer.das @@ -61,9 +61,9 @@ def load_tokenizer_gguf(path : string) : SpmTokenizer { panic("dasLLAMA: cannot open gguf '{path}'") } fmap(f) $(var bytes : array#) { - let m <- parse_gguf_meta(bytes) - let toks <- gguf_str_array(m, bytes, "tokenizer.ggml.tokens") - let scores <- gguf_f32_array(m, bytes, "tokenizer.ggml.scores") + var inscope m <- parse_gguf_meta(bytes) + var inscope toks <- gguf_str_array(m, bytes, "tokenizer.ggml.tokens") + var inscope scores <- gguf_f32_array(m, bytes, "tokenizer.ggml.scores") let n = length(toks) tk.vocab |> reserve(n) tk.scores |> reserve(n) @@ -189,7 +189,7 @@ def load_tokenizer_auto(path : string) : Tokenizer { } var model_kind = "" fmap(f) $(var bytes : array#) { - let m <- parse_gguf_meta(bytes) + var inscope m <- parse_gguf_meta(bytes) if (gguf_has(m, "tokenizer.ggml.model")) { model_kind = gguf_str(m, bytes, "tokenizer.ggml.model") } diff --git a/modules/dasLLAMA/dasllama/dasllama_transformer.das b/modules/dasLLAMA/dasllama/dasllama_transformer.das index 8d827442db..c7947aa4a7 100644 --- a/modules/dasLLAMA/dasllama/dasllama_transformer.das +++ b/modules/dasLLAMA/dasllama/dasllama_transformer.das @@ -7,6 +7,7 @@ module dasllama_transformer shared public // so its [init] registration fires. Consumers `require dasllama/dasllama_transformer` get the full API // plus all registered arches; adding an architecture = a new dasllama_arch_*.das + one require here. require dasllama/dasllama_common public +require dasllama/dasllama_layout // nolint:STYLE030 — [init]-only: registers the disk->compute layout hooks (repacks + GPU gathers) into dasllama_common; unset hooks panic at load require dasllama/dasllama_arch_llama // nolint:STYLE030 — side-effect require: fires the arch's [init] registration require dasllama/dasllama_arch_qwen2 // nolint:STYLE030 — side-effect require: fires the arch's [init] registration require dasllama/dasllama_arch_qwen3 // nolint:STYLE030 — side-effect require: fires the arch's [init] registration diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 7a646b9189..c834ad9d0a 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -766,8 +766,18 @@ def close_to_q51_ref(fx : Q51Fixture; y : array; mr : int64; var maxdiff return ok } -// reference walks the layout companion's resolve (grp1/disk in the tuner process) -def private q51_mr_of(sfx : string) : int64 => sfx == "mr8" ? 8l : (sfx == "mr4" ? 4l : int64(q51q8_layout_gen())) +// per-variant plane interleave off the layout companion registry — declined perms report the +// reference resolve (the lockstep rail). Trusting the SUFFIX instead repacks planes to a grp +// the declined-to-reference kernel then walks at the companion mr — the zen2 maxdiff-120 trap. +def private q51_layout_mrs() : table { + var mrs : table + var lvs <- q51q8_layout_gen_variants() + for (v in lvs) { + mrs[v._0] = invoke(v._1) + } + delete lvs + return <- mrs +} // The q51 family tune pass: gate every variant against the scalar disk oracle on repacked // planes, iso-bench the streamed decode shape, return the best suffix ("" = a gate failed). @@ -775,6 +785,7 @@ def q51_tune_family() : string { var sfx <- build_q51_fixture(704l, 32768l) // streamed: the 26B expert-down K at DRAM scale var hfx <- build_q51_fixture(704l, 2816l) // hot: one expert's down stack var tvs <- q51q8_gemv_gen_variants() + var mrs <- q51_layout_mrs() var best = "" var bestDt = 2147483647 var y : array @@ -782,7 +793,7 @@ def q51_tune_family() : string { var yh : array yh |> resize(int(hfx.d)) for (v in tvs) { - let mr = q51_mr_of(v._0) + let mr = int64(mrs?[v._0] ?? q51q8_layout_gen()) var sq := sfx.qq var ss := sfx.qs var hq := hfx.qq @@ -797,10 +808,12 @@ def q51_tune_family() : string { run_q51_gemv(v._1, hfx, hq, hs, mr, yh) ok = close_to_q51_ref(hfx, yh, mr, maxdiff) && ok if (!ok) { + // best = "" rejects the family; the shared cleanup past the loop frees everything + // (deleting tvs HERE, mid-iteration, is the locked-array trap) print("q51 {v._0} (mr={mr}): MISMATCH (maxdiff {maxdiff}) — aborting tune\n") delete sq; delete ss; delete hq; delete hs - delete sfx; delete hfx; delete tvs; delete y; delete yh - return "" + best = "" + break } var dt = 2147483647 for (_rep in range(3)) { @@ -819,6 +832,7 @@ def q51_tune_family() : string { delete sfx delete hfx delete tvs + delete mrs delete y delete yh return best @@ -1528,7 +1542,11 @@ def tune_mode_run : bool { print("TUNE_GEN_TIME k{fmt}_family {get_time_usec(kq_t0) / 1000} ms\n") let entry = fmt == 40l ? "q40q8_tile_gen" : "k{fmt}q8_tile_gen" if (empty(w)) { - kok = false + // gate failure (the broken perm is named above): record the always-correct + // reference body so the scope still COMPLETES — a missing key re-tunes every start + let wok = tune_manifest_set(entry, "reference") + print("{entry} winner: reference (gate-failure fallback) {wok ? "->" : "-> FAILED TO WRITE"} {tune_manifest_path()}\n") + kok = kok && wok } else { let wok = tune_manifest_set(entry, w) print("{entry} winner: {w} {wok ? "->" : "-> FAILED TO WRITE"} {tune_manifest_path()}\n") @@ -1539,7 +1557,10 @@ def tune_mode_run : bool { let qw = q51_tune_family() print("TUNE_GEN_TIME q51_family {get_time_usec(q51_t0) / 1000} ms\n") if (empty(qw)) { - kok = false + // same gate-failure fallback as the kq loop: complete the scope on the reference body + let qok = tune_manifest_set("q51q8_gemv_gen", "reference") + print("q51q8_gemv_gen winner: reference (gate-failure fallback) {qok ? "->" : "-> FAILED TO WRITE"} {tune_manifest_path()}\n") + kok = kok && qok } else { let qok = tune_manifest_set("q51q8_gemv_gen", qw) print("q51q8_gemv_gen winner: {qw} {qok ? "->" : "-> FAILED TO WRITE"} {tune_manifest_path()}\n") diff --git a/modules/dasLLAMA/harness/parity.das b/modules/dasLLAMA/harness/parity.das index 039d0299d3..b680961f49 100644 --- a/modules/dasLLAMA/harness/parity.das +++ b/modules/dasLLAMA/harness/parity.das @@ -9,6 +9,7 @@ require daslib/clargs require daslib/fio // has_env_variable / get_env_variable (the DASLLAMA_BATCH_GRID_2D A/B rail) require math require dasllama/dasllama_math // setup_dasllama_jobque_ + the runtime knob setters (engine-level, not re-exported) +require dasllama/dasllama_layout // convert_model_to_metal_blob — the blob-flavor transform // dasLLAMA parity harness — the dasLLAMA side of the token-for-token oracle check. // Feed the prompt token IDs the reference (harness/oracle/simple_ids) produced, greedily generate N diff --git a/modules/dasLLAMA/performance/profile_common.das b/modules/dasLLAMA/performance/profile_common.das index f7234b6275..37ff142198 100644 --- a/modules/dasLLAMA/performance/profile_common.das +++ b/modules/dasLLAMA/performance/profile_common.das @@ -895,6 +895,74 @@ struct BenchRun { @optional exec_fmt : string // what this engine EXECUTES for this file (weight formats per site // class + activation/scale treatment) — the apples-to-apples receipt; // das derives it from the loaded Model, llama.cpp rows from flavor + @optional env : string // the arming environment `cmd` was run under — see [[bench_env_line]]. + // Load-bearing: most GPU/tier levers arm via env, so a `cmd` alone + // does NOT reproduce a row and two configs can look identical +} + +// Every env var the dasLLAMA engine + the tune rail consult. Capturing the WHOLE set (not a curated +// "interesting" subset) is deliberate: an unrecorded var is exactly how two different configs come +// to look like the same row. +let private BENCH_ENV_VARS = fixed_array( + "DAS_JOBQUE_AFFINITY", "DAS_JOBQUE_TEAM_RANK_GATE", "DAS_JOBQUE_THREADS", + "DAS_TUNE_MANIFEST", "DAS_TUNE_MODE", + "DASLLAMA_ACCEL", "DASLLAMA_ACCEL_F16", "DASLLAMA_ACCEL_F16_STRIPS", "DASLLAMA_ACCEL_MIN_MMAC", + "DASLLAMA_ACCEL_MIN_NTOK", "DASLLAMA_ALLOW_INTERP_LOAD", "DASLLAMA_BATCH_GRID_2D", + "DASLLAMA_BENCH_VERBOSE", "DASLLAMA_BOX", "DASLLAMA_CALLER_PRIO", "DASLLAMA_CONV_PROF", + "DASLLAMA_COOPMAT", "DASLLAMA_CPU", "DASLLAMA_CPU_PREFILL", "DASLLAMA_EXPERT_REUSE", + "DASLLAMA_FUSED_DECODE", "DASLLAMA_GPU", "DASLLAMA_GPU_ATTN", "DASLLAMA_GPU_CLS", + "DASLLAMA_GPU_COMBINE", "DASLLAMA_GPU_DENSE", "DASLLAMA_GPU_DENSE_ATTN", + "DASLLAMA_GPU_DENSE_SHEXP", "DASLLAMA_GPU_DN", "DASLLAMA_GPU_DND", "DASLLAMA_GPU_HEAT", + "DASLLAMA_GPU_MIN_CTX", "DASLLAMA_GPU_MOE_LAYERS", "DASLLAMA_GPU_MOE_STREAM", + "DASLLAMA_GPU_NAME", "DASLLAMA_GPU_PROF", "DASLLAMA_GPU_QKV", "DASLLAMA_GPU_SHEXP", + "DASLLAMA_GPU_VRAM_MB", "DASLLAMA_IMAGE", "DASLLAMA_METAL_ATTN", "DASLLAMA_METAL_ATTN_D", + "DASLLAMA_METAL_ATTN_SINGLE", "DASLLAMA_METAL_BATCH_CONCURRENT", "DASLLAMA_METAL_BATCH_FUSE", + "DASLLAMA_METAL_BATCH_GEMM_MIN", "DASLLAMA_METAL_BATCH_MM", "DASLLAMA_METAL_BATCH_MV", + "DASLLAMA_METAL_BATCH_MV8", "DASLLAMA_METAL_BATCH_NCB", "DASLLAMA_METAL_BATCH_PIPE", + "DASLLAMA_METAL_BATCH_SK", "DASLLAMA_METAL_DECODE_CONCURRENT", "DASLLAMA_METAL_DECODE_SKIP", + "DASLLAMA_METAL_FUSE", "DASLLAMA_METAL_GEMV_TG", "DASLLAMA_METAL_HAZARD_PARANOID", + "DASLLAMA_METAL_HAZARD_STRICT", "DASLLAMA_METAL_KQ_B8", "DASLLAMA_METAL_KV_MIRROR_MB", + "DASLLAMA_METAL_LOGITS", "DASLLAMA_METAL_MULMM", "DASLLAMA_METAL_NCB", + "DASLLAMA_METAL_PF_CAPTURE", "DASLLAMA_METAL_PIPE_DEBUG", "DASLLAMA_METAL_PREFILL_SKIP", + "DASLLAMA_METAL_SCHED", "DASLLAMA_METAL_SPEC", "DASLLAMA_METAL_UNRETAINED", + "DASLLAMA_MM_SMALL", "DASLLAMA_MODELS_DIR", "DASLLAMA_MTP_DEBUG", "DASLLAMA_NOISY", + "DASLLAMA_OS", "DASLLAMA_PIN_BACKEND", "DASLLAMA_PIN_PREFILL", "DASLLAMA_RAM_GB", + "DASLLAMA_THREADS", "DASLLAMA_TRUTH_REFRESH", "DASLLAMA_VK_FA", "DASLLAMA_VK_HAZARD_PARANOID", + "DASLLAMA_VK_HAZARD_TRACE", "DASLLAMA_VK_MEMPRIO", "DASLLAMA_VK_REBAR", "DASLLAMA_VK_XFERQ") + +// quote on anything a shell reads as syntax, not just spaces — full cross-shell escaping is not +// well-defined (cmd and POSIX disagree on the rules), so widen the trigger rather than pretend +def private shell_quote(s : string) : string { + var bare = !empty(s) + peek_data(s) $(arr) { + for (b in arr) { + let c = int(b) + bare = bare && (is_alpha(c) || is_number(c) || c == '_' || c == '.' || c == '/' + || c == '\\' || c == ':' || c == '=' || c == '-' || c == '+' || c == ',') + } + } + return bare ? s : "\"{s}\"" +} + +//! The run's REAL invocation, from the process argv — binary, interpreter flags, script and args +//! alike. Not a reconstruction: a rebuilt `-- {parsed args}` string loses the binary actually used +//! and silently drops anything the arg parser did not model. +def bench_cmd_line() : string { + var parts <- [for (a in get_command_line_arguments()); shell_quote(a)] + let line = join(parts, " ") + delete parts + return line +} + +//! The set members of [[BENCH_ENV_VARS]] that are actually set, as a copy-pasteable `K=V ...` +//! prefix. Pair with [[bench_cmd_line]] and the row reproduces exactly. +def bench_env_line() : string { + var parts <- [for (name in BENCH_ENV_VARS); + "{name}={shell_quote(get_env_variable(name))}"; + where has_env_variable(name)] + let line = join(parts, " ") + delete parts + return line } //! One GGUF's identity plus every run measured against it. `gguf` (the exact filename) is the diff --git a/modules/dasLLAMA/tests/test_metal_decode_parity.das b/modules/dasLLAMA/tests/test_metal_decode_parity.das index 5c6bacd2ce..f61f438755 100644 --- a/modules/dasLLAMA/tests/test_metal_decode_parity.das +++ b/modules/dasLLAMA/tests/test_metal_decode_parity.das @@ -5,6 +5,7 @@ options stack = 131072 // several by-value Sessions per arm in one frame (test require dastest/testing_boost public require dasllama/dasllama // the public facade — engine + arch registrations + (on Apple) the metal overrides require dasllama/dasllama_math // set_wscale_f16 — the resident path reads the f32 scale plane +require dasllama/dasllama_layout // convert_model_to_metal_blob — the blob-flavor transform require ?das_metal dasllama/dasllama_metal_llama // nolint:STYLE030 — stats/mirror/shutdown rails require ?das_metal dasllama/dasllama_metal_prefill // nolint:STYLE030 — the Metal-prefill→GPU-decode hybrid arm require ?das_metal metal/das_metal_boost // nolint:STYLE030 — metal_live_object_count for the leak gate diff --git a/modules/dasLLAMA/tests/test_metal_prefill_parity.das b/modules/dasLLAMA/tests/test_metal_prefill_parity.das index 8e4461c07c..7e1dcf8d63 100644 --- a/modules/dasLLAMA/tests/test_metal_prefill_parity.das +++ b/modules/dasLLAMA/tests/test_metal_prefill_parity.das @@ -5,6 +5,7 @@ options stack = 131072 // by-value Sessions in the kq + quantized-KV arms (tes require dastest/testing_boost public require dasllama/dasllama // the public facade — pulls the engine + arch registrations + (on Apple) the metal prefill override require dasllama/dasllama_math // set_wscale_f16 — the resident path reads the f32 scale plane +require dasllama/dasllama_layout // convert_model_to_metal_blob — the blob-flavor transform require ?das_metal dasllama/dasllama_metal_prefill // nolint:STYLE030 — min-npos setter + stats + the shutdown/leak rail require ?das_metal dasllama/dasllama_metal_llama // nolint:STYLE030 — decode shutdown (the kq/quantized-KV arms touch the decode driver) require ?das_metal metal/das_metal_boost // nolint:STYLE030 — metal_live_object_count for the leak gate diff --git a/modules/dasLLAMA/tests/test_metal_support_matrix.das b/modules/dasLLAMA/tests/test_metal_support_matrix.das index b575d23791..6ce0b5efbc 100644 --- a/modules/dasLLAMA/tests/test_metal_support_matrix.das +++ b/modules/dasLLAMA/tests/test_metal_support_matrix.das @@ -5,6 +5,7 @@ options stack = 131072 // several by-value Sessions per cell (test_batch_decod require dastest/testing_boost public require dasllama/dasllama // nolint:STYLE029 — the public facade (engine + arch registrations + the metal overrides); off-Apple the static_if halves compile out and only its dasllama_common re-export stays referenced require dasllama/dasllama_math // set_wscale_f16 — the wscale matrix axis +require dasllama/dasllama_layout // convert_model_to_metal_blob — the blob-flavor transform require dasllama/dasllama_parity // nolint:STYLE030 — cached_ids/cached_vals (CPU references mapped like .dlim images), used only inside the Apple static_if half require ?das_metal dasllama/dasllama_metal_llama // nolint:STYLE030 — stats/declines/shutdown rails require ?das_metal dasllama/dasllama_metal_prefill // nolint:STYLE030 — prefill stats/declines rails diff --git a/modules/dasLLAMA/tests/test_mtp.das b/modules/dasLLAMA/tests/test_mtp.das index 70e4e2a2ae..9b7643f99c 100644 --- a/modules/dasLLAMA/tests/test_mtp.das +++ b/modules/dasLLAMA/tests/test_mtp.das @@ -253,3 +253,58 @@ def test_mtp_invariance_27b(t : T?) { delete tr } } + +// GLM-4.5-Air Q4_K_M — the only NON-RECURRENT MTP model we have, so this is the sole arm covering +// mtp_spec_eval's `recr_mask == 0` reject branch (a rejected draft consumes the verify's own row-0 +// logits + h0 instead of paying a second full forward; every qwen35 MTP file is a deltanet hybrid +// and takes the restore-and-re-advance branch instead). It is also the first arch combining +// attn_qkv_bias with a NextN block — the shape where the qkv/out bias arrays were sized by the +// TRUNK layer count, so blk..attn_{q,k,v}.bias never loaded and the draft read past the +// end of t.bq. And glm4moe ships nextn.embed_tokens/shared_head_head as copies of the trunk's own +// tables, which the loader shape-checks rather than disabling the head over. +// Large tier (~73 GB, 2 shards): DASLLAMA_PARITY_FULL=1. +[test] +def test_mtp_invariance_glm(t : T?) { + t |> run("MTP self-spec output-invariance (GLM-4.5-Air Q4_K_M, non-recurrent reject path)") @(t : T?) { + if (!jit_enabled()) { + t |> skip("interpreted (dasLLAMA model tests are JIT-only)") + return + } + let path = path_join(models_dir(), "glm45air/Q4_K_M/GLM-4.5-Air-Q4_K_M-00001-of-00002.gguf") + if (!model_available(t, path)) return + var tr <- load_model(path, QuantMode.q8) // load_model: this arm encodes its prompt in-test + t |> equal(tr.config.n_layer_nextn, 1l) // head ENABLED, not disabled over the table copies + t |> equal(tr.config.n_layers, 46l) // block_count 47 - nextn 1 + // the two properties that make this arm load-bearing — asserted so a future file losing + // either tells us the coverage moved instead of silently passing + t |> equal(tr.config.recr_mask, 0ul) + t |> success(tr.config.attn_qkv_bias, "glm4moe carries qkv bias (the layers_ld sizing case)") + tr.config.seq_len = 2048l + with_pinned_kernels() { + // A CODE prompt on purpose. Strict token-identity is only meaningful on a NEAR-TIE-FREE + // fixture: on K-quant planes the accept path reads BATCH-kernel logits (mm_gemm/mm_b_f) + // while plain decode reads GEMV (mm_kq), and those differ in reduction order — a + // legitimate FP difference that `with_pinned_kernels` does NOT cover (it pins attention + // mode, activations and deltanet, not the matmul family). At Q4_K that delta is far + // below the model's own quantization error, so on tie-dense PROSE the two paths pick + // differently at ties and identity fails for no defect (measured: prose diverges every + // run, this fixture is bit-identical at 95% accept). Code is strongly determined ⇒ no + // ties to break ⇒ identity is a real assertion. Same reasoning as the 27B arm's + // deliberately near-tie-free counting fixture. + // FOLLOW-UP: a margin-gated arm can cover prose — assert that any divergence sits below + // the quantization noise floor; mtp_verify_logits already computes the margin. + let code <- encode(tr, "def fibonacci(n):", true, false) + let total = length(code) + 40 + var nd = 0l + var na = 0l + let ref_ids <- gen_plain(tr, code, total) + let spec_ids <- gen_spec(tr, code, total, nd, na) + t |> success(same(ref_ids, spec_ids), "code-glm: spec decode token-for-token vs plain") + t |> success(nd > 0l, "code-glm: the draft head actually drafted ({nd} drafts)") + // a 100%-accept run would leave the reject branch untouched and the arm would prove + // nothing about it — so require that at least one draft was actually rejected + t |> success(na < nd, "code-glm: {nd - na} of {nd} drafts rejected — reject branch covered") + } + delete tr + } +} diff --git a/modules/dasLLAMA/tests/test_vulkan_tier.das b/modules/dasLLAMA/tests/test_vulkan_tier.das index 5d2205bc12..5c82fa98fc 100644 --- a/modules/dasLLAMA/tests/test_vulkan_tier.das +++ b/modules/dasLLAMA/tests/test_vulkan_tier.das @@ -23,6 +23,16 @@ require math // model's activations enter. The gelu arm stays untested until a gelu-MoE q8 fixture exists. // Without dasVulkan (or with no Vulkan device) every check feints cleanly. +// The CPU-reference arms' tolerances were derived against the sdot4 reference kernels — pin +// that mode ahead of the router's fastest-path default (mm) so the oracles stay deterministic. +// The coopmat kernels' numeric envelope is validated by the drift-class model gates, not here. +[init] +def private pin_reference_gemm_mode { + static_if (typeinfo builtin_module_exists(vulkan)) { + vk_force_coopmat_mode(0) + } +} + let N = 96l // model dim (x row length), %32 let NFE = 64l // expert FFN dim, %32 let NE = 3l // experts in each synthetic stack diff --git a/modules/dasLLAMA/tests/test_vulkan_tier_cm2.das b/modules/dasLLAMA/tests/test_vulkan_tier_cm2.das new file mode 100644 index 0000000000..c5d5541e8b --- /dev/null +++ b/modules/dasLLAMA/tests/test_vulkan_tier_cm2.das @@ -0,0 +1,462 @@ +options gen2 +options stack = 524288 + +require dastest/testing_boost public +require dasllama/dasllama_math +require ?vulkan dasllama/dasllama_math_vulkan // nolint:STYLE030 — the MoE GPU tier under test; the test installs its hooks explicitly (no env gate) +require llvm/daslib/f16_cvt +require math + +// The cm2 (NV_cooperative_matrix2) twin vs a CPU reference — fmt-6 q8n stacks (gguf-native 34B +// interleaved blocks, NO scale plane) through the same public hook surface as the q8 suite: +// decode rides the q8n native-block GEMV (Q8_0 activations, requant mid-chain — int-exact like +// the q8 arms), prefill rides the decode-in-load tensor-tile twin with f16 activations and the +// no-requant f16 mid-chain. Weight and activation scales are powers of two, so every f16 tile +// element is EXACT — the only prefill drift is the f16 accumulate, judged at CM2_TOL. Runs in +// its own process because the suite pin is per-device: this file pins mode 4 (cm2) where +// test_vulkan_tier pins the sdot4 reference mode. Without dasVulkan, a Vulkan device, or the +// cm2 extension, every check feints cleanly. + +[init] +def private pin_cm2_gemm_mode { + static_if (typeinfo builtin_module_exists(vulkan)) { + vk_force_coopmat_mode(4) // downgrades to mm on a non-cm2 device; the probe below gates + } +} + +let N = 96l // model dim (x row length), %32; the down GEMM's d = 96 = 64 + 32 tail tile +let NFE = 64l // expert FFN dim — gate/up d = exactly one 64-wide weight tile +let NE = 3l // experts in each synthetic stack +let WOFF1 = 1000000l // gate / up / down stack base element offsets (arbitrary distinct keys) +let WOFF3 = 2000000l +let WOFF2 = 3000000l +let WOFFQ1 = 4000000l // the mixed arm's fmt-0 gate/up twins +let WOFFQ3 = 5000000l +let SWOFF = 20000000l // the streamed fixture's layer base +let REL_TOL = 1e-3 // int-exact arms (decode GEMV chain) +let CM2_TOL = 2e-2 // f16-accumulate arms (the twin's tensor tiles) + +def private pow2_scale(i : int64) : float { + let p = i % 5l + return p == 0l ? 1.0 : (p == 1l ? 0.5 : (p == 2l ? 2.0 : (p == 3l ? 0.25 : 0.125))) +} + +// small pow2 scales keep |gate| in the sane silu regime AND make every dequantized f16 exact +def private pow2_small(i : int64) : float { + let p = i % 3l + return p == 0l ? 0.00390625 : (p == 1l ? 0.001953125 : 0.0078125) // 2^-8 / 2^-9 / 2^-7 +} + +def private fill_stack(var wq : array; var ws : array; rows, n : int64; seed : int64; small_scales : bool) { + wq |> resize(int(rows * n)) + ws |> resize(int(rows * (n / 32l))) + for (i in range(int(rows * n))) { + wq[i] = int8((int64(i) * 7l + seed) % 255l - 127l) + } + for (i in range(int(rows * (n / 32l)))) { + ws[i] = small_scales ? pow2_small(int64(i) + seed) : pow2_scale(int64(i) + seed) + } +} + +// the q8n wire format: per row per 32-block — [f16 d LE][32 int8 quants] = 34 bytes, ONE plane +def private pack_q8n(wq : array; ws : array; rows, n : int64; var wqb : array) { + let nb = n / 32l + wqb |> resize(int(rows * nb * 34l)) + for (r in range64(rows)) { + for (b in range64(nb)) { + let o = int((r * nb + b) * 34l) + let h = f32_to_f16(ws[int(r * nb + b)]) + wqb[o] = uint8(h & 0xFFu) + wqb[o + 1] = uint8((h >> 8u) & 0xFFu) + for (j in range(32)) { + wqb[o + 2 + j] = uint8(wq[int(r * n + b * 32l) + j]) + } + } + } +} + +// the fmt-0 upload wire format (the mixed arm's gate/up planes) +def private pack_upload(wq : array; ws : array; var wqb : array; var wsb : array) { + wqb |> resize(length(wq)) + for (i in range(length(wq))) { + wqb[i] = uint8(wq[i]) + } + wsb |> resize(length(ws) * 2) + for (i in range(length(ws))) { + let h = f32_to_f16(ws[i]) + wsb[i * 2] = uint8(h & 0xFFu) + wsb[i * 2 + 1] = uint8((h >> 8u) & 0xFFu) + } +} + +def private fill_acts(var xq : array; var xs : array; nrows : int64) { + xq |> resize(int(nrows * N)) + xs |> resize(int(nrows * (N / 32l))) + for (i in range(int(nrows * N))) { + xq[i] = int8((int64(i) * 11l + 3l) % 255l - 127l) + } + for (i in range(int(nrows * (N / 32l)))) { + xs[i] = pow2_small(int64(i) + 2l) + } +} + +def private make_offs(experts : array; row0s : array; cnts : array; woff : int64; ege : int64) : array { + var offs : array + offs |> reserve(length(experts) * 3) + for (r in range(length(experts))) { + offs |> push(woff + experts[r] * ege) // nolint:STYLE012 — triple-stride append + offs |> push(row0s[r]) + offs |> push(cnts[r]) + } + return <- offs +} + +// The CPU reference of the FFN contract. requant=true models the decode chain (Q8_0 actrq +// between the halves, the q8 suite's formula); requant=false models the cm2 prefill chain — +// act(g)*u feeds the down GEMM as floats (the GPU rounds them to f16; CM2_TOL covers it). +def private ref_ffn(offs1, offs3, offs2 : array; nreg : int; + wq1 : array; ws1 : array; wq3 : array; ws3 : array; + wq2 : array; ws2 : array; + xq : array; xs : array; nrows : int64; requant : bool; + w1off, w3off, w2off : int64; var gout : array) { + gout |> resize(int(nrows * N)) + let nb = int(N / 32l) + let nbh = int(NFE / 32l) + var inscope gate : array + var inscope up : array + var inscope h : array + var inscope hq : array + var inscope hs : array + gate |> resize(int(NFE)) + up |> resize(int(NFE)) + h |> resize(int(NFE)) + hq |> resize(int(NFE)) + hs |> resize(nbh) + for (r in range(nreg)) { + let rb1 = int((offs1[r * 3] - w1off) / N) // first gate-plane row of this region's expert + let rb3 = int((offs3[r * 3] - w3off) / N) + let rb2 = int((offs2[r * 3] - w2off) / NFE) + let row0 = int(offs1[r * 3 + 1]) + let cnt = int(offs1[r * 3 + 2]) + for (rw in range(row0, row0 + cnt)) { + for (i in range(int(NFE))) { + var ag = 0.0 + var au = 0.0 + for (b in range(nb)) { + var dg = 0 + var du = 0 + for (j in range(32)) { + let xv = int(xq[rw * int(N) + b * 32 + j]) + dg += int(wq1[(rb1 + i) * int(N) + b * 32 + j]) * xv + du += int(wq3[(rb3 + i) * int(N) + b * 32 + j]) * xv + } + let xscl = xs[rw * nb + b] + ag += float(dg) * ws1[(rb1 + i) * nb + b] * xscl + au += float(du) * ws3[(rb3 + i) * nb + b] * xscl + } + gate[i] = ag + up[i] = au + } + for (i in range(int(NFE))) { + h[i] = (gate[i] / (1.0 + exp(-gate[i]))) * up[i] + } + if (requant) { + for (b in range(nbh)) { + var amax = 0.0 + for (j in range(32)) { + amax = max(amax, abs(h[b * 32 + j])) + } + let d = amax / 127.0 + let id = d != 0.0 ? 1.0 / d : 0.0 + hs[b] = d + for (j in range(32)) { + hq[b * 32 + j] = int(round(h[b * 32 + j] * id)) + } + } + for (o in range(int(N))) { + var y = 0.0 + for (b in range(nbh)) { + var dd = 0 + for (j in range(32)) { + dd += int(wq2[(rb2 + o) * int(NFE) + b * 32 + j]) * hq[b * 32 + j] + } + y += float(dd) * ws2[(rb2 + o) * nbh + b] * hs[b] + } + gout[rw * int(N) + o] = y + } + } else { + for (o in range(int(N))) { + var y = 0.0 + for (i in range(int(NFE))) { + y += float(wq2[(rb2 + o) * int(NFE) + i]) * ws2[(rb2 + o) * nbh + i / 32] * h[i] + } + gout[rw * int(N) + o] = y + } + } + } + } +} + +def private compare_gout(gpu, refv : array) : tuple { + var md = 0.0 + var mr = 0.0 + for (i in range(length(refv))) { + md = max(md, abs(gpu[i] - refv[i])) + mr = max(mr, abs(refv[i])) + } + return (maxdiff = md, maxref = mr) +} + +[test] +def test_vulkan_moe_ffn_cm2(t : T?) { + t |> run("vulkan cm2 twin == CPU reference (q8n decode GEMV + decode-in-load prefill)") @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + install_moe_gpu_tier(@@vk_moe_ffn, @@vk_moe_upload_stack, @@vk_moe_cls, @@vk_moe_dense, @@vk_moe_drop_stacks, @@vk_moe_dn, @@vk_moe_attn) + if (!moe_gpu_expert_q8n()) { + feint("no Vulkan device or no NV_cooperative_matrix2; cm2 twin untestable here\n") + return + } + var inscope wq1 : array + var inscope ws1 : array + var inscope wq3 : array + var inscope ws3 : array + var inscope wq2 : array + var inscope ws2 : array + fill_stack(wq1, ws1, NE * NFE, N, 0l, true) + fill_stack(wq3, ws3, NE * NFE, N, 5l, true) + fill_stack(wq2, ws2, NE * N, NFE, 9l, true) + var inscope wqb : array + var inscope no_ws : array // q8n stacks upload with an EMPTY scale plane + pack_q8n(wq1, ws1, NE * NFE, N, wqb) + let up_ok = moe_gpu_upload_stack(wqb, no_ws, WOFF1, N, NE * NFE, 6) + if (!up_ok) { + feint("q8n upload refused (probe/budget); skipping\n") + return + } + pack_q8n(wq3, ws3, NE * NFE, N, wqb) + t |> success(moe_gpu_upload_stack(wqb, no_ws, WOFF3, N, NE * NFE, 6), "q8n up stack uploaded") + pack_q8n(wq2, ws2, NE * N, NFE, wqb) + t |> success(moe_gpu_upload_stack(wqb, no_ws, WOFF2, NFE, NE * N, 6), "q8n down stack uploaded") + let ege1 = NFE * N + let ege2 = N * NFE + + // decode arm: 2 rows, 1 per region — the q8n native-block GEMV + the Q8_0 requant + // mid-chain; int-exact like the q8 suite's decode arm + var inscope xq : array + var inscope xs : array + fill_acts(xq, xs, 2l) + var inscope offs1 <- make_offs([1l, 2l], [0l, 1l], [1l, 1l], WOFF1, ege1) + var inscope offs3 <- make_offs([1l, 2l], [0l, 1l], [1l, 1l], WOFF3, ege1) + var inscope offs2 <- make_offs([1l, 2l], [0l, 1l], [1l, 1l], WOFF2, ege2) + var inscope gout : array + gout |> resize(int(2l * N)) + matmul_moe_gpu_ffn(gout, offs1, offs3, offs2, 2l, xq, xs, N, NFE, 2l, 6, 6, 6, false) + var inscope refv : array + ref_ffn(offs1, offs3, offs2, 2, wq1, ws1, wq3, ws3, wq2, ws2, xq, xs, 2l, true, + WOFF1, WOFF3, WOFF2, refv) + var c = compare_gout(gout, refv) + t |> success(c.maxref > 0.0, "decode arm produced non-trivial outputs") + t |> success(c.maxdiff <= float(REL_TOL) * c.maxref, + "q8n decode GEMV within tolerance (maxdiff {c.maxdiff} vs {float(REL_TOL) * c.maxref} bar, maxref {c.maxref})") + + // batch arm: 40 rows, 2 regions, deliberately unpadded — underfilled 128-wide token + // tiles + the 96 = 64+32 weight-tail tile on the down twin (clamp/discard coverage) + fill_acts(xq, xs, 40l) + var inscope boffs1 <- make_offs([0l, 2l], [0l, 24l], [24l, 16l], WOFF1, ege1) + var inscope boffs3 <- make_offs([0l, 2l], [0l, 24l], [24l, 16l], WOFF3, ege1) + var inscope boffs2 <- make_offs([0l, 2l], [0l, 24l], [24l, 16l], WOFF2, ege2) + gout |> resize(int(40l * N)) + matmul_moe_gpu_ffn(gout, boffs1, boffs3, boffs2, 2l, xq, xs, N, NFE, 40l, 6, 6, 6, false) + ref_ffn(boffs1, boffs3, boffs2, 2, wq1, ws1, wq3, ws3, wq2, ws2, xq, xs, 40l, false, + WOFF1, WOFF3, WOFF2, refv) + c = compare_gout(gout, refv) + t |> success(c.maxref > 0.0, "batch arm produced non-trivial outputs") + t |> success(c.maxdiff <= float(CM2_TOL) * c.maxref, + "cm2 prefill batch within tolerance (maxdiff {c.maxdiff} vs {float(CM2_TOL) * c.maxref} bar, maxref {c.maxref})") + + // multi-tile arm: a 130-row region spans two token tiles (128 + 2-row tail) + fill_acts(xq, xs, 200l) + var inscope moffs1 <- make_offs([0l, 1l], [0l, 130l], [130l, 70l], WOFF1, ege1) + var inscope moffs3 <- make_offs([0l, 1l], [0l, 130l], [130l, 70l], WOFF3, ege1) + var inscope moffs2 <- make_offs([0l, 1l], [0l, 130l], [130l, 70l], WOFF2, ege2) + gout |> resize(int(200l * N)) + matmul_moe_gpu_ffn(gout, moffs1, moffs3, moffs2, 2l, xq, xs, N, NFE, 200l, 6, 6, 6, false) + ref_ffn(moffs1, moffs3, moffs2, 2, wq1, ws1, wq3, ws3, wq2, ws2, xq, xs, 200l, false, + WOFF1, WOFF3, WOFF2, refv) + c = compare_gout(gout, refv) + t |> success(c.maxref > 0.0, "multi-tile arm produced non-trivial outputs") + t |> success(c.maxdiff <= float(CM2_TOL) * c.maxref, + "cm2 multi-token-tile batch within tolerance (maxdiff {c.maxdiff} vs {float(CM2_TOL) * c.maxref} bar, maxref {c.maxref})") + + // combined arm: the routed weighted-sum reduce composed onto the twin chain, judged + // against the same combine of the GPU's OWN per-row gout (isolates the combine op) + var inscope cw : array + var inscope cinv : array + cw |> resize(40) + cinv |> resize(40) + for (g in range(40)) { + cw[g] = 0.5 + 0.01 * float(g) + cinv[g] = uint((g * 7 + 3) % 40) + } + fill_acts(xq, xs, 40l) + gout |> resize(int(40l * N)) + matmul_moe_gpu_ffn(gout, boffs1, boffs3, boffs2, 2l, xq, xs, N, NFE, 40l, 6, 6, 6, false) + var inscope comb : array + comb |> resize(int(10l * N)) + matmul_moe_gpu_ffn_combined(comb, boffs1, boffs3, boffs2, 2l, xq, xs, cw, cinv, N, NFE, 40l, 10l, 6, 6, 6, false) + var inscope combref : array + combref |> resize(int(10l * N)) + for (p in range(10)) { + for (i in range(int(N))) { + var acc = 0.0 + for (j in range(4)) { + acc += cw[p * 4 + j] * gout[int(cinv[p * 4 + j]) * int(N) + i] + } + combref[p * int(N) + i] = acc + } + } + let cc = compare_gout(comb, combref) + t |> success(cc.maxref > 0.0, "combined arm produced non-trivial outputs") + t |> success(cc.maxdiff <= float(REL_TOL) * cc.maxref, + "cm2 combined arm within tolerance (maxdiff {cc.maxdiff} vs {float(REL_TOL) * cc.maxref} bar, maxref {cc.maxref})") + + // mixed arm: fmt-0 gate/up (the mm/sdot4 rail) + q8n down — the actf16 mid-step + // keyed on the DOWN stack composes with a non-cm2 front half (the gemma triple shape) + var inscope wsb : array + pack_upload(wq1, ws1, wqb, wsb) + t |> success(moe_gpu_upload_stack(wqb, wsb, WOFFQ1, N, NE * NFE, 0), "mixed-arm q8 gate uploaded") + pack_upload(wq3, ws3, wqb, wsb) + t |> success(moe_gpu_upload_stack(wqb, wsb, WOFFQ3, N, NE * NFE, 0), "mixed-arm q8 up uploaded") + var inscope xoffs1 <- make_offs([0l, 2l], [0l, 24l], [24l, 16l], WOFFQ1, ege1) + var inscope xoffs3 <- make_offs([0l, 2l], [0l, 24l], [24l, 16l], WOFFQ3, ege1) + fill_acts(xq, xs, 40l) + gout |> resize(int(40l * N)) + matmul_moe_gpu_ffn(gout, xoffs1, xoffs3, boffs2, 2l, xq, xs, N, NFE, 40l, 0, 0, 6, false) + ref_ffn(xoffs1, xoffs3, boffs2, 2, wq1, ws1, wq3, ws3, wq2, ws2, xq, xs, 40l, false, + WOFFQ1, WOFFQ3, WOFF2, refv) + c = compare_gout(gout, refv) + t |> success(c.maxref > 0.0, "mixed arm produced non-trivial outputs") + t |> success(c.maxdiff <= float(CM2_TOL) * c.maxref, + "q8-gate/up + q8n-down mixed chain within tolerance (maxdiff {c.maxdiff} vs {float(CM2_TOL) * c.maxref} bar, maxref {c.maxref})") + + delete cw + delete cinv + delete comb + delete combref + delete xoffs1 + delete xoffs3 + } else { + feint("dasVulkan not installed; vulkan cm2 tier untestable here\n") + } + } +} + +// streamed arm: one q8n layer group in pinned mirrors (NO scale plane — the ws-skip rails), +// dispatched through the slot rail at batch rows vs the same no-requant reference +[test] +def test_vulkan_moe_ffn_cm2_streamed(t : T?) { + t |> run("vulkan cm2 twin, streamed q8n mirrors == CPU reference") @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + install_moe_gpu_tier(@@vk_moe_ffn, @@vk_moe_upload_stack, @@vk_moe_cls, @@vk_moe_dense, @@vk_moe_drop_stacks, @@vk_moe_dn, @@vk_moe_attn) + if (!moe_gpu_expert_q8n()) { + feint("no Vulkan device or no NV_cooperative_matrix2; cm2 twin untestable here\n") + return + } + var inscope wq1 : array + var inscope ws1 : array + var inscope wq3 : array + var inscope ws3 : array + var inscope wq2 : array + var inscope ws2 : array + fill_stack(wq1, ws1, NE * NFE, N, 21l, true) + fill_stack(wq3, ws3, NE * NFE, N, 26l, true) + fill_stack(wq2, ws2, NE * N, NFE, 30l, true) + var inscope wqb : array + var inscope no_ws : array + pack_q8n(wq1, ws1, NE * NFE, N, wqb) + let up_ok = moe_gpu_upload_stack(wqb, no_ws, SWOFF, N, NE * NFE, 6, [stream = true]) + if (!up_ok) { + feint("q8n stream upload refused (probe/budget); skipping\n") + return + } + pack_q8n(wq3, ws3, NE * NFE, N, wqb) + t |> success(moe_gpu_upload_stack(wqb, no_ws, SWOFF + 1000000l, N, NE * NFE, 6, [stream = true]), "streamed q8n up mirrored") + pack_q8n(wq2, ws2, NE * N, NFE, wqb) + t |> success(moe_gpu_upload_stack(wqb, no_ws, SWOFF + 2000000l, NFE, NE * N, 6, [stream = true]), "streamed q8n down mirrored") + let ege1 = NFE * N + let ege2 = N * NFE + var inscope xq : array + var inscope xs : array + fill_acts(xq, xs, 40l) + var inscope offs1 <- make_offs([0l, 2l], [0l, 24l], [24l, 16l], SWOFF, ege1) + var inscope offs3 <- make_offs([0l, 2l], [0l, 24l], [24l, 16l], SWOFF + 1000000l, ege1) + var inscope offs2 <- make_offs([0l, 2l], [0l, 24l], [24l, 16l], SWOFF + 2000000l, ege2) + var inscope gout : array + gout |> resize(int(40l * N)) + matmul_moe_gpu_ffn(gout, offs1, offs3, offs2, 2l, xq, xs, N, NFE, 40l, 6, 6, 6, false) + var inscope refv : array + ref_ffn(offs1, offs3, offs2, 2, wq1, ws1, wq3, ws3, wq2, ws2, xq, xs, 40l, false, + SWOFF, SWOFF + 1000000l, SWOFF + 2000000l, refv) + let c = compare_gout(gout, refv) + t |> success(c.maxref > 0.0, "streamed arm produced non-trivial outputs") + t |> success(c.maxdiff <= float(CM2_TOL) * c.maxref, + "streamed q8n batch within tolerance (maxdiff {c.maxdiff} vs {float(CM2_TOL) * c.maxref} bar, maxref {c.maxref})") + } else { + feint("dasVulkan not installed; vulkan cm2 tier untestable here\n") + } + } +} + +// arena arm: the resident driver's fmt-6 rails at the seam — a q8n plane placed with an EMPTY +// scale plane into the fmt-6 arena (the ws-floor guards), read back by the q8n GEMV over the +// arena block index. Q8_0 activations, so the chain is int-exact like the decode arm. +[test] +def test_vulkan_arena_cm2(t : T?) { + t |> run("vulkan cm2 arena: q8n place + arena GEMV == CPU reference") @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + install_moe_gpu_tier(@@vk_moe_ffn, @@vk_moe_upload_stack, @@vk_moe_cls, @@vk_moe_dense, @@vk_moe_drop_stacks, @@vk_moe_dn, @@vk_moe_attn) + if (!moe_gpu_expert_q8n()) { + feint("no Vulkan device or no NV_cooperative_matrix2; cm2 arena untestable here\n") + return + } + var inscope wq : array + var inscope ws : array + fill_stack(wq, ws, NFE, N, 13l, false) + var inscope wqb : array + var inscope no_ws : array + pack_q8n(wq, ws, NFE, N, wqb) + if (!vk_arena_reserve(6, NFE * (N / 32l))) { + feint("fmt-6 arena reserve refused (no device); skipping\n") + return + } + let blk = vk_arena_place(wqb, no_ws, N, NFE, 6) + t |> success(blk >= 0l, "q8n arena placement accepted an empty scale plane") + var inscope xq : array + var inscope xs : array + fill_acts(xq, xs, 1l) + var inscope y : array + y |> resize(int(NFE)) + vk_arena_gemv(y, blk, N, NFE, 6, xq, xs) + let nb = int(N / 32l) + var md = 0.0 + var mr = 0.0 + for (r in range(int(NFE))) { + var acc = 0.0 + for (b in range(nb)) { + var dd = 0 + for (j in range(32)) { + dd += int(wq[r * int(N) + b * 32 + j]) * int(xq[b * 32 + j]) + } + acc += float(dd) * ws[r * nb + b] * xs[b] + } + md = max(md, abs(y[r] - acc)) + mr = max(mr, abs(acc)) + } + t |> success(mr > 0.0, "arena arm produced non-trivial outputs") + t |> success(md <= float(REL_TOL) * mr, + "q8n arena GEMV within tolerance (maxdiff {md} vs {float(REL_TOL) * mr} bar, maxref {mr})") + } else { + feint("dasVulkan not installed; vulkan cm2 tier untestable here\n") + } + } +} diff --git a/modules/dasLLVM/daslib/llvm_exe.das b/modules/dasLLVM/daslib/llvm_exe.das index 5cf0119d5a..823ba5c123 100644 --- a/modules/dasLLVM/daslib/llvm_exe.das +++ b/modules/dasLLVM/daslib/llvm_exe.das @@ -1348,7 +1348,19 @@ def public inject_main(program_context : Context?; ctx : LLVMContextRef; } } - let main_ret = LLVMBuildCall2(builder, g_fn_types[start_fn_name], LLVMGetNamedFunction(g_mod, start_fn_name), main_params, "") + // Route main through the runtime's catch boundary (jit_run_main_guarded) — a das panic must + // PRINT and exit nonzero, not unwind out of the bare entry with no message. The guard owns + // the void/int/bool result mapping (0/1/2). + let guard_type = LLVMFunctionType(types.t_int32, + fixed_array(types.LLVMVoidPtrType(), types.LLVMVoidPtrType(), types.t_int32)) + var guard_fn = LLVMGetNamedFunction(g_mod, "jit_run_main_guarded") + if (guard_fn == null) { + guard_fn = LLVMAddFunctionWithType(g_mod, "jit_run_main_guarded", guard_type) + } + let guard_kind = no_return ? 0 : (bool_return ? 2 : 1) + let start_ptr = LLVMBuildPointerCast(builder, LLVMGetNamedFunction(g_mod, start_fn_name), types.LLVMVoidPtrType(), "") + var guard_params <- fixed_array(global_context, start_ptr, types.ConstI32(uint64(guard_kind))) + let main_ret = LLVMBuildCall2(builder, guard_type, guard_fn, guard_params, "main_rc") // Drain debug agents, modules, AOT library, env — the interpreter's teardown minus the handle-leak // dump. Without it the static g_DebugAgents map dtor races ref_count_mutex at exit (#2583). let shutdown_fn_type = LLVMFunctionType(types.t_void, array()) @@ -1357,16 +1369,7 @@ def public inject_main(program_context : Context?; ctx : LLVMContextRef; shutdown_fn = LLVMAddFunctionWithType(g_mod, "jit_shutdown", shutdown_fn_type) } LLVMBuildCall2(builder, shutdown_fn_type, shutdown_fn, array(), "") - if (no_return) { - LLVMBuildRet(builder, types.ConstI32(uint64(0))) - } elif (bool_return) { - // bool main → exit code: true → 0 (success), false → 1 (failure). - let exit_code = LLVMBuildSelect(builder, main_ret, - types.ConstI32(uint64(0)), types.ConstI32(uint64(1)), "main_exit") - LLVMBuildRet(builder, exit_code) - } else { - LLVMBuildRet(builder, main_ret) - } + LLVMBuildRet(builder, main_ret) to_log(LOG_INFO, "LLVM JIT: standalone exe entry point generated '{start_fn_name}', {length(funcs)} functions\n") return (fn = main_fn, link_whole_lib = needs_whole_lib) } diff --git a/src/builtin/module_jit.cpp b/src/builtin/module_jit.cpp index 5f49bd7c0d..9615d7df2c 100644 --- a/src/builtin/module_jit.cpp +++ b/src/builtin/module_jit.cpp @@ -1634,6 +1634,30 @@ DAS_API void jit_run_web_lifecycle ( das::Context * ctx, void * updateFn, #endif } +// Standalone-exe main guard: the generated entry (llvm_exe.das) routes das main through this +// so a runtime exception prints and exits nonzero instead of unwinding out of the entry with +// no message (hosted runs get this boundary from their runWithCatch call sites; a bare exe +// had none — fix for the silent-exit-127 class). resultKind: 0 = void main, 1 = int main +// (value = exit code), 2 = bool main (true -> 0, false -> 1). +DAS_API int32_t jit_run_main_guarded ( das::Context * ctx, void * mainFn, int32_t resultKind ) { + int32_t rc = 0; + bool ok = ctx->runWithCatch([&]() { + if ( resultKind == 1 ) { + rc = ((int32_t(*)(das::Context*))mainFn)(ctx); + } else if ( resultKind == 2 ) { + rc = ((bool(*)(das::Context*))mainFn)(ctx) ? 0 : 1; + } else { + ((void(*)(das::Context*))mainFn)(ctx); + } + }); + if ( !ok ) { + das::TextPrinter tp; + tp << "EXCEPTION: " << (ctx->getException() ? ctx->getException() : "unknown") << "\n"; + return 1; + } + return rc; +} + DAS_API void das_ensure_environment () { das::daScriptEnvironment::ensure(); } diff --git a/utils/mcp/daslang-mcp-msvc.cmd b/utils/mcp/daslang-mcp-msvc.cmd index 6fdc7f0b5f..4909879415 100644 --- a/utils/mcp/daslang-mcp-msvc.cmd +++ b/utils/mcp/daslang-mcp-msvc.cmd @@ -45,9 +45,14 @@ rem Capture the script dir BEFORE any `shift` (plain `shift` also shifts %0, so rem %~dp0 would otherwise stop pointing at this launcher's folder). set "MCPDIR=%~dp0" -rem Prefer the single-config (Ninja) binary, fall back to the MSVC Release layout. -set "DASLANG=%MCPDIR%..\..\bin\daslang.exe" -if not exist "%DASLANG%" set "DASLANG=%MCPDIR%..\..\bin\Release\daslang.exe" +rem The supervisor resolves the binary (newest of the single-config bin\ and the +rem MSVC bin\Release\ layouts) and passes it in DASLANG_MCP_BIN. Standalone runs +rem fall back to the MSVC Release layout first -- when both layouts exist, +rem first-wins on bin\daslang.exe silently ran a STALE binary whose DAS_BUILD_ID +rem no longer matched the tree's dynamic modules (every native `require` failed). +set "DASLANG=%MCPDIR%..\..\bin\Release\daslang.exe" +if not exist "%DASLANG%" set "DASLANG=%MCPDIR%..\..\bin\daslang.exe" +if defined DASLANG_MCP_BIN set "DASLANG=%DASLANG_MCP_BIN%" rem First arg selects the server. A *.exe selects a prebuilt server binary in rem bin/ (the AOT cpp-mcp) run directly; otherwise it's an interpreted .das diff --git a/utils/mcp/mcp_supervisor.py b/utils/mcp/mcp_supervisor.py index d66acb30b9..90919115af 100644 --- a/utils/mcp/mcp_supervisor.py +++ b/utils/mcp/mcp_supervisor.py @@ -120,12 +120,21 @@ def _spawn_and_replay(self): self._kill_proc() # clean up any prior child before respawning if self._stderr_fh is None: self._stderr_fh = open(self.stderr_log, "ab", buffering=0) + env = None + picked = _pick_binary(self.cwd) # re-resolved per spawn: a rebuild mid-session + launcher = list(self.launcher) # can change which layout is newest + if picked is not None: + # Windows goes through the vcvars .cmd, which reads this; POSIX execs the binary + # directly, so the pick has to replace argv[0] or it silently would not apply + env = dict(os.environ, DASLANG_MCP_BIN=picked) + if not IS_WINDOWS: + launcher[0] = picked self.proc = subprocess.Popen( - self.launcher, + launcher, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=self._stderr_fh, - cwd=self.cwd, text=True, encoding="utf-8", errors="replace", bufsize=1, + cwd=self.cwd, text=True, encoding="utf-8", errors="replace", bufsize=1, env=env, ) - _log(f"spawned daslang child pid={self.proc.pid}") + _log(f"spawned daslang child pid={self.proc.pid} bin={picked if IS_WINDOWS else launcher[0]}") if self.init_request is not None: self._write_line(json.dumps(self.init_request, separators=(",", ":"))) self._read_line() # consume & discard the replayed initialize result @@ -144,16 +153,40 @@ def _write_line(self, line: str): raise ChildDead(str(e)) def _read_line(self) -> str: + """Return the child's next JSON-RPC line. Anything else on its stdout is + DIAGNOSTIC NOISE, not a response — a daslang compile failure (stale + binary, build-id mismatch, missing shared_module) prints there, and + returning it as a frame silently corrupts the pipe and makes the client + drop the whole server. Noise is logged and skipped; if the child then + dies, it rides the ChildDead message so the client sees the real error.""" + noise: list[str] = [] while True: try: line = self.proc.stdout.readline() except (OSError, ValueError) as e: - raise ChildDead(str(e)) + raise ChildDead(self._with_noise(str(e), noise)) if line == "": - raise ChildDead("eof") + raise ChildDead(self._with_noise("eof", noise)) line = line.strip() - if line: - return line + if not line: + continue + if line.startswith("{"): + try: + json.loads(line) + return line + except Exception: + pass + _log(f"child noise: {line[:400]}") + if len(noise) < 40: + noise.append(line) + + @staticmethod + def _with_noise(reason: str, noise: list[str]) -> str: + # cap the WHOLE message, not just the noise tail — the reason can itself be an arbitrarily + # long OSError string, and this ends up inside a JSON-RPC error the client has to render + if not noise: + return reason[:2000] + return (f"{reason}; daslang child said: " + " | ".join(noise))[:2000] # ---- forwarding ----------------------------------------------------- def request(self, msg: dict) -> str: @@ -224,18 +257,44 @@ def handle(child: DaslangChild, msg: dict) -> str | None: "error": {"code": -32601, "message": f"method not found: {method}"}}) +BIN_CANDIDATES = ("bin/Release/daslang", "bin/daslang", "build/daslang", "build/bin/daslang") + + +def _pick_binary(repo_root: str) -> str | None: + """The NEWEST existing daslang binary among the single/multi-config output + locations. Newest-wins, not first-wins: a box that has built both layouts + (e.g. a stale Ninja `bin/daslang.exe` beside a fresh MSVC + `bin/Release/daslang.exe`) otherwise runs the stale one, whose DAS_BUILD_ID + no longer matches the tree's dynamic modules — every `require` of a native + module fails and the server never answers a single tool call. + DASLANG_MCP_BIN in the environment pins an explicit binary (bisect hatch); a + pin that does not exist is announced and ignored rather than handed on, since + it would otherwise surface as a bare FileNotFoundError at spawn — or, worse, + get written into .mcp.json — instead of naming the bad path.""" + pinned = os.environ.get("DASLANG_MCP_BIN") + if pinned: + if os.path.exists(pinned): + return pinned + _log(f"DASLANG_MCP_BIN points at a missing path, ignoring it: {pinned}") + print(f"WARNING: DASLANG_MCP_BIN={pinned} does not exist — falling back to auto-pick", + file=sys.stderr, flush=True) + exe = ".exe" if IS_WINDOWS else "" + found = [c for c in (os.path.join(repo_root, rel + exe) for rel in BIN_CANDIDATES) + if os.path.exists(c)] + if not found: + return None + return max(found, key=os.path.getmtime) + + def _default_launcher() -> list[str]: if IS_WINDOWS: + # The .cmd sets up vcvars (cpp_compile_check needs cl.exe on PATH) and + # then runs whichever binary DASLANG_MCP_BIN names — chosen here so the + # newest-wins rule is one implementation, not two. return ["cmd", "/c", os.path.join(SCRIPT_DIR, "daslang-mcp-msvc.cmd")] - # POSIX: run the built binary directly, trying the usual single/multi-config - # output locations (bin/ and build/, cf. setup.das locate_binary) — first - # existing wins. main_das = os.path.join(SCRIPT_DIR, "main.das") - for rel in ("bin/Release/daslang", "bin/daslang", "build/daslang", "build/bin/daslang"): - cand = os.path.join(REPO_ROOT, rel) - if os.path.exists(cand): - return [cand, main_das] - return [os.path.join(REPO_ROOT, "bin", "daslang"), main_das] # best-guess fallback + picked = _pick_binary(REPO_ROOT) + return [picked or os.path.join(REPO_ROOT, "bin", "daslang"), main_das] def _python_launcher() -> str: @@ -263,10 +322,9 @@ def _daslang_binary(repo_root: str) -> str: """The tree's own built daslang binary (cross-tree guard: a worktree's .mcp.json must point at that worktree's binary).""" exe = ".exe" if IS_WINDOWS else "" - for rel in ("bin/Release/daslang", "bin/daslang", "build/daslang", "build/bin/daslang"): - cand = os.path.join(repo_root, rel + exe) - if os.path.exists(cand): - return cand.replace(os.sep, "/") + picked = _pick_binary(repo_root) + if picked is not None: + return picked.replace(os.sep, "/") return os.path.join(repo_root, "bin", "daslang" + exe).replace(os.sep, "/")