Update llama.cpp - ikawrakow#495

Thireus · web-flow · commit c2acf9088487 · 2025-06-06T10:55:45.000+01:00
diff --git a/src/llama.cpp b/src/llama.cpp
@@ -9841,7 +9841,7 @@ llm_expert_gating_func_type   gating_op,
     }
 
     ggml_tensor * par;
-    if (lctx.cparams.fused_moe_up_gate) {
+    if (lctx.cparams.fused_moe_up_gate && up_exps->type == gate_exps->type) {
         par = ggml_moe_up_gate(ctx, up_exps, gate_exps, cur, selected_experts, type_op == LLM_FFN_SILU ? GGML_UNARY_OP_SILU : GGML_UNARY_OP_GELU);
     } else {
         ggml_tensor * up = llm_build_lora_mm_id(lctx, ctx, up_exps, cur, selected_experts); // [n_ff, n_expert_used, n_tokens]

Original file line number	Diff line number	Diff line change
`@@ -9841,7 +9841,7 @@ llm_expert_gating_func_type gating_op,`
`9841`	`9841`	`}`
`9842`	`9842`
`9843`	`9843`	`ggml_tensor * par;`
`9844`		`- if (lctx.cparams.fused_moe_up_gate) {`
	`9844`	`+ if (lctx.cparams.fused_moe_up_gate && up_exps->type == gate_exps->type) {`
`9845`	`9845`	`par = ggml_moe_up_gate(ctx, up_exps, gate_exps, cur, selected_experts, type_op == LLM_FFN_SILU ? GGML_UNARY_OP_SILU : GGML_UNARY_OP_GELU);`
`9846`	`9846`	`} else {`
`9847`	`9847`	`ggml_tensor * up = llm_build_lora_mm_id(lctx, ctx, up_exps, cur, selected_experts); // [n_ff, n_expert_used, n_tokens]`