Will GLM4MoE be supported?

[LostRuins]

Will Glm4MoE be supported? Currently it does not seem to work.

[sophosympatheia]

I am also wondering this.

[MrJackSpade]

Claude wrote this patch, so effectiveness is up for debate, but it does produce a functional model for GLM4-MOE. At least, it produces a model that can be converted to GGUF, quantized, and executes coherently.

From 96c07579cee2835e281db97a4252ee05067613e6 Mon Sep 17 00:00:00 2001
From: Shork <Shork@shork.shork>
Date: Sun, 4 Jan 2026 13:06:29 -0700
Subject: [PATCH] GLMMOE merge

---
 mergekit/_data/architectures/glm4_moe.json | 118 +++++++++++++++++++++
 mergekit/architecture/json_definitions.py  |  46 ++++++--
 mergekit/merge.py                          | 117 ++++++++++++++++++++
 3 files changed, 274 insertions(+), 7 deletions(-)
 create mode 100644 mergekit/_data/architectures/glm4_moe.json

diff --git a/mergekit/_data/architectures/glm4_moe.json b/mergekit/_data/architectures/glm4_moe.json
new file mode 100644
index 0000000..36b2b89
--- /dev/null
+++ b/mergekit/_data/architectures/glm4_moe.json
@@ -0,0 +1,118 @@
+{
+    "model_type": "glm4_moe",
+    "architectures": [
+        "Glm4MoeForCausalLM"
+    ],
+    "pre_weights": [
+        {
+            "name": "model.embed_tokens.weight",
+            "is_embed": true
+        }
+    ],
+    "post_weights": [
+        {
+            "name": "model.norm.weight"
+        },
+        {
+            "name": "lm_head.weight",
+            "is_embed": true,
+            "optional": true,
+            "tied_names": [
+                "model.embed_tokens.weight"
+            ]
+        }
+    ],
+    "num_layers_config_key": "num_hidden_layers",
+    "override_num_layers_config_key": "num_layers_with_moe",
+    "num_experts_config_key": "n_routed_experts",
+    "layer_templates": {
+        "weights": [
+            {
+                "name": "model.layers.${layer_index}.input_layernorm.weight"
+            },
+            {
+                "name": "model.layers.${layer_index}.post_attention_layernorm.weight"
+            },
+            {
+                "name": "model.layers.${layer_index}.self_attn.q_proj.weight"
+            },
+            {
+                "name": "model.layers.${layer_index}.self_attn.q_proj.bias",
+                "optional": true
+            },
+            {
+                "name": "model.layers.${layer_index}.self_attn.k_proj.weight"
+            },
+            {
+                "name": "model.layers.${layer_index}.self_attn.k_proj.bias",
+                "optional": true
+            },
+            {
+                "name": "model.layers.${layer_index}.self_attn.v_proj.weight"
+            },
+            {
+                "name": "model.layers.${layer_index}.self_attn.v_proj.bias",
+                "optional": true
+            },
+            {
+                "name": "model.layers.${layer_index}.self_attn.o_proj.weight"
+            },
+            {
+                "name": "model.layers.${layer_index}.self_attn.q_norm.weight",
+                "optional": true
+            },
+            {
+                "name": "model.layers.${layer_index}.self_attn.k_norm.weight",
+                "optional": true
+            },
+            {
+                "name": "model.layers.${layer_index}.mlp.gate_proj.weight",
+                "optional": true
+            },
+            {
+                "name": "model.layers.${layer_index}.mlp.up_proj.weight",
+                "optional": true
+            },
+            {
+                "name": "model.layers.${layer_index}.mlp.down_proj.weight",
+                "optional": true
+            },
+            {
+                "name": "model.layers.${layer_index}.mlp.gate.weight",
+                "optional": true
+            },
+            {
+                "name": "model.layers.${layer_index}.mlp.gate.e_score_correction_bias",
+                "optional": true
+            },
+            {
+                "name": "model.layers.${layer_index}.mlp.shared_experts.gate_proj.weight",
+                "optional": true
+            },
+            {
+                "name": "model.layers.${layer_index}.mlp.shared_experts.up_proj.weight",
+                "optional": true
+            },
+            {
+                "name": "model.layers.${layer_index}.mlp.shared_experts.down_proj.weight",
+                "optional": true
+            },
+            {
+                "name": "model.layers.${layer_index}.mlp.experts.${expert_index}.gate_proj.weight",
+                "optional": true
+            },
+            {
+                "name": "model.layers.${layer_index}.mlp.experts.${expert_index}.up_proj.weight",
+                "optional": true
+            },
+            {
+                "name": "model.layers.${layer_index}.mlp.experts.${expert_index}.down_proj.weight",
+                "optional": true
+            },
+            {
+                "name": "model.layers.${layer_index}.shared_head.norm.weight",
+                "optional": true
+            }
+        ]
+    }
+}
diff --git a/mergekit/architecture/json_definitions.py b/mergekit/architecture/json_definitions.py
index 23d1010..3faea04 100644
--- a/mergekit/architecture/json_definitions.py
+++ b/mergekit/architecture/json_definitions.py
@@ -32,6 +32,8 @@ class JsonModuleArchDef(BaseModel, frozen=True):
     post_weights: List[WeightInfo]
     num_layers_config_key: Optional[str] = None
     override_num_layers: Optional[int] = None
+    num_experts_config_key: Optional[str] = None
+    override_num_layers_config_key: Optional[str] = None
 
 
 class JsonModuleArchitecture(ModuleArchitecture, BaseModel, frozen=True):
@@ -43,6 +45,7 @@ class JsonModuleArchitecture(ModuleArchitecture, BaseModel, frozen=True):
         item: WeightInfo,
         config: PretrainedConfig,
         layer_idx: Optional[int] = None,
+        expert_idx: Optional[int] = None,
     ) -> WeightInfo:
         num_layers = self.num_layers(config)
 
@@ -50,12 +53,12 @@ class JsonModuleArchitecture(ModuleArchitecture, BaseModel, frozen=True):
         for key in obj_dict:
             if isinstance(obj_dict[key], str):
                 obj_dict[key] = _template_substitution(
-                    obj_dict[key], num_layers, layer_idx
+                    obj_dict[key], num_layers, layer_idx, expert_idx
                 )
             elif isinstance(obj_dict[key], list):
                 obj_dict[key] = [
                     (
-                        _template_substitution(s, num_layers, layer_idx)
+                        _template_substitution(s, num_layers, layer_idx, expert_idx)
                         if isinstance(s, str)
                         else s
                     )
@@ -63,6 +66,12 @@ class JsonModuleArchitecture(ModuleArchitecture, BaseModel, frozen=True):
                 ]
         return type(item).model_validate(obj_dict)
 
+    def num_experts(self, config: PretrainedConfig) -> Optional[int]:
+        """Get the number of experts from the config, if applicable."""
+        if self.definition.num_experts_config_key is None:
+            return None
+        return getattr(config, self.definition.num_experts_config_key, None)
+
     def name(self) -> str:
         return self.definition.expected_model_type
 
@@ -74,10 +83,24 @@ class JsonModuleArchitecture(ModuleArchitecture, BaseModel, frozen=True):
     def layer_weights(
         self, index: int, config: PretrainedConfig
     ) -> Optional[List[WeightInfo]]:
-        return [
-            self._substitute(wi, config=config, layer_idx=index)
-            for wi in self.definition.layer_templates.weights
-        ]
+        result = []
+        num_experts = self.num_experts(config)
+        
+        for wi in self.definition.layer_templates.weights:
+            # Check if this template uses expert_index
+            name_str = wi.name if isinstance(wi.name, str) else ""
+            if "${expert_index}" in name_str or "{expert_index}" in name_str:
+                if num_experts is not None:
+                    # Expand for each expert
+                    for expert_idx in range(num_experts):
+                        result.append(self._substitute(
+                            wi, config=config, layer_idx=index, expert_idx=expert_idx
+                        ))
+                # If no num_experts defined, skip this weight template
+            else:
+                result.append(self._substitute(wi, config=config, layer_idx=index))
+        
+        return result
 
     def post_weights(self, config: PretrainedConfig) -> List[WeightInfo]:
         return [
@@ -113,7 +136,7 @@ class TemplateWithArithmetic(string.Template):
 
 
 def _template_substitution(
-    template: str, num_layers: int, layer_idx: Optional[int] = None
+    template: str, num_layers: int, layer_idx: Optional[int] = None, expert_idx: Optional[int] = None
 ) -> str:
     if "{" not in template:
         return template
@@ -133,6 +156,15 @@ def _template_substitution(
             }
         )
 
+    if expert_idx is not None:
+        substitutions.update(
+            {
+                "expert_index": expert_idx,
+                "expert_index+1": expert_idx + 1,
+                "expert_index-1": expert_idx - 1,
+            }
+        )
+
     return TemplateWithArithmetic(template).substitute(substitutions)
 
 
diff --git a/mergekit/merge.py b/mergekit/merge.py
index c830048..a60df2d 100644
--- a/mergekit/merge.py
+++ b/mergekit/merge.py
@@ -139,6 +139,8 @@ def run_merge(
         options=options,
     )
 
+    _merge_mtp_file(merge_config, out_path, options)
+
     if getattr(arch_info, "post_fill_parameters", False):
         from mergekit.scripts.fill_missing_params import copy_and_fill_missing_params
 
@@ -234,6 +236,121 @@ def _copy_tagalong_files(
     return
 
 
+def _merge_mtp_file(
+    merge_config: MergeConfiguration,
+    out_path: str,
+    options: MergeOptions,
+):
+    """Merge MTP safetensors files if present in source models."""
+    import safetensors
+    import safetensors.torch
+
+    from mergekit.merge_methods.slerp import slerp
+
+    # Get all source models
+    models = merge_config.referenced_models()
+    if not models or len(models) < 2:
+        return
+
+    # Check which models have mtp.safetensors
+    mtp_paths = []
+    for model in models:
+        local_path = model.merged(
+            cache_dir=options.lora_merge_cache,
+            trust_remote_code=options.trust_remote_code,
+            lora_merge_dtype=options.lora_merge_dtype,
+        ).local_path(cache_dir=options.transformers_cache)
+
+        mtp_path = os.path.join(local_path, "mtp.safetensors")
+        if os.path.exists(mtp_path):
+            mtp_paths.append(mtp_path)
+        else:
+            LOG.info(f"No mtp.safetensors found in {model}")
+
+    # Need at least 2 models with MTP files for SLERP
+    if len(mtp_paths) < 2:
+        if len(mtp_paths) == 1:
+            LOG.warning("Only one model has mtp.safetensors - copying as-is")
+            shutil.copy(mtp_paths[0], os.path.join(out_path, "mtp.safetensors"))
+        return
+
+    if len(mtp_paths) > 2:
+        LOG.warning(
+            f"Found {len(mtp_paths)} mtp.safetensors files, but SLERP only supports 2 models - using first 2"
+        )
+        mtp_paths = mtp_paths[:2]
+
+    # Get SLERP parameter
+    t = merge_config.parameters.get("t", 0.5)
+
+    # Load MTP tensors from both models
+    LOG.info(f"Merging mtp.safetensors with SLERP (t={t})")
+
+    with safetensors.safe_open(mtp_paths[0], framework="pt", device="cpu") as f0:
+        with safetensors.safe_open(mtp_paths[1], framework="pt", device="cpu") as f1:
+            # Get all tensor names from both files
+            keys0 = set(f0.keys())
+            keys1 = set(f1.keys())
+
+            # Find common and unique tensors
+            common_keys = keys0 & keys1
+            only_in_0 = keys0 - keys1
+            only_in_1 = keys1 - keys0
+
+            if only_in_0:
+                LOG.info(f"Tensors only in first model: {len(only_in_0)}")
+            if only_in_1:
+                LOG.info(f"Tensors only in second model: {len(only_in_1)}")
+
+            # Merge tensors
+            merged_tensors = {}
+
+            # Merge common tensors with SLERP
+            for key in common_keys:
+                tensor0 = f0.get_tensor(key)
+                tensor1 = f1.get_tensor(key)
+
+                # Apply SLERP interpolation
+                merged = slerp(t, tensor0, tensor1)
+                merged_tensors[key] = merged.to(tensor0.dtype).contiguous()
+
+            # Handle tensors only in second model - copy as-is
+            for key in only_in_1:
+                tensor = f1.get_tensor(key)
+                merged_tensors[key] = tensor.contiguous()
+
+    # Save merged MTP file
+    output_mtp = os.path.join(out_path, "mtp.safetensors")
+    LOG.info(f"Saving merged mtp.safetensors with {len(merged_tensors)} tensors")
+    safetensors.torch.save_file(merged_tensors, output_mtp, metadata={"format": "pt"})
+
+    # Update model.safetensors.index.json to include MTP tensors
+    index_path = os.path.join(out_path, "model.safetensors.index.json")
+    if os.path.exists(index_path):
+        import json
+
+        with open(index_path, "r", encoding="utf-8") as f:
+            index_data = json.load(f)
+
+        # Add MTP tensor mappings to weight_map
+        for key in merged_tensors.keys():
+            index_data["weight_map"][key] = "mtp.safetensors"
+
+        # Update total_size if present
+        mtp_size = sum(
+            t.numel() * t.element_size() for t in merged_tensors.values()
+        )
+        if "metadata" in index_data and "total_size" in index_data["metadata"]:
+            index_data["metadata"]["total_size"] += mtp_size
+
+        with open(index_path, "w", encoding="utf-8") as f:
+            json.dump(index_data, f, indent=2)
+
+        LOG.info(f"Updated model.safetensors.index.json with {len(merged_tensors)} MTP tensor mappings")
+    else:
+        LOG.warning("No model.safetensors.index.json found - MTP tensors not indexed")
+
+
 def _copy_tokenizer(
     merge_config: MergeConfiguration, out_path: str, options: MergeOptions
 ):
-- 
2.46.0.windows.1