|
| 1 | +from __future__ import annotations |
| 2 | + |
| 3 | +from typing import Iterable, TYPE_CHECKING |
| 4 | + |
| 5 | +import torch |
| 6 | + |
| 7 | +if TYPE_CHECKING: |
| 8 | + from torch import Tensor |
| 9 | + |
| 10 | +from .base import ModelBase, TextModel, gguf, logger |
| 11 | + |
| 12 | + |
| 13 | +@ModelBase.register("MellumForCausalLM") |
| 14 | +class MellumModel(TextModel): |
| 15 | + model_arch = gguf.MODEL_ARCH.MELLUM |
| 16 | + |
| 17 | + def set_gguf_parameters(self): |
| 18 | + super().set_gguf_parameters() |
| 19 | + if (moe_intermediate_size := self.hparams.get("moe_intermediate_size")) is not None: |
| 20 | + self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size) |
| 21 | + logger.info(f"gguf: expert feed forward length = {moe_intermediate_size}") |
| 22 | + |
| 23 | + use_sliding_window = self.hparams.get("use_sliding_window") |
| 24 | + sliding_window = self.hparams.get("sliding_window") |
| 25 | + if (use_sliding_window is True or use_sliding_window is None) and sliding_window is not None: |
| 26 | + self.gguf_writer.add_sliding_window(sliding_window) |
| 27 | + logger.info(f"gguf: sliding window = {sliding_window}") |
| 28 | + self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in self.hparams["layer_types"]]) |
| 29 | + logger.info(f"gguf: sliding window pattern length = {len(self.hparams['layer_types'])}") |
| 30 | + |
| 31 | + _experts: list[dict[str, Tensor]] | None = None |
| 32 | + |
| 33 | + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: |
| 34 | + if name.find("experts") != -1: |
| 35 | + n_experts = self.find_hparam(["num_local_experts", "num_experts"]) |
| 36 | + assert bid is not None |
| 37 | + |
| 38 | + if self._experts is None: |
| 39 | + self._experts = [{} for _ in range(self.block_count)] |
| 40 | + |
| 41 | + self._experts[bid][name] = data_torch |
| 42 | + |
| 43 | + if len(self._experts[bid]) >= n_experts * 3: |
| 44 | + for w_name in ["down_proj", "gate_proj", "up_proj"]: |
| 45 | + datas: list[Tensor] = [] |
| 46 | + |
| 47 | + for xid in range(n_experts): |
| 48 | + ename = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight" |
| 49 | + datas.append(self._experts[bid][ename]) |
| 50 | + del self._experts[bid][ename] |
| 51 | + |
| 52 | + data_torch = torch.stack(datas, dim=0) |
| 53 | + |
| 54 | + merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight" |
| 55 | + |
| 56 | + yield from super().modify_tensors(data_torch, merged_name, bid) |
| 57 | + return |
| 58 | + else: |
| 59 | + return |
| 60 | + |
| 61 | + yield from super().modify_tensors(data_torch, name, bid) |
0 commit comments