diff --git a/README.md b/README.md index 8583f39c..0eedd7e0 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,33 @@ vllm serve unsloth/Qwen3.5-4B-MTP-GGUF:Q4_K_M \ For a GGUF without a `nextn` block, omit `--speculative-config`; the backbone loads normally without MTP. +### vLLM-Omni MiniMax-H3 + +The validated MiniMax-H3 path uses a non-pruned DiT GGUF while its text encoder +and VAEs stay on their Hugging Face weights: + +```python +from vllm_omni.entrypoints.omni import Omni + +omni = Omni( + model="/path/to/MiniMax-H3/FL2VA", + tensor_parallel_size=1, + text_encoder_tp_size=1, + diffusion_quantization_config={ + "method": "gguf", + "gguf_model": ( + "leejet/MiniMax-H3-GGUF/" + "minimax_h3_fl2va-Q4_K_M.gguf" + ), + }, +) +``` + +Use one partition per process. A single `gguf_model` cannot initialize H3's +combined FL2VA and Ref2VA DiTs. Pruned checkpoints containing +`adaln_t_table` are not yet supported; use a file without `pruned` in its +name. + ## Tested model coverage The plugin uses vLLM's model implementations and a generic GGUF weight @@ -94,6 +121,7 @@ starting points: | Vision-language | Qwen 3.6 | UD-IQ2_XXS backbone with BF16 projector | | Image generation | Z-Image-Turbo | Q4_0 | | Image generation | FLUX.2-klein | Q8_0 | +| Video generation | MiniMax-H3 | Q4_K_M | Other vLLM-supported architectures may work when their GGUF tensor names map to the corresponding Hugging Face model. A model appearing in vLLM's general diff --git a/tests/diffusion/test_diffusion_loader.py b/tests/diffusion/test_diffusion_loader.py index dd8b4835..552bede6 100644 --- a/tests/diffusion/test_diffusion_loader.py +++ b/tests/diffusion/test_diffusion_loader.py @@ -109,6 +109,26 @@ def load_weights(self, weights) -> set[str]: return loaded +class _MappedTextEncoderModel(_FakeModel): + """Model whose encoder loader maps checkpoint names to fused parameters.""" + + def __init__(self) -> None: + super().__init__() + self.text_encoder = nn.Linear(2, 2, bias=False) + + def load_weights(self, weights) -> set[str]: + mapped = ( + ( + "text_encoder.weight" + if name == "text_encoder.checkpoint_projection.weight" + else name, + tensor, + ) + for name, tensor in weights + ) + return super().load_weights(mapped) + + def _make_sources(): return [ DiffusionWeightSource(prefix="transformer.", subfolder="transformer"), @@ -282,6 +302,84 @@ def hf_fn(source: DiffusionWeightSource): assert torch.allclose(model.vae.weight, torch.full((2, 2), 4.0)) +def test_load_gguf_rejects_multiple_matching_component_sources( + monkeypatch: pytest.MonkeyPatch, +): + """A single GGUF must not initialize two partition-specific DiTs.""" + model = _FakeModel() + + class _Adapter: + def weights_iterator(self): + raise AssertionError( + "weights must not be read after source validation fails" + ) + + import vllm_gguf_plugin.weights_adapter.diffusion.loader as _loader_mod + + monkeypatch.setattr( + _loader_mod, "resolve_gguf_model_path", lambda **kw: "dummy.gguf" + ) + monkeypatch.setattr( + _loader_mod, "get_diffusion_gguf_adapter", lambda *a, **kw: _Adapter() + ) + + sources = [ + DiffusionWeightSource(prefix="transformer.", subfolder="transformer"), + DiffusionWeightSource(prefix="transformers_ref.", subfolder="transformer"), + ] + with pytest.raises(ValueError, match="must match exactly one weight source"): + load_diffusion_gguf_weights( + gguf_model="dummy.gguf", + model=model, + model_class_name="MiniMaxH3Pipeline", + model_type=None, + sources=sources, + hf_weights_fn=lambda _source: iter(()), + ) + + +def test_hf_component_weights_reach_model_checkpoint_mapper( + monkeypatch: pytest.MonkeyPatch, +): + """HF component keys may need model-specific mapping before they are loadable.""" + model = _MappedTextEncoderModel() + + class _Adapter: + def weights_iterator(self): + yield "weight", torch.ones((2, 2)) + yield "bias", torch.zeros(2) + + import vllm_gguf_plugin.weights_adapter.diffusion.loader as _loader_mod + + monkeypatch.setattr( + _loader_mod, "resolve_gguf_model_path", lambda **kw: "dummy.gguf" + ) + monkeypatch.setattr( + _loader_mod, "get_diffusion_gguf_adapter", lambda *a, **kw: _Adapter() + ) + + sources = [ + DiffusionWeightSource(prefix="transformer.", subfolder="transformer"), + DiffusionWeightSource(prefix="text_encoder.", subfolder="text_encoder"), + ] + + def hf_fn(source: DiffusionWeightSource): + if source.prefix == "text_encoder.": + yield "text_encoder.checkpoint_projection.weight", torch.full((2, 2), 5.0) + + loaded = load_diffusion_gguf_weights( + gguf_model="dummy.gguf", + model=model, + model_class_name=None, + model_type=None, + sources=sources, + hf_weights_fn=hf_fn, + ) + + assert "text_encoder.weight" in loaded + assert torch.allclose(model.text_encoder.weight, torch.full((2, 2), 5.0)) + + def test_load_gguf_skips_hf_when_complete(monkeypatch: pytest.MonkeyPatch): """No HF fallback when GGUF covers all transformer weights.""" model = _FakeModel() diff --git a/tests/diffusion/test_minimax_h3_adapter.py b/tests/diffusion/test_minimax_h3_adapter.py new file mode 100644 index 00000000..013c0b74 --- /dev/null +++ b/tests/diffusion/test_minimax_h3_adapter.py @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for the MiniMax-H3 diffusion GGUF adapter.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +import torch + +from vllm_gguf_plugin.weights_adapter.diffusion import ( + MiniMaxH3DiffusionGGUFAdapter, + get_diffusion_gguf_adapter, +) + +pytestmark = [pytest.mark.cpu] + + +def test_minimax_h3_adapter_selected_for_pipeline(): + adapter = get_diffusion_gguf_adapter( + "dummy.gguf", + model_class_name="MiniMaxH3Pipeline", + model_type=None, + ) + assert isinstance(adapter, MiniMaxH3DiffusionGGUFAdapter) + assert adapter.unquantized_modules == ("text_encoder",) + + +@pytest.mark.parametrize("model_type", ["minimax_h3", "minimax-h3", "minimaxh3"]) +def test_minimax_h3_adapter_selected_for_model_type(model_type: str): + assert MiniMaxH3DiffusionGGUFAdapter.is_compatible( + model_class_name=None, + model_type=model_type, + ) + + +def test_quantized_qkv_keeps_converter_fused_layout( + monkeypatch: pytest.MonkeyPatch, +): + import vllm_gguf_plugin.weights_adapter.diffusion.minimax_h3 as h3_module + + qweight = torch.arange(12).reshape(3, 4) + monkeypatch.setattr( + h3_module, + "gguf_quant_weights_iterator", + lambda _path: iter( + [ + ("blocks.0.attn.qkv_proj.qweight_type", torch.tensor(10)), + ("blocks.0.attn.qkv_proj.qweight", qweight), + ] + ), + ) + + weights = dict(MiniMaxH3DiffusionGGUFAdapter("dummy.gguf").weights_iterator()) + + assert weights["blocks.0.attn.qkv_proj.qweight_type"].item() == 10 + assert weights["blocks.0.attn.qkv_proj.qweight"] is qweight + + +def test_dense_qkv_is_restored_to_grouped_checkpoint_layout( + monkeypatch: pytest.MonkeyPatch, +): + import vllm_gguf_plugin.weights_adapter.diffusion.minimax_h3 as h3_module + + projection_rows = 56 * 128 + fused = torch.arange(3 * projection_rows).reshape(-1, 1) + monkeypatch.setattr( + h3_module, + "gguf_quant_weights_iterator", + lambda _path: iter([("blocks.0.attn.qkv_proj.weight", fused)]), + ) + + weights = dict(MiniMaxH3DiffusionGGUFAdapter("dummy.gguf").weights_iterator()) + grouped = weights["blocks.0.attn.qkv_proj.weight"].reshape(56, 3 * 128) + + assert torch.equal(grouped[0, :128], fused[:128, 0]) + assert torch.equal( + grouped[0, 128:256], fused[projection_rows : projection_rows + 128, 0] + ) + assert torch.equal( + grouped[0, 256:], fused[2 * projection_rows : 2 * projection_rows + 128, 0] + ) + + +def test_pruned_adaln_layout_is_rejected_early(monkeypatch: pytest.MonkeyPatch): + import vllm_gguf_plugin.weights_adapter.diffusion.minimax_h3 as h3_module + + reader = SimpleNamespace(tensors=[SimpleNamespace(name="adaln_t_table")]) + monkeypatch.setattr(h3_module.gguf, "GGUFReader", lambda _path: reader) + + adapter = MiniMaxH3DiffusionGGUFAdapter("pruned.gguf") + with pytest.raises(ValueError, match="Unsupported MiniMax-H3 GGUF schema"): + adapter.unquantized_module_names() + + +def test_incomplete_time_embedder_layout_is_rejected_early( + monkeypatch: pytest.MonkeyPatch, +): + import vllm_gguf_plugin.weights_adapter.diffusion.minimax_h3 as h3_module + + reader = SimpleNamespace( + tensors=[ + SimpleNamespace( + name="time_embedder.proj_in.weight", + tensor_type=SimpleNamespace(name="F32"), + ) + ] + ) + monkeypatch.setattr(h3_module.gguf, "GGUFReader", lambda _path: reader) + + adapter = MiniMaxH3DiffusionGGUFAdapter("incomplete.gguf") + with pytest.raises(ValueError, match="Unsupported MiniMax-H3 GGUF schema"): + adapter.unquantized_module_names() diff --git a/vllm_gguf_plugin/weights_adapter/diffusion/__init__.py b/vllm_gguf_plugin/weights_adapter/diffusion/__init__.py index 7c81509d..bbfb588c 100644 --- a/vllm_gguf_plugin/weights_adapter/diffusion/__init__.py +++ b/vllm_gguf_plugin/weights_adapter/diffusion/__init__.py @@ -9,10 +9,12 @@ load_diffusion_gguf_weights, resolve_gguf_model_path, ) +from .minimax_h3 import MiniMaxH3DiffusionGGUFAdapter from .qwen_image import QwenImageDiffusionGGUFAdapter from .z_image import ZImageDiffusionGGUFAdapter _ADAPTER_CLASSES: list[type[DiffusionGGUFAdapter]] = [ + MiniMaxH3DiffusionGGUFAdapter, QwenImageDiffusionGGUFAdapter, ZImageDiffusionGGUFAdapter, Flux2KleinDiffusionGGUFAdapter, @@ -40,6 +42,7 @@ def get_diffusion_gguf_adapter( "DiffusionWeightSource", "Flux2KleinDiffusionGGUFAdapter", "MappedTensor", + "MiniMaxH3DiffusionGGUFAdapter", "QwenImageDiffusionGGUFAdapter", "ZImageDiffusionGGUFAdapter", "get_diffusion_gguf_adapter", diff --git a/vllm_gguf_plugin/weights_adapter/diffusion/loader.py b/vllm_gguf_plugin/weights_adapter/diffusion/loader.py index b3f69d32..5f0d4a6f 100644 --- a/vllm_gguf_plugin/weights_adapter/diffusion/loader.py +++ b/vllm_gguf_plugin/weights_adapter/diffusion/loader.py @@ -168,15 +168,6 @@ def _gguf_weights_for_loadable_names( ) -def _hf_weights_for_loadable_names( - weights: Iterable[tuple[str, torch.Tensor]], - loadable_names: set[str], -) -> Iterable[tuple[str, torch.Tensor]]: - for name, tensor in weights: - if name in loadable_names: - yield name, tensor - - def load_diffusion_gguf_weights( gguf_model: str, model: nn.Module, @@ -223,9 +214,18 @@ def load_diffusion_gguf_weights( adapter = get_diffusion_gguf_adapter(gguf_file, model_class_name, model_type) loaded: set[str] = set() loadable_names: set[str] | None = None + gguf_sources = [source for source in sources if _is_gguf_source(source, adapter)] + if len(gguf_sources) != 1: + matched_prefixes = ", ".join(source.prefix for source in gguf_sources) or "none" + raise ValueError( + f"{type(adapter).__name__} must match exactly one weight source for a " + f"single gguf_model, but matched {len(gguf_sources)} " + f"({matched_prefixes})." + ) + gguf_source = gguf_sources[0] for source in sources: - if _is_gguf_source(source, adapter): + if source is gguf_source: loadable_names = loadable_names or _get_loadable_names(model) gguf_iter = ( (source.prefix + name, tensor) @@ -235,16 +235,13 @@ def load_diffusion_gguf_weights( _gguf_weights_for_loadable_names(gguf_iter, loadable_names) ) else: - # Non-transformer components always load from HF. - loadable_names = loadable_names or _get_loadable_names(model) + # Non-transformer components always load from HF. Preserve their + # checkpoint names because model loaders may map or fuse them. loaded |= model.load_weights( - _hf_weights_for_loadable_names( - ( - (name, tensor) - for name, tensor in hf_weights_fn(source) - if not source.prefix or name.startswith(source.prefix) - ), - loadable_names, + ( + (name, tensor) + for name, tensor in hf_weights_fn(source) + if not source.prefix or name.startswith(source.prefix) ) ) diff --git a/vllm_gguf_plugin/weights_adapter/diffusion/minimax_h3.py b/vllm_gguf_plugin/weights_adapter/diffusion/minimax_h3.py new file mode 100644 index 00000000..531d1819 --- /dev/null +++ b/vllm_gguf_plugin/weights_adapter/diffusion/minimax_h3.py @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +from collections.abc import Iterable + +import gguf +import torch + +from .base import ( + UNQUANTIZED_GGUF_TYPE_NAMES, + DiffusionGGUFAdapter, + gguf_quant_weights_iterator, +) + +_NUM_QUERY_GROUPS = 56 +_HEAD_DIM = 128 + + +def _fused_qkv_to_grouped(weight: torch.Tensor) -> torch.Tensor: + """Restore H3's grouped checkpoint layout for its dense weight loader. + + H3 GGUF converters store QKV as contiguous Q, K, V rows. Quantized + ``qweight`` tensors enter vLLM's QKV loader directly in that layout. Dense + ``weight`` tensors, however, pass through MiniMaxH3DiTModel.load_weights, + which expects the original per-head ``[q, k, v]`` checkpoint layout and + performs the grouped-to-fused conversion itself. + """ + rows_per_projection = _NUM_QUERY_GROUPS * _HEAD_DIM + expected_rows = 3 * rows_per_projection + if weight.shape[0] != expected_rows: + raise ValueError( + "MiniMax-H3 GGUF QKV tensor has incompatible output dimension: " + f"got {tuple(weight.shape)}, expected first dimension {expected_rows}." + ) + + rest_shape = weight.shape[1:] + q, k, v = weight.split(rows_per_projection, dim=0) + return torch.cat( + [ + q.reshape(_NUM_QUERY_GROUPS, _HEAD_DIM, *rest_shape), + k.reshape(_NUM_QUERY_GROUPS, _HEAD_DIM, *rest_shape), + v.reshape(_NUM_QUERY_GROUPS, _HEAD_DIM, *rest_shape), + ], + dim=1, + ).reshape(expected_rows, *rest_shape) + + +class MiniMaxH3DiffusionGGUFAdapter(DiffusionGGUFAdapter): + """GGUF adapter for MiniMax-H3 single-partition pipelines.""" + + # H3 forwards the diffusion quantization config to its Qwen3-VL encoder. + # The DiT GGUF does not contain encoder tensors, so keep that component on + # its Hugging Face weights. + unquantized_modules = ("text_encoder",) + + @staticmethod + def is_compatible( + model_class_name: str | None, + model_type: str | None, + ) -> bool: + if model_class_name and model_class_name.startswith("MiniMaxH3"): + return True + if not model_type: + return False + normalized_model_type = model_type.lower().replace("-", "_") + return normalized_model_type in {"minimax_h3", "minimaxh3"} + + def unquantized_weight_names(self) -> Iterable[str]: + reader = gguf.GGUFReader(self.gguf_file) + tensor_names = {tensor.name for tensor in reader.tensors} + required_time_embedder_names = { + "time_embedder.proj_in.weight", + "time_embedder.proj_in.bias", + "time_embedder.proj_out.weight", + "time_embedder.proj_out.bias", + } + if "adaln_t_table" in tensor_names or not required_time_embedder_names.issubset( + tensor_names + ): + raise ValueError( + "Unsupported MiniMax-H3 GGUF schema. The current vLLM-Omni " + "MiniMax-H3 architecture requires the complete time_embedder " + "weights and does not support adaln_t_table. Use a non-pruned " + "FL2VA or Ref2VA GGUF checkpoint." + ) + + for tensor in reader.tensors: + if tensor.tensor_type.name in UNQUANTIZED_GGUF_TYPE_NAMES: + yield tensor.name + + def weights_iterator(self) -> Iterable[tuple[str, torch.Tensor]]: + for name, weight in gguf_quant_weights_iterator(self.gguf_file): + if name.endswith(".attn.qkv_proj.weight"): + weight = _fused_qkv_to_grouped(weight) + yield name, weight