diff --git a/docs/source/deployment-guide/deployment-guide-for-qwen3.8-qwen3.5-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-qwen3.8-qwen3.5-on-trtllm.md index 1fa1fbac82f5..d2f258e04f7c 100644 --- a/docs/source/deployment-guide/deployment-guide-for-qwen3.8-qwen3.5-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-qwen3.8-qwen3.5-on-trtllm.md @@ -5,9 +5,10 @@ This guide describes how to serve the Qwen3.8 MoE and Qwen3.5 MoE hybrid models with the TensorRT LLM PyTorch backend. It covers the following checkpoints: * **Qwen3.8-2.4T-A95B MoE FP8** — 2.4 trillion total parameters and 95 billion active parameters per token. +* **Qwen3.8-27B dense NVFP4** — 27 billion parameters with mixed NVFP4 and FP8 quantization. * **Qwen3.5-397B-A17B MoE NVFP4** — 397 billion total parameters and 17 billion active parameters per token. -The models share the `qwen3_5_moe_text` decoder architecture and use the same TensorRT LLM implementation, registered as `Qwen3_5MoeForCausalLM`. Both interleave three gated-delta-network (GDN) linear-attention layers with one grouped-query-attention (GQA) layer, and both use 512 routed experts with top-10 routing. +The models share the `qwen3_5_moe_text` decoder architecture and use the same TensorRT LLM implementation, registered as `Qwen3_5MoeForCausalLM`. Both interleave three gated-delta-network (GDN) linear-attention layers with one grouped-query-attention (GQA) layer, and both use 512 routed experts with top-10 routing. The dense Qwen3.8-27B checkpoint uses the `Qwen3_5ForConditionalGeneration` multimodal wrapper and the `Qwen3_5ForCausalLM` text decoder. Sharing an implementation does not make their deployment configurations interchangeable. Qwen3.8 MoE is substantially larger, uses an FP8 checkpoint, and is text-only. The NVIDIA Qwen3.5 MoE NVFP4 checkpoint uses a multimodal wrapper around the shared text decoder. Parallelism, quantization backends, cache sizing, and expert placement must be selected for the exact checkpoint. @@ -68,6 +69,7 @@ MTP3 performance results use a controlled accepted-draft count of 2.3. * [Qwen/Qwen3.8-2.4T-A95B-FP8](https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B-FP8) (FP8; the checkpoint the Qwen3.8 profiles in this guide target) * [Qwen/Qwen3.8-2.4T-A95B](https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B) (base, BF16) +* [RadixArk/Qwen3.8-27B-NVFP4](https://huggingface.co/RadixArk/Qwen3.8-27B-NVFP4) * [nvidia/Qwen3.5-397B-A17B-NVFP4](https://huggingface.co/nvidia/Qwen3.5-397B-A17B-NVFP4) * [Qwen/Qwen3.5-397B-A17B](https://huggingface.co/Qwen/Qwen3.5-397B-A17B) (base, BF16) diff --git a/tensorrt_llm/_torch/models/modeling_qwen3_5.py b/tensorrt_llm/_torch/models/modeling_qwen3_5.py index 10499969fac4..eab4bf6e61f3 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen3_5.py +++ b/tensorrt_llm/_torch/models/modeling_qwen3_5.py @@ -614,8 +614,11 @@ def _normalize_qwen35_quant_config_dict(model_config, keep_lm_head_quant=False): # promote W4A16_NVFP4 -> NVFP4 so the CuteDSL/TRTLLM GEMM path can # consume the checkpoint's packed FP4 weights and static input scales. dense_mlp_match = re.search(r"\.mlp\.(gate_proj|up_proj|down_proj)$", name) - if dense_mlp_match and cfg.quant_algo == QuantAlgo.W4A16_NVFP4: - if convert_to_nvfp4: + if dense_mlp_match and cfg.quant_algo in ( + QuantAlgo.W4A16_NVFP4, + QuantAlgo.NVFP4, + ): + if convert_to_nvfp4 and cfg.quant_algo == QuantAlgo.W4A16_NVFP4: cfg = cfg.model_copy(update={"quant_algo": QuantAlgo.NVFP4}) proj = dense_mlp_match.group(1) name = name[: -len(dense_mlp_match.group(0))] + f".mlp.mlp.{proj}" diff --git a/tensorrt_llm/_torch/models/modeling_qwen_image_bench.py b/tensorrt_llm/_torch/models/modeling_qwen_image_bench.py index aacb121f0d66..2a39a9663e39 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen_image_bench.py +++ b/tensorrt_llm/_torch/models/modeling_qwen_image_bench.py @@ -82,7 +82,7 @@ def load_weights( self.mm_encoder.load_weights(weights) qwen3_5_weight_mapper = Qwen3_5MoeHfWeightMapper() - qwen3_5_weight_mapper.init_model_and_config(self.llm, self.model_config) + qwen3_5_weight_mapper.init_model_and_config(self.llm, self.llm.model_config) filtered_weights = {k: v for k, v in weights.items() if not k.startswith("model.visual.")} self.llm.load_weights(filtered_weights, qwen3_5_weight_mapper) diff --git a/tests/unittest/_torch/modeling/test_modeling_qwen3_5_vl.py b/tests/unittest/_torch/modeling/test_modeling_qwen3_5_vl.py index 57537e004166..0060ca4e0715 100644 --- a/tests/unittest/_torch/modeling/test_modeling_qwen3_5_vl.py +++ b/tests/unittest/_torch/modeling/test_modeling_qwen3_5_vl.py @@ -5,8 +5,11 @@ import os from copy import deepcopy from pathlib import Path +from types import SimpleNamespace from typing import List, Optional +from unittest.mock import patch +import pytest import torch import transformers from test_modeling_multimodal import MultimodalScenario, TestModelingMultimodal @@ -19,7 +22,10 @@ from tensorrt_llm._torch.models.checkpoints.auto_mapper import AutoCheckpointMapper from tensorrt_llm._torch.models.checkpoints.hf.qwen3_5_weight_mapper import Qwen3_5MoeHfWeightMapper from tensorrt_llm._torch.models.modeling_auto import AutoModelForCausalLM -from tensorrt_llm._torch.models.modeling_qwen3_5 import _normalize_qwen35_vl_config +from tensorrt_llm._torch.models.modeling_qwen3_5 import ( + _normalize_qwen35_quant_config_dict, + _normalize_qwen35_vl_config, +) from tensorrt_llm._torch.pyexecutor.config_utils import ( extract_mamba_kv_cache_params, load_pretrained_config, @@ -27,6 +33,8 @@ from tensorrt_llm._torch.pyexecutor.model_loader import validate_and_set_mamba_ssm_cache_dtype from tensorrt_llm.inputs import ContentFormat from tensorrt_llm.inputs.registry import MULTIMODAL_PLACEHOLDER_REGISTRY +from tensorrt_llm.models.modeling_utils import QuantConfig +from tensorrt_llm.quantization.mode import QuantAlgo # Dense sibling of test_modeling_qwen3_5_vl_moe.py. The dense Qwen3.5-VL # (Qwen/Qwen3.5-27B, arch Qwen3_5ForConditionalGeneration, model_type qwen3_5) @@ -174,6 +182,76 @@ def test_qwen35_dense_vl_resolves_model_and_mapper(tmp_path: Path) -> None: ) +@pytest.mark.parametrize("sm_version", [90, 100, 103, 120, 121]) +def test_qwen35_dense_vl_normalizes_native_nvfp4_mlp_paths(sm_version: int) -> None: + model_config = SimpleNamespace( + pretrained_config=SimpleNamespace(num_hidden_layers=64), + quant_config_dict={ + f"model.language_model.layers.0.mlp.{proj}": QuantConfig(quant_algo=QuantAlgo.NVFP4) + for proj in ("gate_proj", "up_proj", "down_proj") + }, + ) + + with patch( + "tensorrt_llm._torch.models.modeling_qwen3_5.get_sm_version", + return_value=sm_version, + ): + _normalize_qwen35_quant_config_dict(model_config) + + assert set(model_config.quant_config_dict) == { + f"model.layers.0.mlp.mlp.{proj}" for proj in ("gate_proj", "up_proj", "down_proj") + } + assert all( + config.quant_algo == QuantAlgo.NVFP4 for config in model_config.quant_config_dict.values() + ) + + +@pytest.mark.parametrize( + ("sm_version", "expected_algo"), + [ + (90, QuantAlgo.W4A16_NVFP4), + (100, QuantAlgo.NVFP4), + (103, QuantAlgo.NVFP4), + (120, QuantAlgo.W4A16_NVFP4), + (121, QuantAlgo.W4A16_NVFP4), + ], +) +def test_qwen35_dense_vl_preserves_w4a16_nvfp4_behavior( + sm_version: int, expected_algo: QuantAlgo +) -> None: + model_config = SimpleNamespace( + pretrained_config=SimpleNamespace(num_hidden_layers=64), + quant_config_dict={ + "model.language_model.layers.0.mlp.gate_proj": QuantConfig( + quant_algo=QuantAlgo.W4A16_NVFP4 + ) + }, + ) + + with patch( + "tensorrt_llm._torch.models.modeling_qwen3_5.get_sm_version", + return_value=sm_version, + ): + _normalize_qwen35_quant_config_dict(model_config) + + config = model_config.quant_config_dict["model.layers.0.mlp.mlp.gate_proj"] + assert config.quant_algo == expected_algo + + +def test_qwen35_dense_vl_leaves_fp8_mlp_paths_unchanged() -> None: + name = "model.language_model.layers.0.mlp.gate_proj" + model_config = SimpleNamespace( + pretrained_config=SimpleNamespace(num_hidden_layers=64), + quant_config_dict={name: QuantConfig(quant_algo=QuantAlgo.FP8)}, + ) + + _normalize_qwen35_quant_config_dict(model_config) + + assert model_config.quant_config_dict == { + "model.layers.0.mlp.gate_proj": QuantConfig(quant_algo=QuantAlgo.FP8) + } + + def test_qwen35_dense_vl_disable_mm_encoder_skips_vision_tower( tmp_path: Path, ) -> None: diff --git a/tests/unittest/_torch/modeling/test_qwen_image_bench_modeling.py b/tests/unittest/_torch/modeling/test_qwen_image_bench_modeling.py index 739b287b26cc..4c06e51aeee2 100644 --- a/tests/unittest/_torch/modeling/test_qwen_image_bench_modeling.py +++ b/tests/unittest/_torch/modeling/test_qwen_image_bench_modeling.py @@ -3,6 +3,7 @@ import json from types import SimpleNamespace +from unittest.mock import Mock, patch import pytest import torch @@ -236,3 +237,22 @@ def test_qwen_image_bench_forwards_speculative_interface(): assert model.model is text_model assert model.lm_head is lm_head assert model.load_draft_weights("weights") is sentinel + + +def test_qwen_image_bench_mapper_uses_normalized_inner_model_config(): + inner_model_config = object() + llm = SimpleNamespace(model_config=inner_model_config, load_weights=Mock()) + model = QwenImageBenchModel.__new__(QwenImageBenchModel) + object.__setattr__(model, "llm", llm) + + with ( + patch( + "tensorrt_llm._torch.models.modeling_qwen_image_bench._is_mm_disagg", + return_value=True, + ), + patch.object(Qwen3_5MoeHfWeightMapper, "init_model_and_config") as init_mapper, + ): + model.load_weights({}) + + init_mapper.assert_called_once_with(llm, inner_model_config) + llm.load_weights.assert_called_once()