feat(qwen3.8): add Qwen3.8-27B model family

benchmarks/performance/release.yaml

@@ -758,6 +758,16 @@ entries:
       runner: hf-transformers
       mode: torch-compile
       compile_scope: model.forward
+  - id: qwen3_8.generate
+    family: qwen3_8
+    operation: generate
+    model: qwen38-27b
+    workload:
+      testcase: qwen38-27b
+    baseline:
+      runner: hf-transformers
+      mode: torch-compile
+      compile_scope: model.forward
   - id: qwen3_omni.generate_audio
     family: qwen3_omni
     operation: generate_audio

python/tensorrt_model_connect/families/qwen3_8/MODEL.toml

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+id = "qwen3_8"
+plugin = "qwen3_8"
+module = "plugin"
+aliases = [
+  "qwen38",
+  "qwen3.8",
+  "qwen3_8",
+]
+prefixes = [
+  "qwen3_8",
+]
+# Qwen3.8 checkpoints reuse the Qwen3.5 `model_type`/`architectures` strings, so
+# alias matching alone cannot separate the two families. Declaring the shared
+# architecture pattern routes config-shaped lookups through this family first;
+# `Qwen38Plugin.matches_config` then claims only genuine Qwen3.8 checkpoints and
+# leaves Qwen3.5 to fall through to its own family.
+architecture_patterns = [
+  "qwen3_5forconditionalgeneration",
+]
+debug_runner = "debug_runner.py|runner_from_bundle"
+debug_runtime_strategies = [
+  "qwen3_8_hybrid_mamba_attention",
+]

python/tensorrt_model_connect/families/qwen3_8/__init__.py

@@ -0,0 +1,62 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import importlib
+import sys
+import types
+from typing import Any
+
+_plugin = None
+
+
+def _load_plugin_module():
+    global _plugin
+    if _plugin is None:
+        _plugin = importlib.import_module(f"{__name__}.plugin")
+        globals().update({
+            _name: _value
+            for _name, _value in vars(_plugin).items()
+            if not _name.startswith("__")
+        })
+    return _plugin
+
+
+def __getattr__(name: str) -> Any:
+    if name.startswith("__"):
+        raise AttributeError(name)
+    plugin_module = _load_plugin_module()
+    if name == "plugin":
+        return getattr(plugin_module, "plugin")
+    try:
+        return getattr(plugin_module, name)
+    except AttributeError:
+        raise AttributeError(name) from None
+
+
+def __dir__() -> list[str]:
+    plugin_module = _load_plugin_module()
+    return sorted(set(globals()) | {
+        _name for _name in vars(plugin_module) if not _name.startswith("__")
+    })
+
+
+class _FamilyModule(types.ModuleType):
+    def __setattr__(self, name, value):
+        # Importlib publishes a directly imported plugin submodule on its parent.
+        # Keep the public package attribute bound to the FamilyPlugin instance.
+        if name == "plugin" and isinstance(value, types.ModuleType):
+            super().__setattr__("_plugin", value)
+            super().__setattr__("plugin", value.plugin)
+            return
+        super().__setattr__(name, value)
+        if (
+            not name.startswith("__")
+            and name not in {"_plugin", "plugin"}
+            and not isinstance(value, types.ModuleType)
+        ):
+            setattr(_load_plugin_module(), name, value)
+
+
+sys.modules[__name__].__class__ = _FamilyModule

python/tensorrt_model_connect/families/qwen3_8/checkpoint_mapper.py

@@ -0,0 +1,199 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""1:1 port of standard_checkpoint_mapper.cpp + tensor_math.cpp to Python.
+
+Loads HF safetensors and maps keys to the flat weight dict expected by
+standard_decoder_builder.py. All projections are transposed from HF
+[out, in] layout to [in, out] for TRT matmul.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import numpy as np
+
+# Register bfloat16 dtype with numpy (needed for safetensors without torch).
+try:
+    import ml_dtypes  # noqa: F401
+except ImportError:
+    pass
+
+from safetensors import safe_open
+
+
+def _target_np_dtype(precision: str) -> np.dtype:
+    """Map precision string to numpy dtype for weight storage."""
+    if precision in ("fp16", "bf16"):
+        return np.float16
+    return np.float32
+
+
+def _transpose_2d(arr: np.ndarray, name: str, precision: str = "fp32") -> np.ndarray:
+    """Transpose [rows, cols] -> [cols, rows] in C-contiguous target dtype."""
+    if arr.ndim != 2:
+        raise ValueError(f"Expected rank-2 tensor for transpose: {name}")
+    return np.ascontiguousarray(arr.T, dtype=_target_np_dtype(precision))
+
+
+class WeightDict(dict):
+    """A dict mapping logical weight names to flat float32 arrays.
+
+    Keys follow the convention used by standard_decoder_builder.py:
+      - embedding: [vocab, hidden]
+      - layer.{i}.input_norm: [hidden]
+      - layer.{i}.w_q: [hidden, attention_size]
+      - layer.{i}.w_k: [hidden, kv_attention_size]
+      - layer.{i}.w_v: [hidden, kv_attention_size]
+      - layer.{i}.q_bias: [attention_size]       (optional)
+      - layer.{i}.k_bias: [kv_attention_size]    (optional)
+      - layer.{i}.v_bias: [kv_attention_size]    (optional)
+      - layer.{i}.q_norm: [attention_size]        (optional)
+      - layer.{i}.k_norm: [kv_attention_size]     (optional)
+      - layer.{i}.w_o: [attention_size, hidden]
+      - layer.{i}.post_attn_norm: [hidden]
+      - layer.{i}.w_gate: [hidden, mlp_size]
+      - layer.{i}.w_up: [hidden, mlp_size]
+      - layer.{i}.w_down: [mlp_size, hidden]
+      - final_norm: [hidden]
+      - w_out: [hidden, vocab]
+    """
+
+
+# ---------------------------------------------------------------------------
+# Safetensors I/O helpers
+# ---------------------------------------------------------------------------
+
+
+def _detect_framework() -> str:
+    """Use 'torch' if available (handles BF16 natively), else 'numpy'."""
+    try:
+        import torch  # noqa: F401
+
+        return "torch"
+    except ImportError:
+        return "numpy"
+
+
+class _TorchBinReader:
+    """Adapter that wraps a pytorch .bin state dict with the safetensors reader
+    interface (keys() / get_tensor())."""
+
+    def __init__(self, path: Path):
+        import torch
+
+        self._state = torch.load(str(path), map_location="cpu", weights_only=True)
+
+    def keys(self) -> list[str]:
+        return list(self._state.keys())
+
+    def get_tensor(self, name: str):
+        return self._state[name]
+
+
+class _ReaderCollection(list):
+    """Reader list with a cached tensor-name -> reader lookup table."""
+
+    def __init__(self, readers: list, *, tensor_map: dict[str, object] | None = None):
+        super().__init__(readers)
+        if tensor_map is None:
+            tensor_map = {}
+            for reader in readers:
+                for key in reader.keys():
+                    tensor_map[key] = reader
+        self.tensor_map = tensor_map
+
+
+def _open_safetensors(model_dir: Path) -> list:
+    """Open all safetensor shards (or pytorch .bin) in a model directory."""
+    fw = _detect_framework()
+    single = model_dir / "model.safetensors"
+    if single.exists():
+        return _ReaderCollection([safe_open(str(single), framework=fw)])
+
+    index_path = model_dir / "model.safetensors.index.json"
+    if index_path.exists():
+        import json
+
+        index = json.loads(index_path.read_text())
+        weight_map = index.get("weight_map", {})
+        shard_files = sorted(set(weight_map.values()))
+        readers_by_file = {
+            shard: safe_open(str(model_dir / shard), framework=fw) for shard in shard_files
+        }
+        tensor_map = {name: readers_by_file[shard] for name, shard in weight_map.items()}
+        return _ReaderCollection(
+            [readers_by_file[shard] for shard in shard_files],
+            tensor_map=tensor_map,
+        )
+
+    # Diffusers format: diffusion_pytorch_model.safetensors
+    diff_single = model_dir / "diffusion_pytorch_model.safetensors"
+    if diff_single.exists():
+        return _ReaderCollection([safe_open(str(diff_single), framework=fw)])
+
+    diff_index = model_dir / "diffusion_pytorch_model.safetensors.index.json"
+    if diff_index.exists():
+        import json
+
+        index = json.loads(diff_index.read_text())
+        weight_map = index.get("weight_map", {})
+        shard_files = sorted(set(weight_map.values()))
+        readers_by_file = {
+            shard: safe_open(str(model_dir / shard), framework=fw) for shard in shard_files
+        }
+        tensor_map = {name: readers_by_file[shard] for name, shard in weight_map.items()}
+        return _ReaderCollection(
+            [readers_by_file[shard] for shard in shard_files],
+            tensor_map=tensor_map,
+        )
+
+    # Fallback: pytorch_model.bin (older HF models)
+    bin_single = model_dir / "pytorch_model.bin"
+    if bin_single.exists():
+        return _ReaderCollection([_TorchBinReader(bin_single)])
+
+    raise FileNotFoundError(
+        f"No model.safetensors, index.json, or pytorch_model.bin in {model_dir}"
+    )
+
+
+def _has_tensor(readers: list, name: str) -> bool:
+    tensor_map = getattr(readers, "tensor_map", None)
+    if tensor_map is not None:
+        return name in tensor_map
+    for r in readers:
+        if name in r.keys():
+            return True
+    return False
+
+
+def _to_numpy_fp32(t) -> np.ndarray:
+    """Convert a safetensors/torch tensor to numpy float32 with minimal copies."""
+    if hasattr(t, "numpy"):
+        dtype = getattr(t, "dtype", None)
+        if str(dtype) == "torch.float32":
+            return t.numpy()
+        return t.float().numpy()
+
+    dtype_str = str(t.dtype)
+    if t.dtype == np.uint16 or dtype_str == "bfloat16":
+        t = t.view(np.uint16).astype(np.uint32) << 16
+        return t.view(np.float32)
+    if dtype_str == "float16":
+        return t.astype(np.float32)
+    return np.asarray(t, dtype=np.float32)
+
+
+def _load_tensor(readers: list, name: str) -> np.ndarray:
+    tensor_map = getattr(readers, "tensor_map", None)
+    if tensor_map is not None:
+        reader = tensor_map.get(name)
+        if reader is None:
+            raise KeyError(f"Tensor not found: {name}")
+        return _to_numpy_fp32(reader.get_tensor(name))
+    for r in readers:
+        if name in r.keys():
+            return _to_numpy_fp32(r.get_tensor(name))
+    raise KeyError(f"Tensor not found: {name}")