diff --git a/pyproject.toml b/pyproject.toml index 6207e57c2b8..9e6d07a75a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -174,8 +174,9 @@ megatron = [ # and bridge is installed deps-free, so it has to be listed here. "megatron-energon==7.4.1", # megatron-bridge hard-imports modelopt but is installed deps-free (below), so - # megatron needs it explicitly. 0.44.0: PyPI stable of bridge r0.5.0's 0.44.x. - "nvidia-modelopt==0.44.0", + # megatron needs it explicitly. 0.47.0: first release with the Nemotron-H + # Lightning recipe (modelopt_recipes) that qat.mode=modelopt_mixed loads. + "nvidia-modelopt==0.47.0", # transformers NOT pinned here (see fsdp note): settled via override-deps below. ] diff --git a/uv.lock b/uv.lock index f0178c90ceb..79d568fb143 100644 --- a/uv.lock +++ b/uv.lock @@ -2712,7 +2712,7 @@ wheels = [ [[package]] name = "nvidia-modelopt" -version = "0.44.0" +version = "0.47.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "ninja" }, @@ -2732,7 +2732,7 @@ dependencies = [ { name = "tqdm" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/83/ab/7e12dd238638624cb9d48904e9205abe16a5b26bdd5f9b91e3357821cf90/nvidia_modelopt-0.44.0-py3-none-any.whl", hash = "sha256:9b54a853dfda161db97a0dfce4d7c24269d1d19966b5e2026094186af897f74d", size = 1604658, upload-time = "2026-05-13T20:47:04.007Z" }, + { url = "https://files.pythonhosted.org/packages/6d/46/f32f6183f6a26db8c6bf5c70f89f68ae2cc55c3fbb981016a772c93c4fc2/nvidia_modelopt-0.47.0-py3-none-any.whl", hash = "sha256:db13e62d7efe4a1512e8b8c3f479e28d64022306f94a194f13234e7dca1d3120", size = 2186201, upload-time = "2026-09-23T19:06:44.103Z" }, ] [[package]] @@ -5539,7 +5539,7 @@ requires-dist = [ { name = "numpy", specifier = ">=2.0.0" }, { name = "numpy", marker = "extra == 'verl-core'", specifier = ">=2.0.0" }, { name = "nvidia-mathdx", marker = "extra == 'megatron'" }, - { name = "nvidia-modelopt", marker = "extra == 'megatron'", specifier = "==0.44.0" }, + { name = "nvidia-modelopt", marker = "extra == 'megatron'", specifier = "==0.47.0" }, { name = "nvtx", marker = "extra == 'verl-core'" }, { name = "onnxscript", marker = "extra == 'megatron'" }, { name = "packaging", specifier = ">=20.0" }, diff --git a/verl/trainer/config/_generated_ppo_megatron_trainer.yaml b/verl/trainer/config/_generated_ppo_megatron_trainer.yaml index 3f001c45fa5..75ec7529af9 100644 --- a/verl/trainer/config/_generated_ppo_megatron_trainer.yaml +++ b/verl/trainer/config/_generated_ppo_megatron_trainer.yaml @@ -56,6 +56,8 @@ actor_rollout_ref: use_distributed_optimizer: true use_dist_checkpointing: false dist_checkpointing_path: null + hf_weights_path: null + mixed_precision_export: false dynamic_context_parallel: false max_seqlen_per_dp_cp_rank: null dist_checkpointing_prefix: '' @@ -263,6 +265,8 @@ actor_rollout_ref: use_distributed_optimizer: true use_dist_checkpointing: false dist_checkpointing_path: null + hf_weights_path: null + mixed_precision_export: false dynamic_context_parallel: ${oc.select:actor_rollout_ref.actor.megatron.dynamic_context_parallel,False} max_seqlen_per_dp_cp_rank: ${oc.select:actor_rollout_ref.actor.megatron.max_seqlen_per_dp_cp_rank,null} dist_checkpointing_prefix: '' @@ -598,6 +602,8 @@ critic: use_distributed_optimizer: true use_dist_checkpointing: false dist_checkpointing_path: null + hf_weights_path: null + mixed_precision_export: false dynamic_context_parallel: false max_seqlen_per_dp_cp_rank: null dist_checkpointing_prefix: '' diff --git a/verl/trainer/config/engine/megatron.yaml b/verl/trainer/config/engine/megatron.yaml index 0ed6528e80b..9ad3adc2cf7 100644 --- a/verl/trainer/config/engine/megatron.yaml +++ b/verl/trainer/config/engine/megatron.yaml @@ -37,6 +37,12 @@ use_dist_checkpointing: False # distributed checkpointing path dist_checkpointing_path: null +# HF checkpoint to build/initialize the Megatron model from, if not model.path +hf_weights_path: null + +# Requantize every weight sync into model.path's ModelOpt MIXED_PRECISION format +mixed_precision_export: false + # Whether to use dynamic context parallel scheduling dynamic_context_parallel: False diff --git a/verl/utils/checkpoint/megatron_checkpoint_manager.py b/verl/utils/checkpoint/megatron_checkpoint_manager.py index 3c7d487a779..e481a052b03 100644 --- a/verl/utils/checkpoint/megatron_checkpoint_manager.py +++ b/verl/utils/checkpoint/megatron_checkpoint_manager.py @@ -384,6 +384,22 @@ def _build_model_sharded_state_dict(self, metadata: dict) -> dict: model_sharded_state_dict[key] = model.sharded_state_dict(metadata=model_metadata) return model_sharded_state_dict + def _pop_modelopt_extra_state(self, model_sd: dict) -> dict: + """Take ``_extra_state`` of a ModelOpt-quantized model from the live model, not the checkpoint. + + The quantizers are rebuilt from the recipe at start-up, so the stored quantizer state is not + needed, and checkpoints written before the deployment scales left the quantizer attributes + carry them in every expert's ``_extra_state`` (tens of GiB of host memory per node to read). + """ + modules = (m for model in self.model for m in model.modules()) + if not any(hasattr(m, "modelopt_set_extra_state_callbacks") for m in modules): + return {} + live = {} + for key, sharded in model_sd.items(): + live[key] = {k: v.data for k, v in sharded.items() if "_extra_state" in k} + model_sd[key] = {k: v for k, v in sharded.items() if k not in live[key]} + return live + def _build_optimizer_state_dict( self, model_sharded_state_dict: dict, @@ -907,10 +923,13 @@ def load_checkpoint(self, local_path: str, hdfs_path: str = None, del_local_afte # ── Load model weights ────────────────────────────────────────────── if self.should_load_dist_ckpt_model or (self.should_load_hf_model and self.peft_cls is not None): model_sd = self._maybe_filter_peft_state_dict(dict(model_sharded_state_dict)) + live_extra_state = self._pop_modelopt_extra_state(model_sd) loaded_model = load_dist_checkpointing( sharded_state_dict=model_sd, ckpt_dir=model_dist_path, ) + for key, extra_state in live_extra_state.items(): + loaded_model[key].update(extra_state) assert "model" in loaded_model or any( f"model{vpp_rank}" in loaded_model for vpp_rank in range(len(self.model)) ), f"Model state dict not found in {loaded_model.keys()}. Please check the checkpoint file {local_path}." diff --git a/verl/utils/modelopt/mixed_precision_export.py b/verl/utils/modelopt/mixed_precision_export.py new file mode 100644 index 00000000000..e9c995a95aa --- /dev/null +++ b/verl/utils/modelopt/mixed_precision_export.py @@ -0,0 +1,187 @@ +# Copyright 2026 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Weight sync into a ModelOpt MIXED_PRECISION deployment (NVFP4 + FP8) from ``qat.mode=modelopt_mixed``. + +The trainer holds BF16 masters under ModelOpt's Lightning recipe (static, calibrated weight +quantizers; see ``quantize.apply_mixed_precision_qat``). Every sync quantizes the current weights +with ModelOpt's exporter rule (``to_quantized_weight``) and the static scales of those quantizers, +the same call the QAT forward dequantizes, and sends the quantizers' static activation scales +(``input_scale``, ``k_scale``, ``v_scale``). + +Megatron-Bridge yields each weight after its EP gather, so every rank needs the scales of all +experts: they are gathered once over the EP group at the first sync. +""" + +from __future__ import annotations + +import json +import os +import re +from collections.abc import Iterable, Iterator + +import torch + + +def load_mixed_precision_config(model_path: str) -> dict | None: + path = os.path.join(model_path, "hf_quant_config.json") + if not os.path.isfile(path): + return None + with open(path) as f: + quant = json.load(f)["quantization"] + if quant.get("quant_algo") != "MIXED_PRECISION": + return None + return quant + + +def _category(module: str) -> str: + for name, pattern in ( + ("routed_experts", r"\.experts\.\d+\."), + ("shared_experts", r"\.shared_experts\."), + ("mamba_in_proj", r"\.in_proj$"), + ("mamba_out_proj", r"\.out_proj$"), + ): + if re.search(pattern, module): + return name + return module + + +class ModelOptMixedPrecisionExporter: + def __init__(self, model_path: str, quant: dict): + from verl.utils.modelopt.quantize import DEPLOY_STATE + + if not DEPLOY_STATE["scales"]: + raise RuntimeError("mixed_precision_export needs the quantizers of qat.mode=modelopt_mixed") + self.layers = {name: cfg["quant_algo"] for name, cfg in quant["quantized_layers"].items()} + self.state = DEPLOY_STATE + self.scales: dict | None = None + self.verify = os.environ.get("VERL_QAT_VERIFY_DEPLOYED") == "1" + self.num_syncs = 0 + + def _gather_scales(self) -> dict: + from megatron.core import parallel_state as mpu + + local = self.state["scales"] + scales = dict(local) + if mpu.get_expert_model_parallel_world_size() > 1: + group = mpu.get_expert_model_parallel_group() + gathered = [None] * torch.distributed.get_world_size(group=group) + # float8_e4m3fn tensors do not survive all_gather_object's pickling: send their bytes. + payload = { + name: { + **entry, + "scales": tuple( + (str(s.dtype), s.contiguous().reshape(-1).view(torch.uint8)) for s in entry["scales"] + ), + } + for name, entry in local.items() + if entry["expert"] + } + torch.distributed.all_gather_object(gathered, payload, group=group) + for rank_scales in gathered: + for name, entry in rank_scales.items(): + restored = tuple( + raw.view(getattr(torch, dtype.removeprefix("torch."))).reshape(ref.shape) + for (dtype, raw), ref in zip(entry["scales"], self._like(local, name), strict=True) + ) + scales[name] = {**entry, "scales": restored} + self._check_replicas(local) + missing = set(self.layers) - set(scales) + if missing or any(scales[name]["algo"] != algo for name, algo in self.layers.items()): + raise RuntimeError(f"quantizers do not cover the deployment: missing {sorted(missing)[:5]}") + return scales + + @staticmethod + def _check_replicas(local: dict) -> None: + """Every rank calibrates the non-expert layers itself: their scales must agree on all ranks.""" + import hashlib + + def digest(scales): + h = hashlib.sha256() + for s in scales: + h.update(str((s.dtype, tuple(s.shape))).encode()) + h.update(s.contiguous().reshape(-1).view(torch.uint8).numpy().tobytes()) + return h.hexdigest() + + from verl.utils.modelopt.quantize import DEPLOY_STATE + + mine = {name: digest(entry["scales"]) for name, entry in local.items() if not entry["expert"]} + mine.update( + {f"{name}.{suffix}": digest((t,)) for name, d in DEPLOY_STATE["static"].items() for suffix, t in d.items()} + ) + if not torch.distributed.is_initialized() or torch.distributed.get_world_size() == 1: + return + gathered = [None] * torch.distributed.get_world_size() + torch.distributed.all_gather_object(gathered, mine) + differing = sorted({name for other in gathered for name in mine if other.get(name) != mine[name]}) + if differing: + raise RuntimeError( + f"non-expert quantizer scales differ across ranks: {differing[:8]} ({len(differing)} layers)" + ) + print( + f"[QAT-XRANK rank={torch.distributed.get_rank()}] {len(mine)} replicated scale sets identical on " + f"{len(gathered)} ranks", + flush=True, + ) + + @staticmethod + def _like(local: dict, name: str) -> tuple: + """Scales of a local expert of the same projection: the shapes of every expert's scales.""" + proj = name.rpartition(".")[2] + return next(e["scales"] for n, e in local.items() if e["expert"] and n.endswith("." + proj)) + + def __call__(self, named_tensors: Iterable[tuple[str, torch.Tensor]]) -> Iterator[tuple[str, torch.Tensor]]: + from verl.utils.modelopt.quantize import _dequantize, official_quantize + + if self.scales is None: + self.scales = self._gather_scales() + self.num_syncs += 1 + checked: dict[str, list] = {} + for name, tensor in named_tensors: + if "_quantizer." in name: + continue + module, _, leaf = name.rpartition(".") + entry = self.scales.get(module) if leaf == "weight" else None + if entry is None: + yield name, tensor + else: + scales = tuple(s.to(tensor.device) for s in entry["scales"]) + quantized = official_quantize(entry["algo"], tensor, scales) + if self.verify and module in self.state["local"]: + # The weight the QAT forward of this rank uses vs the tensors on the wire. + quantizer, weight = self.state["local"][module] + with torch.no_grad(): + forward = quantizer(weight.detach()) + deployed = _dequantize(entry["algo"], quantized).to(forward.dtype) + diff = (forward.float() - deployed.float()).abs() + row = checked.setdefault(_category(module), [0, 0, 0, 0.0]) + row[0] += 1 + row[1] += diff.numel() + row[2] += int((diff != 0).sum()) + row[3] = max(row[3], float(diff.max())) + for suffix, value in quantized.items(): + yield f"{module}.{suffix}", value + if leaf == "weight": + for suffix, static in self.state["static"].get(module, {}).items(): + yield f"{module}.{suffix}", static.to(tensor.device) + if self.verify: + from verl.utils.modelopt.quantize import state_digest + + rank = torch.distributed.get_rank() if torch.distributed.is_initialized() else 0 + print(f"[QAT-STATE rank={rank} sync={self.num_syncs}] {state_digest()}", flush=True) + for category, (tensors, elements, mismatches, max_abs) in sorted(checked.items()): + print( + f"[QAT-VERIFY rank={rank} sync={self.num_syncs}] {category}: tensors={tensors} " + f"elements={elements} forward!=deployed={mismatches} max_abs={max_abs:.3e}", + flush=True, + ) diff --git a/verl/utils/modelopt/qat_utils.py b/verl/utils/modelopt/qat_utils.py index ca002c5132c..889348106e4 100644 --- a/verl/utils/modelopt/qat_utils.py +++ b/verl/utils/modelopt/qat_utils.py @@ -23,7 +23,7 @@ def _get_qat_field(qat_config, key, default=None): return getattr(qat_config, key, default) -def apply_qat_to_modules(modules, qat_config): +def apply_qat_to_modules(modules, qat_config, deploy_path=None): """Apply ModelOpt fake quantization to a list of Megatron module chunks.""" from verl.utils.modelopt.quantize import apply_qat @@ -33,7 +33,7 @@ def apply_qat_to_modules(modules, qat_config): ignore_patterns = list(ignore_patterns) for i in range(len(modules)): - modules[i] = apply_qat(modules[i], qat_mode, ignore_patterns=ignore_patterns) + modules[i] = apply_qat(modules[i], qat_mode, ignore_patterns=ignore_patterns, deploy_path=deploy_path) return modules diff --git a/verl/utils/modelopt/quantize.py b/verl/utils/modelopt/quantize.py index dee7274099c..2bd1c2cdab9 100644 --- a/verl/utils/modelopt/quantize.py +++ b/verl/utils/modelopt/quantize.py @@ -16,10 +16,16 @@ """ModelOpt NVFP4 quantization config and application for Megatron QAT.""" import copy +import json +import os +import re +import weakref import modelopt.torch.quantization as mtq +import torch import torch.nn as nn from modelopt.torch.quantization.config import _default_disabled_quantizer_cfg +from modelopt.torch.quantization.nn import TensorQuantizer _NVFP4_W4A16_QUANTIZER_CFG = { "*weight_quantizer": { @@ -69,12 +75,260 @@ def build_quantize_config( return {"quant_cfg": quant_cfg, "algorithm": "max"} +# ModelOpt's recipe for the Nemotron-H Lightning NVFP4 deployment: static NVFP4 4/6 (MSE) routed and +# shared experts and lm_head, FP8 Mamba in/out_proj weights and inputs, FP8 KV cache, MTP disabled. +_LIGHTNING_RECIPE = "models/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16/ptq/w4a16_nvfp4_4o6" + +# Filled by apply_mixed_precision_qat for this rank; read by mixed_precision_export. +# scales: HF module -> {"algo", "expert", "scales": (...)} of the weights quantized on this rank +# static: HF module -> {suffix: tensor} static activation scales (input_scale / k_scale / v_scale) +# local: HF module -> (weight quantizer, weight parameter) for the forward-vs-deployed check +# act: quantizer name -> static FP8 input / KV quantizer +DEPLOY_STATE: dict = {"scales": {}, "static": {}, "local": {}, "act": {}} + + +# Deployment scales per weight quantizer. Not an attribute of the quantizer: ModelOpt pickles every +# quantizer attribute into the module's ``_extra_state``, once per expert for grouped linears. +_DEPLOY: "weakref.WeakKeyDictionary[nn.Module, tuple]" = weakref.WeakKeyDictionary() + + +def state_digest() -> dict[str, str]: + """Digests of this rank's quantizer state: live amax buffers, the scales the forward uses, activation amax.""" + import hashlib + + def digest(tensors): + h = hashlib.sha256() + for t in tensors: + t = t.detach().contiguous().cpu() + h.update(str((t.dtype, tuple(t.shape))).encode()) + h.update(t.reshape(-1).view(torch.uint8).numpy().tobytes()) + return h.hexdigest()[:16] + + amax, scales = [], [] + for name in sorted(DEPLOY_STATE["local"]): + quantizer = DEPLOY_STATE["local"][name][0] + amax.append(quantizer._amax) + if getattr(quantizer, "_global_amax", None) is not None: + amax.append(quantizer._global_amax) + scales.extend(_DEPLOY[quantizer][1]) + act = [DEPLOY_STATE["act"][name]._amax for name in sorted(DEPLOY_STATE["act"])] + return {"weight_amax": digest(amax), "deploy_scales": digest(scales), "act_amax": digest(act)} + + +def _dequantize(algorithm: str, tensors: dict[str, torch.Tensor]) -> torch.Tensor: + """FP32 value of deployment tensors (ModelOpt's reference NVFP4 dequantize, not its Triton fast path).""" + from modelopt.torch.quantization.qtensor.nvfp4_tensor import NVFP4QTensor + + if algorithm == "FP8": + return tensors["weight"].float() * tensors["weight_scale"].float() + packed = tensors["weight"] + shape = torch.Size((*packed.shape[:-1], packed.shape[-1] * 2)) + return NVFP4QTensor(shape, torch.float32, packed).dequantize( + dtype=torch.float32, + scale=tensors["weight_scale"], + double_scale=tensors["weight_scale_2"].float().reshape(()), + block_sizes={-1: 16}, + ) + + +def official_scales(algorithm: str, quantizer, weight: torch.Tensor) -> tuple: + """The deployment scales ModelOpt's exporter derives from a calibrated (static) weight quantizer.""" + from modelopt.torch.export.quant_utils import get_scaling_factor + from modelopt.torch.quantization.qtensor.nvfp4_tensor import NVFP4QTensor + + if algorithm == "FP8": + return (get_scaling_factor(quantizer).reshape(()),) + scale_2 = NVFP4QTensor.get_weights_scaling_factor_2_from_quantizer(quantizer).reshape(()) + scale = NVFP4QTensor.get_weights_scaling_factor_from_quantizer(quantizer, weight, scale_2)[0] + return scale, scale_2 + + +def official_quantize(algorithm: str, weight: torch.Tensor, scales: tuple) -> dict[str, torch.Tensor]: + """ModelOpt's exporter real quantization (``to_quantized_weight``) with the static scales.""" + from modelopt.torch.export.quant_utils import QUANTIZATION_W4A16_NVFP4, to_quantized_weight + + weight = weight.to(torch.bfloat16) + if algorithm == "FP8": + (scale,) = scales + # to_quantized_weight's FP8 rule, E4M3(BF16(w / scale)), written out: on the GPU PyTorch rounds + # its 0-dim scale to BF16 before dividing, which the exported checkpoint (CPU) does not. + quotient = (weight.float() / scale.float()).to(torch.bfloat16) + code = quotient.float().clamp(-448.0, 448.0).to(torch.float8_e4m3fn) + return {"weight": code, "weight_scale": scale} + scale, scale_2 = scales + packed = to_quantized_weight(weight, scale, QUANTIZATION_W4A16_NVFP4, scale_2, 16) + return {"weight": packed, "weight_scale": scale, "weight_scale_2": scale_2} + + +def _deployed_ste_forward(quantizer, inputs): + """Forward value = the deployed weight (dequantized exporter output); gradient straight through. + + ModelOpt's fake-quant kernels are not bit-identical to its exporter (FP8: FP32 vs BF16 quotient; + NVFP4: rounding at ties), so the trained weight would differ from the served one. + """ + with torch.no_grad(): + algorithm, scales = _DEPLOY[quantizer] + deployed = _dequantize(algorithm, official_quantize(algorithm, inputs.detach(), scales)) + return inputs + (deployed.to(inputs.dtype) - inputs).detach() + + +def _deployment_modules(core: nn.Module, ep_rank: int): + """Yield (kind, HF module name, mcore module name, module[, local expert]) of the deployment layers.""" + for name, module in core.named_modules(): + if name.startswith("mtp.") or ".mtp." in name: + continue + match = re.search(r"decoder\.layers\.(\d+)\.(.+)$", name) + if name == "output_layer": + yield "nvfp4", "lm_head", name, module, None + if match is None: + continue + layer, rest = match.groups() + hf = f"backbone.layers.{layer}.mixer" + if rest in ("mixer.in_proj", "mixer.out_proj"): + yield "fp8", f"{hf}.{rest.split('.')[1]}", name, module, None + elif rest in ("mlp.shared_experts.linear_fc1", "mlp.shared_experts.linear_fc2"): + yield ( + "nvfp4", + f"{hf}.shared_experts.{'up_proj' if rest.endswith('fc1') else 'down_proj'}", + name, + module, + None, + ) + elif rest in ("mlp.experts.linear_fc1", "mlp.experts.linear_fc2"): + proj = "up_proj" if rest.endswith("fc1") else "down_proj" + for local in range(module.num_gemms): + yield "nvfp4", f"{hf}.experts.{ep_rank * module.num_gemms + local}.{proj}", name, module, local + elif rest == "self_attention.core_attention": + yield "kv", hf, name, module, None + + +def apply_mixed_precision_qat(model: nn.Module, deploy_path: str) -> nn.Module: + """ModelOpt's Lightning recipe on the MCore model, forward on the deployed weights. + + Weights: static MSE calibration of the recipe. FP8 input / KV quantizers: the deployment + checkpoint's static scales, pinned through the recipe's ``constant_amax`` (no calibration data). + """ + import modelopt.torch.quantization as mtq + from megatron.core import parallel_state as mpu + from megatron.core.utils import unwrap_model + from modelopt.recipe import load_recipe + from modelopt.torch.export.quant_utils import get_kv_cache_scaling_factor, get_scaling_factor + from safetensors import safe_open + + core = unwrap_model(model) + ep_rank = mpu.get_expert_model_parallel_rank() + with open(os.path.join(deploy_path, "hf_quant_config.json")) as f: + deployed_layers = json.load(f)["quantization"]["quantized_layers"] + with open(os.path.join(deploy_path, "model.safetensors.index.json")) as f: + weight_map = json.load(f)["weight_map"] + + def checkpoint_scalar(key): + with safe_open(os.path.join(deploy_path, weight_map[key]), framework="pt") as f: + return float(f.get_tensor(key).double().reshape(())) + + # (quantizer name, checkpoint key) of the static activation scales + static_keys = [] + for kind, hf, name, _module, _local in _deployment_modules(core, ep_rank): + if kind == "fp8": + static_keys.append((f"{name}.input_quantizer", f"{hf}.input_scale")) + elif kind == "kv": + static_keys.append((f"{name}.k_bmm_quantizer", f"{hf}.k_proj.k_scale")) + static_keys.append((f"{name}.v_bmm_quantizer", f"{hf}.v_proj.v_scale")) + config = load_recipe(_LIGHTNING_RECIPE).quantize.model_dump() + config["quant_cfg"] = list(config["quant_cfg"]) + [ + { + "quantizer_name": quantizer, + "enable": True, + "cfg": {"num_bits": (4, 3), "axis": None, "constant_amax": checkpoint_scalar(key) * 448.0}, + } + for quantizer, key in static_keys + ] + # Weight calibration is rank-local and the activation amax is constant: nothing to sync. + config["algorithm"] = {**config["algorithm"], "distributed_sync": False} + mtq.quantize(core, config, forward_loop=None) + # The quantizers are rebuilt from the recipe at every start, so a checkpoint load only has to fill + # their buffers. ModelOpt's restore callback (meant for an unquantized model, run before the weight + # load) would rebuild them inside the same load and drop the Mamba FP8 amax. + for module in core.modules(): + if hasattr(module, "allow_post_restore"): + module.allow_post_restore = False + + state = {"scales": {}, "static": {}, "local": {}, "act": {}} + modules = dict(core.named_modules()) + for quantizer, _key in static_keys: + state["act"][quantizer] = modules[quantizer] + mtp = [ + (n, m) for n, m in modules.items() if isinstance(m, TensorQuantizer) and (n.startswith("mtp.") or ".mtp." in n) + ] + if any(m.is_enabled for _, m in mtp): + raise RuntimeError(f"MTP quantizers enabled: {[n for n, m in mtp if m.is_enabled][:5]}") + quantized = set() + for kind, hf, name, module, local in _deployment_modules(core, ep_rank): + if kind == "kv": + k_scale, v_scale = get_kv_cache_scaling_factor(module, clamp_fp8_scales=False) + state["static"][f"{hf}.k_proj"] = {"k_scale": k_scale.float().cpu()} + state["static"][f"{hf}.v_proj"] = {"v_scale": v_scale.float().cpu()} + continue + algorithm = "FP8" if kind == "fp8" else "W4A16_NVFP4" + if deployed_layers.get(hf, {}).get("quant_algo") != algorithm: + raise RuntimeError(f"{name}: deployment lists {deployed_layers.get(hf)} for {hf}, recipe gives {algorithm}") + quantizer = module.weight_quantizer if local is None else module.weight_quantizer[local] + weight = module.weight if local is None else getattr(module, f"weight{local}") + if not quantizer.is_enabled or getattr(quantizer, "_amax", None) is None: + raise RuntimeError(f"{name}: the recipe left {hf} without a calibrated weight quantizer") + with torch.no_grad(): + scales = official_scales(algorithm, quantizer, weight) + _DEPLOY[quantizer] = (algorithm, scales) + quantizer.forward = _deployed_ste_forward.__get__(quantizer) + state["scales"][hf] = {"algo": algorithm, "expert": local is not None, "scales": tuple(s.cpu() for s in scales)} + state["local"][hf] = (quantizer, weight) + quantized.add(hf) + if kind == "fp8": + state["static"][hf] = {"input_scale": get_scaling_factor(module.input_quantizer).float().cpu()} + ep_size = mpu.get_expert_model_parallel_world_size() + expected = { + hf + for hf in deployed_layers + if ".experts." not in hf + or re.search(r"\.experts\.(\d+)\.", hf) is None + or int(re.search(r"\.experts\.(\d+)\.", hf).group(1)) * ep_size // _num_experts(deployed_layers) == ep_rank + } + if quantized != expected: + raise RuntimeError( + f"recipe/deployment mismatch on EP rank {ep_rank}: missing {sorted(expected - quantized)[:5]}, " + f"extra {sorted(quantized - expected)[:5]}" + ) + for quantizer, key in static_keys: + module_key, _, suffix = key.rpartition(".") + got = state["static"][module_key][suffix].reshape(()).item() + want = torch.tensor(checkpoint_scalar(key), dtype=torch.float32).item() + if got != want: + raise RuntimeError(f"{quantizer}: static scale {got} != deployment {key} {want}") + DEPLOY_STATE.update(state) + print( + f"[QAT modelopt_mixed] EP rank {ep_rank}: {len(quantized)} weight quantizers " + f"(static, forward = deployed weight), {len(static_keys)} static activation/KV quantizers, " + f"MTP quantizers enabled 0 of {len(mtp)}", + flush=True, + ) + if os.environ.get("VERL_QAT_VERIFY_DEPLOYED") == "1": + print(f"[QAT-STATE rank={torch.distributed.get_rank()} init] {state_digest()}", flush=True) + return model + + +def _num_experts(deployed_layers: dict) -> int: + return 1 + max(int(m.group(1)) for m in (re.search(r"\.experts\.(\d+)\.", k) for k in deployed_layers) if m) + + def apply_qat( model: nn.Module, qat_mode: str, ignore_patterns: list[str] | None = None, + deploy_path: str | None = None, ) -> nn.Module: """Apply Quantization-Aware Training to a Megatron model.""" + if qat_mode == "modelopt_mixed": + return apply_mixed_precision_qat(model, deploy_path) config = build_quantize_config(qat_mode, ignore_patterns) mtq.quantize(model, config) return model diff --git a/verl/workers/config/engine.py b/verl/workers/config/engine.py index ba826f52b02..fb5d34373b7 100644 --- a/verl/workers/config/engine.py +++ b/verl/workers/config/engine.py @@ -166,6 +166,11 @@ class McoreEngineConfig(EngineConfig): use_distributed_optimizer (bool): Whether to use distributed optimizer. use_dist_checkpointing (bool): Whether to use distributed checkpointing. dist_checkpointing_path (Optional[str]): Path for distributed checkpointing. + hf_weights_path (Optional[str]): HF checkpoint the Megatron model is built and + initialized from when it differs from ``model.path`` (e.g. BF16 masters + for a quantized rollout checkpoint). + mixed_precision_export (bool): Requantize every weight sync into the + ModelOpt MIXED_PRECISION checkpoint format of ``model.path``. dist_ckpt_optim_fully_reshardable (bool): Use fully reshardable optimizer checkpoints. distrib_optim_fully_reshardable_mem_efficient (bool): Use memory-efficient fully reshardable format. seed (int): Random seed for reproducibility. @@ -198,6 +203,8 @@ class McoreEngineConfig(EngineConfig): pad_to_length_bucket: int = 512 use_dist_checkpointing: bool = False dist_checkpointing_path: Optional[str] = None + hf_weights_path: Optional[str] = None + mixed_precision_export: bool = False dist_checkpointing_prefix: str = "" dist_ckpt_optim_fully_reshardable: bool = False distrib_optim_fully_reshardable_mem_efficient: bool = False @@ -225,6 +232,23 @@ def __post_init__(self) -> None: if self.tensor_model_parallel_size == 1: warnings.warn("set sequence parallel to false as TP size is 1", stacklevel=2) self.sequence_parallel = False + if self.qat.get("enable", False): + # modelopt_mixed QAT has no exporter of its own; the rollout serves the + # ModelOpt checkpoint and needs requantized weights at every sync. + if (self.qat.get("mode") == "modelopt_mixed") != self.mixed_precision_export: + raise ValueError( + "qat.mode=modelopt_mixed requires mixed_precision_export=True and vice versa, " + f"got qat.mode={self.qat.get('mode')}, mixed_precision_export={self.mixed_precision_export}" + ) + # Megatron uses the tensor parallel size for experts when theirs is unset. + expert_tp = self.expert_tensor_parallel_size or self.tensor_model_parallel_size + if self.qat.get("mode") == "modelopt_mixed" and (self.tensor_model_parallel_size != 1 or expert_tp != 1): + # The static scales are calibrated on each rank's shard, the exporter + # quantizes the full tensor (ModelOpt rejects static NVFP4 with TP > 1 too). + raise ValueError( + "qat.mode=modelopt_mixed requires tensor_model_parallel_size=1 and " + f"expert_tensor_parallel_size=1, got {self.tensor_model_parallel_size} and {expert_tp}" + ) @dataclass diff --git a/verl/workers/engine/megatron/transformer_impl.py b/verl/workers/engine/megatron/transformer_impl.py index b75c00c96e5..0aa4ac52aec 100644 --- a/verl/workers/engine/megatron/transformer_impl.py +++ b/verl/workers/engine/megatron/transformer_impl.py @@ -310,9 +310,11 @@ def _build_tf_config(self): self.provider = None from verl.models.mcore.bridge import AutoBridge + self._hf_weights_path = self.engine_config.hf_weights_path or self.model_config.local_path + # Use Megatron-Bridge to convert HF config to Megatron config bridge = AutoBridge.from_hf_pretrained( - self.model_config.local_path, trust_remote_code=self.model_config.trust_remote_code + self._hf_weights_path, trust_remote_code=self.model_config.trust_remote_code ) # Get Megatron provider and configure it provider = bridge.to_megatron_provider(load_weights=False) @@ -359,7 +361,7 @@ def _build_tf_config(self): else: provider_overrides["enable_routing_replay"] = True - if self._qat_enabled: + if self._qat_enabled and self._qat_config.mode != "modelopt_mixed": from megatron.bridge.models.gpt_provider import modelopt_transformer_layer_spec provider.transformer_layer_spec = modelopt_transformer_layer_spec @@ -456,7 +458,7 @@ def _build_megatron_module(self): else: allowed_mismatched_params = ["output_layer.weight"] if self.is_value_model else [] self.bridge.load_hf_weights( - module, self.model_config.local_path, allowed_mismatched_params=allowed_mismatched_params + module, self._hf_weights_path, allowed_mismatched_params=allowed_mismatched_params ) if torch.distributed.get_rank() == 0: @@ -531,7 +533,7 @@ def initialize(self): if self._qat_enabled and not self.engine_config.forward_only: from verl.utils.modelopt import apply_qat_to_modules - self.module = apply_qat_to_modules(self.module, self._qat_config) + self.module = apply_qat_to_modules(self.module, self._qat_config, deploy_path=self.model_config.local_path) self._maybe_enable_fused_kernels() @@ -975,7 +977,9 @@ def _mbridge_export_tasks(self): if getattr(self.bridge, "export_weight_dtype", None) == "fp8": return None if self._hf_export_tasks is None: - self._hf_export_tasks = self.bridge.get_conversion_tasks(self.module) + # ModelOpt quantizer buffers (_amax) have no HF mapping: Bridge leaves a None slot for each. + tasks = self.bridge.get_conversion_tasks(self.module) + self._hf_export_tasks = [task for task in tasks if task is not None] return self._hf_export_tasks def get_per_tensor_param(self, base_sync_done=False, **kwargs): @@ -1000,14 +1004,30 @@ def get_per_tensor_param(self, base_sync_done=False, **kwargs): else self.bridge.export_hf_weights(self.module, conversion_tasks=conversion_tasks) ) + if not adapter_only and self.engine_config.mixed_precision_export: + per_tensor_param = self._mixed_precision_exporter()(per_tensor_param) + # QAT: process weights through QATWeightExporter for quantized weight sync to vLLM - if self._qat_enabled: + if self._qat_enabled and self._qat_config.mode != "modelopt_mixed": from verl.utils.modelopt import export_qat_weights per_tensor_param = export_qat_weights(per_tensor_param, self.module, self._qat_config.mode, self.bridge) return per_tensor_param, peft_config + def _mixed_precision_exporter(self): + if not hasattr(self, "_mp_exporter"): + from verl.utils.modelopt.mixed_precision_export import ( + ModelOptMixedPrecisionExporter, + load_mixed_precision_config, + ) + + quant = load_mixed_precision_config(self.model_config.local_path) + if quant is None: + raise ValueError(f"mixed_precision_export: {self.model_config.local_path} is not MIXED_PRECISION") + self._mp_exporter = ModelOptMixedPrecisionExporter(self.model_config.local_path, quant) + return self._mp_exporter + def _mcore_export_index(self): """Build (once) the per-parameter delta export index: geometry specs and form-B probes derived from the bridge's conversion tasks. See @@ -1029,6 +1049,11 @@ def get_per_tensor_param_shard(self, **kwargs): comm-stubbed probe owns all shard geometry). Pure export, no side effects. Params owned by another pipeline stage yield an empty shard (zero-count lockstep rows; see the index builder).""" + if self.engine_config.mixed_precision_export: + raise NotImplementedError( + "mixed_precision_export requantizes only the full-tensor weight sync " + "(get_per_tensor_param); use checkpoint_engine.backend=naive" + ) load_megatron_model_to_gpu(self.module, load_grad=False) index = self._mcore_export_index() diff --git a/verl/workers/rollout/vllm_rollout/vllm_async_server.py b/verl/workers/rollout/vllm_rollout/vllm_async_server.py index 03a578128c2..344e600542e 100644 --- a/verl/workers/rollout/vllm_rollout/vllm_async_server.py +++ b/verl/workers/rollout/vllm_rollout/vllm_async_server.py @@ -1356,7 +1356,8 @@ def _apply_quantization(self) -> tuple[Optional[str], dict]: # Handle QAT (Quantization-Aware Training) configuration qat_config_dict = getattr(self.config, "qat", {}) or {} - if qat_config_dict.get("enable", False): + # modelopt_mixed QAT serves the checkpoint's own ModelOpt config as is. + if qat_config_dict.get("enable", False) and qat_config_dict.get("mode") != "modelopt_mixed": from verl.utils.qat import QATConfig, load_quantization_config qat_config = QATConfig(**qat_config_dict)