Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -174,8 +174,9 @@ megatron = [
# and bridge is installed deps-free, so it has to be listed here.
"megatron-energon==7.4.1",
# megatron-bridge hard-imports modelopt but is installed deps-free (below), so
# megatron needs it explicitly. 0.44.0: PyPI stable of bridge r0.5.0's 0.44.x.
"nvidia-modelopt==0.44.0",
# megatron needs it explicitly. 0.47.0: first release with the Nemotron-H
# Lightning recipe (modelopt_recipes) that qat.mode=modelopt_mixed loads.
"nvidia-modelopt==0.47.0",
# transformers NOT pinned here (see fsdp note): settled via override-deps below.
]

Expand Down
6 changes: 3 additions & 3 deletions uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 6 additions & 0 deletions verl/trainer/config/_generated_ppo_megatron_trainer.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ actor_rollout_ref:
use_distributed_optimizer: true
use_dist_checkpointing: false
dist_checkpointing_path: null
hf_weights_path: null
mixed_precision_export: false
dynamic_context_parallel: false
max_seqlen_per_dp_cp_rank: null
dist_checkpointing_prefix: ''
Expand Down Expand Up @@ -263,6 +265,8 @@ actor_rollout_ref:
use_distributed_optimizer: true
use_dist_checkpointing: false
dist_checkpointing_path: null
hf_weights_path: null
mixed_precision_export: false
dynamic_context_parallel: ${oc.select:actor_rollout_ref.actor.megatron.dynamic_context_parallel,False}
max_seqlen_per_dp_cp_rank: ${oc.select:actor_rollout_ref.actor.megatron.max_seqlen_per_dp_cp_rank,null}
dist_checkpointing_prefix: ''
Expand Down Expand Up @@ -598,6 +602,8 @@ critic:
use_distributed_optimizer: true
use_dist_checkpointing: false
dist_checkpointing_path: null
hf_weights_path: null
mixed_precision_export: false
dynamic_context_parallel: false
max_seqlen_per_dp_cp_rank: null
dist_checkpointing_prefix: ''
Expand Down
6 changes: 6 additions & 0 deletions verl/trainer/config/engine/megatron.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,12 @@ use_dist_checkpointing: False
# distributed checkpointing path
dist_checkpointing_path: null

# HF checkpoint to build/initialize the Megatron model from, if not model.path
hf_weights_path: null

# Requantize every weight sync into model.path's ModelOpt MIXED_PRECISION format
mixed_precision_export: false

# Whether to use dynamic context parallel scheduling
dynamic_context_parallel: False

Expand Down
19 changes: 19 additions & 0 deletions verl/utils/checkpoint/megatron_checkpoint_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -384,6 +384,22 @@ def _build_model_sharded_state_dict(self, metadata: dict) -> dict:
model_sharded_state_dict[key] = model.sharded_state_dict(metadata=model_metadata)
return model_sharded_state_dict

def _pop_modelopt_extra_state(self, model_sd: dict) -> dict:
"""Take ``_extra_state`` of a ModelOpt-quantized model from the live model, not the checkpoint.

The quantizers are rebuilt from the recipe at start-up, so the stored quantizer state is not
needed, and checkpoints written before the deployment scales left the quantizer attributes
carry them in every expert's ``_extra_state`` (tens of GiB of host memory per node to read).
"""
modules = (m for model in self.model for m in model.modules())
if not any(hasattr(m, "modelopt_set_extra_state_callbacks") for m in modules):
return {}
live = {}
for key, sharded in model_sd.items():
live[key] = {k: v.data for k, v in sharded.items() if "_extra_state" in k}
model_sd[key] = {k: v for k, v in sharded.items() if k not in live[key]}
return live

def _build_optimizer_state_dict(
self,
model_sharded_state_dict: dict,
Expand Down Expand Up @@ -907,10 +923,13 @@ def load_checkpoint(self, local_path: str, hdfs_path: str = None, del_local_afte
# ── Load model weights ──────────────────────────────────────────────
if self.should_load_dist_ckpt_model or (self.should_load_hf_model and self.peft_cls is not None):
model_sd = self._maybe_filter_peft_state_dict(dict(model_sharded_state_dict))
live_extra_state = self._pop_modelopt_extra_state(model_sd)
loaded_model = load_dist_checkpointing(
sharded_state_dict=model_sd,
ckpt_dir=model_dist_path,
)
for key, extra_state in live_extra_state.items():
loaded_model[key].update(extra_state)
assert "model" in loaded_model or any(
f"model{vpp_rank}" in loaded_model for vpp_rank in range(len(self.model))
), f"Model state dict not found in {loaded_model.keys()}. Please check the checkpoint file {local_path}."
Expand Down
187 changes: 187 additions & 0 deletions verl/utils/modelopt/mixed_precision_export.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
# Copyright 2026 Bytedance Ltd. and/or its affiliates
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Weight sync into a ModelOpt MIXED_PRECISION deployment (NVFP4 + FP8) from ``qat.mode=modelopt_mixed``.

The trainer holds BF16 masters under ModelOpt's Lightning recipe (static, calibrated weight
quantizers; see ``quantize.apply_mixed_precision_qat``). Every sync quantizes the current weights
with ModelOpt's exporter rule (``to_quantized_weight``) and the static scales of those quantizers,
the same call the QAT forward dequantizes, and sends the quantizers' static activation scales
(``input_scale``, ``k_scale``, ``v_scale``).

Megatron-Bridge yields each weight after its EP gather, so every rank needs the scales of all
experts: they are gathered once over the EP group at the first sync.
"""

from __future__ import annotations

import json
import os
import re
from collections.abc import Iterable, Iterator

import torch


def load_mixed_precision_config(model_path: str) -> dict | None:
path = os.path.join(model_path, "hf_quant_config.json")
if not os.path.isfile(path):
return None
with open(path) as f:
quant = json.load(f)["quantization"]
if quant.get("quant_algo") != "MIXED_PRECISION":
return None
return quant


def _category(module: str) -> str:
for name, pattern in (
("routed_experts", r"\.experts\.\d+\."),
("shared_experts", r"\.shared_experts\."),
("mamba_in_proj", r"\.in_proj$"),
("mamba_out_proj", r"\.out_proj$"),
):
if re.search(pattern, module):
return name
return module


class ModelOptMixedPrecisionExporter:
def __init__(self, model_path: str, quant: dict):
from verl.utils.modelopt.quantize import DEPLOY_STATE

if not DEPLOY_STATE["scales"]:
raise RuntimeError("mixed_precision_export needs the quantizers of qat.mode=modelopt_mixed")
self.layers = {name: cfg["quant_algo"] for name, cfg in quant["quantized_layers"].items()}
self.state = DEPLOY_STATE
self.scales: dict | None = None
self.verify = os.environ.get("VERL_QAT_VERIFY_DEPLOYED") == "1"
self.num_syncs = 0

def _gather_scales(self) -> dict:
from megatron.core import parallel_state as mpu

local = self.state["scales"]
scales = dict(local)
if mpu.get_expert_model_parallel_world_size() > 1:
group = mpu.get_expert_model_parallel_group()
gathered = [None] * torch.distributed.get_world_size(group=group)
# float8_e4m3fn tensors do not survive all_gather_object's pickling: send their bytes.
payload = {
name: {
**entry,
"scales": tuple(
(str(s.dtype), s.contiguous().reshape(-1).view(torch.uint8)) for s in entry["scales"]
),
}
for name, entry in local.items()
if entry["expert"]
}
torch.distributed.all_gather_object(gathered, payload, group=group)
for rank_scales in gathered:
for name, entry in rank_scales.items():
restored = tuple(
raw.view(getattr(torch, dtype.removeprefix("torch."))).reshape(ref.shape)
for (dtype, raw), ref in zip(entry["scales"], self._like(local, name), strict=True)
)
scales[name] = {**entry, "scales": restored}
self._check_replicas(local)
missing = set(self.layers) - set(scales)
if missing or any(scales[name]["algo"] != algo for name, algo in self.layers.items()):
raise RuntimeError(f"quantizers do not cover the deployment: missing {sorted(missing)[:5]}")
return scales

@staticmethod
def _check_replicas(local: dict) -> None:
"""Every rank calibrates the non-expert layers itself: their scales must agree on all ranks."""
import hashlib

def digest(scales):
h = hashlib.sha256()
for s in scales:
h.update(str((s.dtype, tuple(s.shape))).encode())
h.update(s.contiguous().reshape(-1).view(torch.uint8).numpy().tobytes())
return h.hexdigest()

from verl.utils.modelopt.quantize import DEPLOY_STATE

mine = {name: digest(entry["scales"]) for name, entry in local.items() if not entry["expert"]}
mine.update(
{f"{name}.{suffix}": digest((t,)) for name, d in DEPLOY_STATE["static"].items() for suffix, t in d.items()}
)
if not torch.distributed.is_initialized() or torch.distributed.get_world_size() == 1:
return
gathered = [None] * torch.distributed.get_world_size()
torch.distributed.all_gather_object(gathered, mine)
differing = sorted({name for other in gathered for name in mine if other.get(name) != mine[name]})
if differing:
raise RuntimeError(
f"non-expert quantizer scales differ across ranks: {differing[:8]} ({len(differing)} layers)"
)
print(
f"[QAT-XRANK rank={torch.distributed.get_rank()}] {len(mine)} replicated scale sets identical on "
f"{len(gathered)} ranks",
flush=True,
)

@staticmethod
def _like(local: dict, name: str) -> tuple:
"""Scales of a local expert of the same projection: the shapes of every expert's scales."""
proj = name.rpartition(".")[2]
return next(e["scales"] for n, e in local.items() if e["expert"] and n.endswith("." + proj))

def __call__(self, named_tensors: Iterable[tuple[str, torch.Tensor]]) -> Iterator[tuple[str, torch.Tensor]]:
from verl.utils.modelopt.quantize import _dequantize, official_quantize

if self.scales is None:
self.scales = self._gather_scales()
self.num_syncs += 1
checked: dict[str, list] = {}
for name, tensor in named_tensors:
if "_quantizer." in name:
continue
module, _, leaf = name.rpartition(".")
entry = self.scales.get(module) if leaf == "weight" else None
if entry is None:
yield name, tensor
else:
scales = tuple(s.to(tensor.device) for s in entry["scales"])
quantized = official_quantize(entry["algo"], tensor, scales)
if self.verify and module in self.state["local"]:
# The weight the QAT forward of this rank uses vs the tensors on the wire.
quantizer, weight = self.state["local"][module]
with torch.no_grad():
forward = quantizer(weight.detach())
deployed = _dequantize(entry["algo"], quantized).to(forward.dtype)
diff = (forward.float() - deployed.float()).abs()
row = checked.setdefault(_category(module), [0, 0, 0, 0.0])
row[0] += 1
row[1] += diff.numel()
row[2] += int((diff != 0).sum())
row[3] = max(row[3], float(diff.max()))
for suffix, value in quantized.items():
yield f"{module}.{suffix}", value
if leaf == "weight":
for suffix, static in self.state["static"].get(module, {}).items():
yield f"{module}.{suffix}", static.to(tensor.device)
if self.verify:
from verl.utils.modelopt.quantize import state_digest

rank = torch.distributed.get_rank() if torch.distributed.is_initialized() else 0
print(f"[QAT-STATE rank={rank} sync={self.num_syncs}] {state_digest()}", flush=True)
for category, (tensors, elements, mismatches, max_abs) in sorted(checked.items()):
print(
f"[QAT-VERIFY rank={rank} sync={self.num_syncs}] {category}: tensors={tensors} "
f"elements={elements} forward!=deployed={mismatches} max_abs={max_abs:.3e}",
flush=True,
)
4 changes: 2 additions & 2 deletions verl/utils/modelopt/qat_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ def _get_qat_field(qat_config, key, default=None):
return getattr(qat_config, key, default)


def apply_qat_to_modules(modules, qat_config):
def apply_qat_to_modules(modules, qat_config, deploy_path=None):
"""Apply ModelOpt fake quantization to a list of Megatron module chunks."""
from verl.utils.modelopt.quantize import apply_qat

Expand All @@ -33,7 +33,7 @@ def apply_qat_to_modules(modules, qat_config):
ignore_patterns = list(ignore_patterns)

for i in range(len(modules)):
modules[i] = apply_qat(modules[i], qat_mode, ignore_patterns=ignore_patterns)
modules[i] = apply_qat(modules[i], qat_mode, ignore_patterns=ignore_patterns, deploy_path=deploy_path)
return modules


Expand Down
Loading
Loading