Skip to content

Commit 0f64eb6

Browse files
vividfclaude
andcommitted
quant: declarative quantization (plan/placement, modelopt engine, PTQ/QAT, self-describing ckpt)
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent 076ed7c commit 0f64eb6

32 files changed

Lines changed: 7506 additions & 1185 deletions

autoware_ml/builders/model_builder.py

Lines changed: 38 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222

2323
from autoware_ml.models.multi_task_base_model import MultiTaskBaseModel
2424
from autoware_ml.preprocessing.data_preprocessor import DataPreprocessor
25+
from autoware_ml.quantization.checkpoint import find_quantization
2526
from autoware_ml.utils.checkpoints import apply_matching_weights
2627

2728
logger = logging.getLogger(__name__)
@@ -50,7 +51,13 @@ def build_model(
5051
enforce_full_coverage: bool = False,
5152
) -> MultiTaskBaseModel:
5253
"""
53-
Build a model from the Hydra configuration.
54+
Build a model from the Hydra configuration and load its weights.
55+
56+
A quantized (PTQ / QAT) checkpoint describes itself: when one of ``weights_path``
57+
carries the embedded quantization description, the identical quantized module
58+
tree is rebuilt from it and verified before the weights load. Callers never need a
59+
``quantization`` config section — ``deploy`` and ``test`` score an INT8 checkpoint
60+
exactly like an FP one.
5461
5562
Args:
5663
cfg: Hydra configuration.
@@ -59,7 +66,8 @@ def build_model(
5966
weights_path: Path(s) to the weights file(s) to load into the model.
6067
resume_checkpoint_path: Path to the checkpoint file to resume training from.
6168
set_eval: Whether to set the model to evaluation mode after loading weights.
62-
enforce_full_coverage: Whether to enforce that all model parameters are covered by the weights.
69+
enforce_full_coverage: Whether to enforce that all model parameters are covered by
70+
the weights (always enforced for a quantized checkpoint).
6371
6472
Returns:
6573
Pytorch-Lightning MultiTaskBaseModel for multi-task learning/inference.
@@ -72,14 +80,35 @@ def build_model(
7280
raise ValueError("'--resume-checkpoint' and '--weights' are mutually exclusive.")
7381

7482
if weights_path is not None:
75-
apply_matching_weights(
76-
model,
77-
weights_path,
78-
map_location=device,
79-
logger=logger,
80-
enforce_full_coverage=enforce_full_coverage,
81-
set_eval=set_eval,
83+
weight_paths = (
84+
[Path(weights_path)]
85+
if isinstance(weights_path, (str, Path))
86+
else [Path(path) for path in weights_path]
8287
)
88+
quantized = find_quantization(weight_paths)
89+
if quantized is not None:
90+
from autoware_ml.quantization.loader import load_quantized_model
91+
92+
path, description = quantized
93+
logger.info(
94+
"Quantized checkpoint detected (%s, mode=%s): rebuilding the quantized tree "
95+
"from its embedded description.",
96+
path,
97+
description.config.mode,
98+
)
99+
load_quantized_model(model, weight_paths, description, device)
100+
if not set_eval:
101+
model.train()
102+
else:
103+
apply_matching_weights(
104+
model,
105+
weight_paths,
106+
map_location=device,
107+
device=device,
108+
logger=logger,
109+
enforce_full_coverage=enforce_full_coverage,
110+
set_eval=set_eval,
111+
)
83112

84113
if resume_checkpoint_path is not None:
85114
progress = torch.load(
Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
# Copyright 2026 TIER IV, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""
16+
Quantization framework (model-agnostic).
17+
18+
PTQ / QAT building blocks based on NVIDIA's modelopt toolkit, organized in layers:
19+
20+
- :mod:`~autoware_ml.quantization.plan` — the single interface between deployment
21+
stages and quantization: ``QuantRules`` (a model's declaration) + ``QuantizationPlan``
22+
(rules bound to config; ``prepare`` builds the tree AND records a ``PlacementRecord``
23+
of every placement decision).
24+
- :mod:`~autoware_ml.quantization.core` — model-agnostic engine on nvidia-modelopt
25+
(descriptor tables, in-place module conversion through modelopt's ``QuantModuleRegistry``,
26+
BN fusion, calibration, quantizer state).
27+
- :mod:`~autoware_ml.quantization.recipes` — architecture-specific Q/DQ placement as
28+
matcher+action recipes: quantized block classes selected by ``ResidualBlockSpec`` /
29+
``ESEBlockSpec`` rows (residual blocks, VoVNet eSE), plus the MaxPool input wrapper.
30+
- :mod:`~autoware_ml.quantization.config` — typed view of the Hydra ``quantization`` section.
31+
- :mod:`~autoware_ml.quantization.checkpoint` — self-describing quantized checkpoints (config +
32+
placement record embedded next to the ``state_dict``; no sidecar files).
33+
- :mod:`~autoware_ml.quantization.loader` — rebuild + verify + load from that description.
34+
- :mod:`~autoware_ml.quantization.qat_callback` — Lightning callback that turns a training run
35+
into frozen-amax QAT fine-tuning.
36+
37+
A model's quantization declaration (e.g. CenterPoint's ``QuantRules``) lives next to the model
38+
and is exposed through the model's ``build_quantization_plan()`` hook — the engine never imports
39+
a model.
40+
41+
The invariant every stage preserves: the quantize stage (PTQ / QAT) and the loader all build the
42+
quantized module tree by calling the *same* ``build_quantization_plan(config).prepare(model)``,
43+
so the calibrated ``state_dict`` and the later ``load_state_dict`` line up by construction —
44+
and the placement record embedded in the checkpoint lets the loader machine-check that instead
45+
of trusting it. Because the config travels inside the checkpoint, ``deploy`` and ``test`` need
46+
no ``quantization`` section at all.
47+
48+
The names exported here are the package's real external API. Deeper internals (descriptor
49+
tables, the single Conv-BN fold, the block registry) stay importable from their defining
50+
modules but are deliberately not re-exported.
51+
"""
52+
53+
from .checkpoint import (
54+
QUANTIZATION_KEY,
55+
QuantizationDescription,
56+
find_quantization,
57+
read_quantization,
58+
save_quantized_checkpoint,
59+
)
60+
from .config import CalibrationConfig, PTQConfig, QATConfig, QuantizationConfig
61+
from .core.calibration import Calibrator
62+
from .core.fusion import fuse_model_bn
63+
from .core.quantizer_state import (
64+
disable_quantizers_in,
65+
print_quantizer_status,
66+
quantizers_disabled,
67+
set_quantizers_enabled,
68+
validate_quantizer_amax,
69+
)
70+
from .core.replace import (
71+
expand_skip_quantize,
72+
match_skip_quantize_roots,
73+
replace_quantizable_modules,
74+
)
75+
from .loader import load_quantized_model
76+
from .plan import PlacementDecision, PlacementRecord, QuantizationPlan, QuantRules
77+
78+
__all__ = [
79+
# Typed config
80+
"QuantizationConfig",
81+
"CalibrationConfig",
82+
"PTQConfig",
83+
"QATConfig",
84+
# Plan (the single interface between deployment stages and quantization)
85+
"QuantRules",
86+
"QuantizationPlan",
87+
"PlacementRecord",
88+
"PlacementDecision",
89+
# Self-describing checkpoints
90+
"QUANTIZATION_KEY",
91+
"QuantizationDescription",
92+
"save_quantized_checkpoint",
93+
"read_quantization",
94+
"find_quantization",
95+
"load_quantized_model",
96+
# Replace / placement
97+
"replace_quantizable_modules",
98+
"expand_skip_quantize",
99+
"match_skip_quantize_roots",
100+
# Calibration
101+
"Calibrator",
102+
# Fusion
103+
"fuse_model_bn",
104+
# Quantizer state
105+
"disable_quantizers_in",
106+
"quantizers_disabled",
107+
"set_quantizers_enabled",
108+
"validate_quantizer_amax",
109+
"print_quantizer_status",
110+
]
Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
# Copyright 2026 TIER IV, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Self-describing quantized checkpoints.
16+
17+
A quantized checkpoint carries, next to its ``state_dict``, a ``quantization`` entry
18+
holding the :class:`~autoware_ml.quantization.config.QuantizationConfig` that built
19+
its tree and the :class:`~autoware_ml.quantization.plan.PlacementRecord` the
20+
build recorded. That is everything a later ``build_model`` needs to rebuild the
21+
identical quantized tree and verify it — so ``deploy`` and ``test`` never read a
22+
``quantization`` config section, and PTQ and QAT checkpoints (a Lightning
23+
checkpoint with the same entry) load through one path.
24+
25+
There are no sidecar files: the calibrated ``_amax`` buffers live in the
26+
``state_dict`` like any other buffer.
27+
"""
28+
29+
from __future__ import annotations
30+
31+
import logging
32+
from collections.abc import Mapping, Sequence
33+
from dataclasses import dataclass
34+
from pathlib import Path
35+
from typing import Any
36+
37+
import torch
38+
39+
from autoware_ml.quantization.config import QuantizationConfig
40+
from autoware_ml.quantization.plan import PlacementRecord
41+
42+
logger = logging.getLogger(__name__)
43+
44+
#: Top-level checkpoint key holding the quantization description.
45+
QUANTIZATION_KEY = "quantization"
46+
47+
48+
@dataclass(frozen=True)
49+
class QuantizationDescription:
50+
"""What a quantized checkpoint says about itself.
51+
52+
The embedded format carries no version field on purpose: checkpoints are
53+
reproducible artifacts (re-run ``autoware-ml quantize``), and a format drift
54+
fails loudly anyway — ``QuantizationConfig.from_dict`` rejects unknown keys and
55+
a missing key raises here.
56+
"""
57+
58+
config: QuantizationConfig
59+
placement_record: PlacementRecord
60+
61+
def to_payload(self) -> dict[str, Any]:
62+
"""Serialize for embedding under :data:`QUANTIZATION_KEY`."""
63+
return {
64+
"config": self.config.to_dict(),
65+
"placement_record": self.placement_record.to_json_dict(),
66+
}
67+
68+
@classmethod
69+
def from_payload(cls, payload: Mapping[str, Any]) -> QuantizationDescription:
70+
"""Deserialize an embedded payload.
71+
72+
Raises:
73+
KeyError: When the payload does not have this build's layout — the
74+
checkpoint predates a format change; re-produce it with
75+
``autoware-ml quantize``.
76+
"""
77+
return cls(
78+
config=QuantizationConfig.from_dict(payload["config"]),
79+
placement_record=PlacementRecord.from_json_dict(payload["placement_record"]),
80+
)
81+
82+
83+
def attach_quantization(checkpoint: dict[str, Any], description: QuantizationDescription) -> None:
84+
"""Embed ``description`` into a checkpoint dict in place (used by ``on_save_checkpoint``)."""
85+
checkpoint[QUANTIZATION_KEY] = description.to_payload()
86+
87+
88+
def save_quantized_checkpoint(
89+
model: torch.nn.Module, path: str | Path, description: QuantizationDescription
90+
) -> Path:
91+
"""Write ``{"state_dict", "quantization"}`` — the PTQ producer's output.
92+
93+
The layout is a subset of a Lightning checkpoint, so PTQ and QAT checkpoints read
94+
identically.
95+
"""
96+
path = Path(path)
97+
path.parent.mkdir(parents=True, exist_ok=True)
98+
checkpoint: dict[str, Any] = {"state_dict": model.state_dict()}
99+
attach_quantization(checkpoint, description)
100+
torch.save(checkpoint, path)
101+
logger.info(
102+
"Saved quantized checkpoint: %s (%d decisions in the embedded placement record)",
103+
path,
104+
len(description.placement_record),
105+
)
106+
return path
107+
108+
109+
def read_quantization(checkpoint: Mapping[str, Any]) -> QuantizationDescription | None:
110+
"""Return the embedded description of a loaded checkpoint dict, or ``None`` for an FP one."""
111+
payload = checkpoint.get(QUANTIZATION_KEY)
112+
if payload is None:
113+
return None
114+
return QuantizationDescription.from_payload(payload)
115+
116+
117+
def read_quantization_from_file(path: str | Path) -> QuantizationDescription | None:
118+
"""Return the embedded description of a checkpoint file, or ``None`` for an FP one.
119+
120+
Only the payload is inspected; tensors are memory-mapped, not materialized.
121+
"""
122+
checkpoint = torch.load(str(path), map_location="cpu", weights_only=False, mmap=True)
123+
return read_quantization(checkpoint)
124+
125+
126+
def find_quantization(
127+
weight_paths: Sequence[str | Path],
128+
) -> tuple[Path, QuantizationDescription] | None:
129+
"""Find the one quantized checkpoint among ``weight_paths``.
130+
131+
Returns:
132+
``(path, description)`` of the quantized checkpoint, or ``None`` when every
133+
checkpoint is a plain FP one.
134+
135+
Raises:
136+
ValueError: When more than one checkpoint is quantized — a quantized tree is a
137+
whole-model construction; merging two of them is not defined.
138+
"""
139+
found = [
140+
(Path(path), description)
141+
for path in weight_paths
142+
if (description := read_quantization_from_file(path)) is not None
143+
]
144+
if not found:
145+
return None
146+
if len(found) > 1:
147+
raise ValueError(
148+
"More than one --weights checkpoint is quantized "
149+
f"({[str(p) for p, _ in found]}); a quantized model loads from exactly one."
150+
)
151+
return found[0]

0 commit comments

Comments
 (0)