|
| 1 | +"""Export the SAM3-LiteText MobileCLIP text encoder to ONNX. |
| 2 | +
|
| 3 | +SAM3-LiteText (arXiv:2602.12173) keeps the SAM3 ViT-H vision encoder, geometry |
| 4 | +encoder and mask decoder intact and only replaces the text encoder with a |
| 5 | +distilled MobileCLIP student. usls therefore reuses the existing SAM3 image |
| 6 | +vision/decoder ONNX (jamjamjon/assets `sam3` release) and only needs this |
| 7 | +lightweight text encoder. |
| 8 | +
|
| 9 | +The exported ONNX is a drop-in replacement for the SAM3 text encoder: |
| 10 | + inputs : input_ids[B, 32], attention_mask[B, 32] |
| 11 | + outputs: text_features[B, 32, 256], text_mask[B, 32] (bool, True = valid) |
| 12 | +
|
| 13 | +Variants (HuggingFace `vil-uob/sam3-litetext-{s0,s1,l}`): |
| 14 | + s0 -> MobileCLIP-S0, s1 -> MobileCLIP-S1, l -> MobileCLIP2-L |
| 15 | +
|
| 16 | +Usage: |
| 17 | + uv run export_text_encoder.py --model vil-uob/sam3-litetext-s0 --prefix sam3-litetext-s0 --precision fp32 |
| 18 | + uv run export_text_encoder.py --model vil-uob/sam3-litetext-s0 --prefix sam3-litetext-s0 --precision fp16 |
| 19 | +
|
| 20 | +fp16 conversion uses NVIDIA Model Optimizer AutoCast (precision-aware; keeps |
| 21 | +numerically unsafe nodes in fp32), which handles MobileCLIP's `.float()` |
| 22 | +LayerNorm/Softmax regions that onnxconverter_common cannot. |
| 23 | +""" |
| 24 | +from __future__ import annotations |
| 25 | + |
| 26 | +import argparse |
| 27 | +from pathlib import Path |
| 28 | + |
| 29 | +import numpy as np |
| 30 | +import onnx |
| 31 | +import onnxruntime as ort |
| 32 | +import torch |
| 33 | +import torch.nn as nn |
| 34 | +from transformers import AutoModel |
| 35 | + |
| 36 | + |
| 37 | +class LiteTextTextEncoder(nn.Module): |
| 38 | + """Wraps the HF model's text path into a plain-tensor ONNX interface.""" |
| 39 | + |
| 40 | + def __init__(self, model): |
| 41 | + super().__init__() |
| 42 | + self.model = model |
| 43 | + |
| 44 | + def forward(self, input_ids, attention_mask): |
| 45 | + text = self.model.get_text_features( |
| 46 | + input_ids=input_ids, attention_mask=attention_mask, return_dict=True |
| 47 | + ) |
| 48 | + # pooler_output is the per-token projected features [B, seq, 256]; |
| 49 | + # text_mask uses True = valid token (matches the SAM3 decoder). |
| 50 | + return text.pooler_output, attention_mask.bool() |
| 51 | + |
| 52 | + |
| 53 | +def main() -> None: |
| 54 | + ap = argparse.ArgumentParser(description=__doc__) |
| 55 | + ap.add_argument("--model", default="vil-uob/sam3-litetext-s0") |
| 56 | + ap.add_argument("--out-dir", default="onnx-sam3-litetext") |
| 57 | + ap.add_argument("--prefix", default="sam3-litetext-s0") |
| 58 | + ap.add_argument("--precision", choices=["fp32", "fp16"], default="fp32") |
| 59 | + ap.add_argument("--seq", type=int, default=32, help="text context length (fixed by the SAM3 decoder)") |
| 60 | + args = ap.parse_args() |
| 61 | + |
| 62 | + out_dir = Path(args.out_dir) |
| 63 | + out_dir.mkdir(parents=True, exist_ok=True) |
| 64 | + suffix = "-fp16" if args.precision == "fp16" else "" |
| 65 | + out = out_dir / f"{args.prefix}-text-encoder{suffix}.onnx" |
| 66 | + |
| 67 | + model = AutoModel.from_pretrained(args.model).eval() |
| 68 | + wrapper = LiteTextTextEncoder(model).eval() |
| 69 | + |
| 70 | + # Dummy prompt "dog": BOS, token, EOS, then EOS-padding to `seq`. |
| 71 | + ids = torch.full((1, args.seq), 49407, dtype=torch.long) |
| 72 | + ids[0, :3] = torch.tensor([49406, 1929, 49407]) |
| 73 | + attn = torch.ones(1, args.seq, dtype=torch.long) |
| 74 | + |
| 75 | + with torch.no_grad(): |
| 76 | + ref_tf, ref_tm = wrapper(ids, attn) |
| 77 | + print("torch text_features", tuple(ref_tf.shape), "text_mask", tuple(ref_tm.shape)) |
| 78 | + |
| 79 | + torch.onnx.export( |
| 80 | + wrapper, (ids, attn), str(out), |
| 81 | + input_names=["input_ids", "attention_mask"], |
| 82 | + output_names=["text_features", "text_mask"], |
| 83 | + opset_version=17, do_constant_folding=True, dynamo=False, |
| 84 | + dynamic_axes={"input_ids": {0: "batch"}, "attention_mask": {0: "batch"}, |
| 85 | + "text_features": {0: "batch"}, "text_mask": {0: "batch"}}, |
| 86 | + ) |
| 87 | + print("exported:", out) |
| 88 | + |
| 89 | + if args.precision == "fp16": |
| 90 | + from modelopt.onnx.autocast import convert_to_mixed_precision |
| 91 | + |
| 92 | + model_fp16 = convert_to_mixed_precision( |
| 93 | + str(out), low_precision_type="fp16", keep_io_types=True |
| 94 | + ) |
| 95 | + onnx.save(model_fp16, str(out)) |
| 96 | + print("converted to fp16 (modelopt AutoCast)") |
| 97 | + |
| 98 | + # Verify ONNX Runtime matches PyTorch. |
| 99 | + sess = ort.InferenceSession(str(out), providers=["CPUExecutionProvider"]) |
| 100 | + np_dtype = {"tensor(float)": np.float32, "tensor(float16)": np.float16, |
| 101 | + "tensor(int64)": np.int64, "tensor(bool)": np.bool_} |
| 102 | + feeds = {"input_ids": ids.numpy(), "attention_mask": attn.numpy()} |
| 103 | + cast = {i.name: feeds[i.name].astype(np_dtype[i.type]) for i in sess.get_inputs()} |
| 104 | + got = dict(zip([o.name for o in sess.get_outputs()], sess.run(None, cast))) |
| 105 | + r = ref_tf.numpy().astype(np.float64) |
| 106 | + g = np.asarray(got["text_features"], np.float64) |
| 107 | + diff = np.abs(r - g) |
| 108 | + cos = float(r.ravel() @ g.ravel() / (np.linalg.norm(r.ravel()) * np.linalg.norm(g.ravel()) + 1e-12)) |
| 109 | + print(f"verify text_features: max_abs={diff.max():.3e} mean_abs={diff.mean():.3e} cos={cos:.6f}") |
| 110 | + mism = int((np.asarray(got["text_mask"]).astype(bool) != ref_tm.numpy()).sum()) |
| 111 | + print(f"verify text_mask: mismatched={mism}") |
| 112 | + |
| 113 | + |
| 114 | +if __name__ == "__main__": |
| 115 | + main() |
0 commit comments