Skip to content

Commit 59cbb97

Browse files
committed
[test] AnyFlow embedder tests: initialize ReplicatedLinear weights for CPU runs
FastVideo's ReplicatedLinear allocates weights via torch.empty and relies on a downstream load_weights pass to populate them. CPU unit tests bypass that pass, so weights start as NaN/Inf and the embedder forward produces NaN everywhere. Add _init_uninitialized_weights that Xavier-init's every >=2D param and zeros the rest before .eval(). Verified on GMI (gpu-h200-06 with 8x H200): all 43 tests pass.
1 parent 5a5c085 commit 59cbb97

1 file changed

Lines changed: 20 additions & 0 deletions

File tree

fastvideo/tests/training/distill/test_anyflow_pretrain.py

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,12 +42,31 @@ def test_wan_arch_defaults_preserve_bit_identity() -> None:
4242
# ---------------------------------------------------------------------------
4343

4444

45+
def _init_uninitialized_weights(module: torch.nn.Module, seed: int = 0) -> None:
46+
"""FastVideo's ``ReplicatedLinear`` allocates weights with
47+
``torch.empty`` and relies on a downstream ``load_weights`` pass to
48+
populate them. Unit tests bypass that pass, so weights start as
49+
uninitialized garbage (typically NaN/Inf). Manually init every
50+
Linear / RMSNorm / LayerNorm parameter so the forward produces
51+
deterministic finite outputs.
52+
"""
53+
torch.manual_seed(seed)
54+
with torch.no_grad():
55+
for p in module.parameters():
56+
if p.ndim >= 2:
57+
# Xavier-uniform scaled by inverse fan-in for stable forwards.
58+
torch.nn.init.xavier_uniform_(p)
59+
else:
60+
p.zero_()
61+
62+
4563
def _make_embedder(
4664
*,
4765
r_embedder: bool,
4866
fusion: str = "additive",
4967
gate: float = 0.25,
5068
deltatime_type: str = "r",
69+
init_seed: int = 0,
5170
):
5271
from fastvideo.models.dits.wanvideo import WanTimeTextImageEmbedding
5372

@@ -61,6 +80,7 @@ def _make_embedder(
6180
r_embedder_gate_value=gate,
6281
r_embedder_deltatime_type=deltatime_type,
6382
)
83+
_init_uninitialized_weights(emb, seed=init_seed)
6484
emb.eval()
6585
return emb
6686

0 commit comments

Comments
 (0)