Skip to content

Commit e7746f3

Browse files
committed
fix(wan_i2v): repeat conditioning images along batch dimension in prepare_latents
- Repeat image and last_image along batch dimension in WanPipelineI2V_2_1 and WanPipelineI2V_2_2 to match effective_batch_size, resolving concatenation dimension mismatch when batch_size > 1. - Validate divisibility of batch_size against image and last_image batch sizes. - Validate alignment between image and last_image batch sizes when both are greater than 1. - Add focused unit tests in wan_i2v_prepare_latents_test.py.
1 parent 8e3e843 commit e7746f3

3 files changed

Lines changed: 221 additions & 4 deletions

File tree

src/maxdiffusion/pipelines/wan/wan_pipeline_i2v_2p1.py

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,11 @@ def prepare_latents(
9797
num_videos_per_prompt: int = 1,
9898
trace: Optional[dict] = None,
9999
) -> Tuple[jax.Array, jax.Array, Optional[jax.Array]]:
100+
"""Prepares latents and condition for I2V inference.
101+
102+
Note: num_videos_per_prompt is kept for backwards compatibility; repetition
103+
is driven by batch_size.
104+
"""
100105
if hasattr(image, "detach"):
101106
image = image.detach().cpu().numpy()
102107
image = jnp.array(image)
@@ -106,10 +111,22 @@ def prepare_latents(
106111
last_image = last_image.detach().cpu().numpy()
107112
last_image = jnp.array(last_image)
108113

109-
if num_videos_per_prompt > 1:
110-
image = jnp.repeat(image, num_videos_per_prompt, axis=0)
111-
if last_image is not None:
112-
last_image = jnp.repeat(last_image, num_videos_per_prompt, axis=0)
114+
if batch_size % image.shape[0] != 0:
115+
raise ValueError(f"Batch size ({batch_size}) must be divisible by image batch size ({image.shape[0]}).")
116+
117+
if last_image is not None:
118+
if batch_size % last_image.shape[0] != 0:
119+
raise ValueError(f"Batch size ({batch_size}) must be divisible by last_image batch size ({last_image.shape[0]}).")
120+
if image.shape[0] > 1 and last_image.shape[0] > 1 and image.shape[0] != last_image.shape[0]:
121+
raise ValueError(
122+
f"image batch size ({image.shape[0]}) and last_image batch size ({last_image.shape[0]}) must match when"
123+
" both are greater than 1."
124+
)
125+
if last_image.shape[0] < batch_size:
126+
last_image = jnp.repeat(last_image, batch_size // last_image.shape[0], axis=0)
127+
128+
if image.shape[0] < batch_size:
129+
image = jnp.repeat(image, batch_size // image.shape[0], axis=0)
113130

114131
num_channels_latents = self.vae.z_dim
115132
num_latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1

src/maxdiffusion/pipelines/wan/wan_pipeline_i2v_2p2.py

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,11 @@ def prepare_latents(
149149
num_videos_per_prompt: int = 1,
150150
trace: Optional[dict] = None,
151151
) -> Tuple[jax.Array, jax.Array, Optional[jax.Array]]:
152+
"""Prepares latents and condition for I2V inference.
153+
154+
Note: num_videos_per_prompt is kept for backwards compatibility; repetition
155+
is driven by batch_size.
156+
"""
152157
if hasattr(image, "detach"):
153158
image = image.detach().cpu().numpy()
154159
image = jnp.array(image)
@@ -158,6 +163,23 @@ def prepare_latents(
158163
last_image = last_image.detach().cpu().numpy()
159164
last_image = jnp.array(last_image)
160165

166+
if batch_size % image.shape[0] != 0:
167+
raise ValueError(f"Batch size ({batch_size}) must be divisible by image batch size ({image.shape[0]}).")
168+
169+
if last_image is not None:
170+
if batch_size % last_image.shape[0] != 0:
171+
raise ValueError(f"Batch size ({batch_size}) must be divisible by last_image batch size ({last_image.shape[0]}).")
172+
if image.shape[0] > 1 and last_image.shape[0] > 1 and image.shape[0] != last_image.shape[0]:
173+
raise ValueError(
174+
f"image batch size ({image.shape[0]}) and last_image batch size ({last_image.shape[0]}) must match when"
175+
" both are greater than 1."
176+
)
177+
if last_image.shape[0] < batch_size:
178+
last_image = jnp.repeat(last_image, batch_size // last_image.shape[0], axis=0)
179+
180+
if image.shape[0] < batch_size:
181+
image = jnp.repeat(image, batch_size // image.shape[0], axis=0)
182+
161183
num_channels_latents = self.vae.z_dim
162184
num_latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1
163185
latent_height = height // self.vae_scale_factor_spatial
Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
"""
2+
Copyright 2026 Google LLC
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
https://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
"""
16+
17+
import unittest
18+
from unittest.mock import MagicMock
19+
import jax
20+
import jax.numpy as jnp
21+
22+
from maxdiffusion.pipelines.wan.wan_pipeline_i2v_2p1 import WanPipelineI2V_2_1
23+
from maxdiffusion.pipelines.wan.wan_pipeline_i2v_2p2 import WanPipelineI2V_2_2
24+
25+
26+
class WanI2VPrepareLatentsTest(unittest.TestCase):
27+
28+
def _create_mock_pipeline(self, pipeline_cls):
29+
"""Creates a mock pipeline instance with required VAE attributes."""
30+
pipeline = object.__new__(pipeline_cls)
31+
pipeline.vae = MagicMock(z_dim=16)
32+
pipeline.vae_scale_factor_temporal = 4
33+
pipeline.vae_scale_factor_spatial = 8
34+
35+
def mock_prepare_latents_i2v_base(image, num_frames, dtype, last_image=None, trace=None):
36+
num_latent_frames = (num_frames - 1) // pipeline.vae_scale_factor_temporal + 1
37+
latent_height = 32 // pipeline.vae_scale_factor_spatial
38+
latent_width = 32 // pipeline.vae_scale_factor_spatial
39+
latent_condition = jnp.zeros(
40+
(image.shape[0], num_latent_frames, latent_height, latent_width, pipeline.vae.z_dim),
41+
dtype=dtype,
42+
)
43+
return latent_condition, None
44+
45+
pipeline.prepare_latents_i2v_base = MagicMock(side_effect=mock_prepare_latents_i2v_base)
46+
return pipeline
47+
48+
def test_single_image_repetition(self):
49+
"""Verifies that a single conditioning image is repeated when batch_size > 1."""
50+
rng = jax.random.key(0)
51+
for pipeline_cls in (WanPipelineI2V_2_1, WanPipelineI2V_2_2):
52+
with self.subTest(pipeline=pipeline_cls.__name__):
53+
pipeline = self._create_mock_pipeline(pipeline_cls)
54+
image = jnp.zeros((1, 3, 32, 32))
55+
batch_size = 4
56+
latents, condition, _ = pipeline.prepare_latents(
57+
image=image,
58+
batch_size=batch_size,
59+
height=32,
60+
width=32,
61+
num_frames=5,
62+
dtype=jnp.float32,
63+
rng=rng,
64+
)
65+
self.assertEqual(latents.shape[0], batch_size)
66+
self.assertEqual(condition.shape[0], batch_size)
67+
call_image = pipeline.prepare_latents_i2v_base.call_args[0][0]
68+
self.assertEqual(call_image.shape[0], batch_size)
69+
70+
def test_batched_image_repetition(self):
71+
"""Verifies that multiple conditioning images are repeated correctly when divisible."""
72+
rng = jax.random.key(0)
73+
for pipeline_cls in (WanPipelineI2V_2_1, WanPipelineI2V_2_2):
74+
with self.subTest(pipeline=pipeline_cls.__name__):
75+
pipeline = self._create_mock_pipeline(pipeline_cls)
76+
image = jnp.zeros((2, 3, 32, 32))
77+
batch_size = 4
78+
latents, condition, _ = pipeline.prepare_latents(
79+
image=image,
80+
batch_size=batch_size,
81+
height=32,
82+
width=32,
83+
num_frames=5,
84+
dtype=jnp.float32,
85+
rng=rng,
86+
)
87+
self.assertEqual(latents.shape[0], batch_size)
88+
self.assertEqual(condition.shape[0], batch_size)
89+
call_image = pipeline.prepare_latents_i2v_base.call_args[0][0]
90+
self.assertEqual(call_image.shape[0], batch_size)
91+
92+
def test_with_last_image(self):
93+
"""Verifies that both start and last images are repeated when provided."""
94+
rng = jax.random.key(0)
95+
for pipeline_cls in (WanPipelineI2V_2_1, WanPipelineI2V_2_2):
96+
with self.subTest(pipeline=pipeline_cls.__name__):
97+
pipeline = self._create_mock_pipeline(pipeline_cls)
98+
image = jnp.zeros((1, 3, 32, 32))
99+
last_image = jnp.zeros((1, 3, 32, 32))
100+
batch_size = 3
101+
latents, condition, _ = pipeline.prepare_latents(
102+
image=image,
103+
batch_size=batch_size,
104+
height=32,
105+
width=32,
106+
num_frames=5,
107+
dtype=jnp.float32,
108+
rng=rng,
109+
last_image=last_image,
110+
)
111+
self.assertEqual(latents.shape[0], batch_size)
112+
self.assertEqual(condition.shape[0], batch_size)
113+
call_image = pipeline.prepare_latents_i2v_base.call_args[0][0]
114+
call_last_image = pipeline.prepare_latents_i2v_base.call_args[0][3]
115+
self.assertEqual(call_image.shape[0], batch_size)
116+
self.assertEqual(call_last_image.shape[0], batch_size)
117+
118+
def test_indivisible_image_batch_size_raises(self):
119+
"""Verifies ValueError when batch_size is not divisible by image batch size."""
120+
rng = jax.random.key(0)
121+
for pipeline_cls in (WanPipelineI2V_2_1, WanPipelineI2V_2_2):
122+
with self.subTest(pipeline=pipeline_cls.__name__):
123+
pipeline = self._create_mock_pipeline(pipeline_cls)
124+
image = jnp.zeros((2, 3, 32, 32))
125+
with self.assertRaisesRegex(ValueError, "divisible by image batch size"):
126+
pipeline.prepare_latents(
127+
image=image,
128+
batch_size=3,
129+
height=32,
130+
width=32,
131+
num_frames=5,
132+
dtype=jnp.float32,
133+
rng=rng,
134+
)
135+
136+
def test_indivisible_last_image_batch_size_raises(self):
137+
"""Verifies ValueError when batch_size is not divisible by last_image batch size."""
138+
rng = jax.random.key(0)
139+
for pipeline_cls in (WanPipelineI2V_2_1, WanPipelineI2V_2_2):
140+
with self.subTest(pipeline=pipeline_cls.__name__):
141+
pipeline = self._create_mock_pipeline(pipeline_cls)
142+
image = jnp.zeros((1, 3, 32, 32))
143+
last_image = jnp.zeros((2, 3, 32, 32))
144+
with self.assertRaisesRegex(ValueError, "divisible by last_image batch size"):
145+
pipeline.prepare_latents(
146+
image=image,
147+
batch_size=3,
148+
height=32,
149+
width=32,
150+
num_frames=5,
151+
dtype=jnp.float32,
152+
rng=rng,
153+
last_image=last_image,
154+
)
155+
156+
def test_mismatched_image_and_last_image_batch_sizes_raises(self):
157+
"""Verifies ValueError when image and last_image have conflicting batch sizes > 1."""
158+
rng = jax.random.key(0)
159+
for pipeline_cls in (WanPipelineI2V_2_1, WanPipelineI2V_2_2):
160+
with self.subTest(pipeline=pipeline_cls.__name__):
161+
pipeline = self._create_mock_pipeline(pipeline_cls)
162+
image = jnp.zeros((2, 3, 32, 32))
163+
last_image = jnp.zeros((3, 3, 32, 32))
164+
with self.assertRaisesRegex(ValueError, "must match when both are greater than 1"):
165+
pipeline.prepare_latents(
166+
image=image,
167+
batch_size=6,
168+
height=32,
169+
width=32,
170+
num_frames=5,
171+
dtype=jnp.float32,
172+
rng=rng,
173+
last_image=last_image,
174+
)
175+
176+
177+
if __name__ == "__main__":
178+
unittest.main()

0 commit comments

Comments
 (0)