Skip to content

feat: add Wan2.2 Animate 2 video models #5309

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Minamiyama wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: add Wan2.2 Animate 2 video models #5309

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions xinference/api/restful_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2474,6 +2474,7 @@ async def create_videos_from_images(
request: Request,
model: str = Form(...),
image: UploadFile = File(media_type="application/octet-stream"),
video: Optional[UploadFile] = File(None, media_type="application/octet-stream"),
prompt: Optional[Union[str, List[str]]] = Form(None),
negative_prompt: Optional[Union[str, List[str]]] = Form(None),
n: Optional[int] = Form(1),
Expand All @@ -2493,6 +2494,8 @@ async def create_videos_from_images(
parsed_kwargs = json.loads(kwargs)
else:
parsed_kwargs = {}
if video is not None:
parsed_kwargs["video"] = await video.read()
request_id = parsed_kwargs.get("request_id")
self._add_running_task(request_id)
video_list = await model_ref.image_to_video(
Expand Down
15 changes: 15 additions & 0 deletions xinference/client/restful/async_restful_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -666,6 +666,7 @@ async def image_to_video(
prompt: str,
negative_prompt: Optional[str] = None,
n: int = 1,
video: Optional[Union[str, bytes]] = None,
**kwargs,
) -> "VideoList":
"""
Expand All @@ -681,6 +682,8 @@ async def image_to_video(
The prompt or prompts not to guide the image generation.
n: `int`, defaults to 1
The number of videos to generate per prompt. Must be between 1 and 10.
video: `Union[str, bytes]`, optional
The driving video for character animation models.
Returns
-------
VideoList
Expand All @@ -699,6 +702,18 @@ async def image_to_video(
for key, value in params.items():
files.append((key, (None, value)))
files.append(("image", ("image", image, "application/octet-stream")))
if video is not None:
if isinstance(video, str):
with open(video, "rb") as f:
video_data = f.read()
else:
video_data = video
files.append(
(
"video",
("video", video_data, "application/octet-stream"),
)
)
Comment thread
Minamiyama marked this conversation as resolved.
response = await self.session.post(url, data=files, headers=self.auth_headers)
if response.status != 200:
raise RuntimeError(
Expand Down
15 changes: 15 additions & 0 deletions xinference/client/restful/restful_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -599,6 +599,7 @@ def image_to_video(
prompt: str,
negative_prompt: Optional[str] = None,
n: int = 1,
video: Optional[Union[str, bytes]] = None,
**kwargs,
) -> "VideoList":
"""
Expand All @@ -614,6 +615,8 @@ def image_to_video(
The prompt or prompts not to guide the image generation.
n: `int`, defaults to 1
The number of videos to generate per prompt. Must be between 1 and 10.
video: `Union[str, bytes]`, optional
The driving video for character animation models.
Returns
-------
VideoList
Expand All @@ -631,6 +634,18 @@ def image_to_video(
for key, value in params.items():
files.append((key, (None, value)))
files.append(("image", ("image", image, "application/octet-stream")))
if video is not None:
if isinstance(video, str):
with open(video, "rb") as f:
video_data = f.read()
else:
video_data = video
files.append(
(
"video",
("video", video_data, "application/octet-stream"),
)
)
Comment thread
Minamiyama marked this conversation as resolved.
response = self.session.post(url, files=files, headers=self.auth_headers)
if response.status_code != 200:
raise RuntimeError(
Expand Down
50 changes: 49 additions & 1 deletion xinference/model/video/diffusers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import logging
import operator
import os
import tempfile
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
Expand Down Expand Up @@ -156,6 +157,12 @@ def load(self):
pipeline = self._model = HunyuanVideoPipeline.from_pretrained(
self._model_path, transformer=transformer, **kwargs
)
elif self.model_spec.model_family == "WanAnimate2":
from diffusers import WanAnimate2Pipeline

pipeline = self._model = WanAnimate2Pipeline.from_pretrained(
self._model_path, **kwargs
)
elif self.model_spec.model_family == "Wan":
from diffusers import AutoencoderKLWan, WanImageToVideoPipeline, WanPipeline
from transformers import CLIPVisionModel
Expand Down Expand Up @@ -312,9 +319,50 @@ def image_to_video(
assert callable(self._model)
generate_kwargs = self._model_spec.default_generate_config.copy()
generate_kwargs.update(kwargs)
generate_kwargs["num_videos_per_prompt"] = n
if num_inference_steps:
generate_kwargs["num_inference_steps"] = num_inference_steps

if self.model_spec.model_family == "WanAnimate2":
if n != 1:
raise ValueError(
"Wan-Animate-2 only supports generating one video per request"
)

video = generate_kwargs.pop("video", None)
if video is None:
raise ValueError("`video` is required for Wan-Animate-2")

fps = generate_kwargs.get("fps", 24)
temp_video_path = None
output: Any
try:
if isinstance(video, bytes):
with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as f:
temp_video_path = f.name
f.write(video)
video = temp_video_path
elif isinstance(video, str):
if not os.path.isfile(video):
raise FileNotFoundError(
f"Video path does not exist or is not a file: {video}"
)
else:
raise TypeError("`video` must be video bytes or a local path")
Comment on lines +335 to +350

@gemini-code-assist gemini-code-assist Bot Aug 12, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

There are three issues in this block:

  1. fps is retrieved using get but not popped from generate_kwargs. Since fps is not a valid parameter for the underlying WanAnimate2Pipeline.__call__ method, leaving it in generate_kwargs will cause a TypeError crash.
  2. If f.write(video) fails, temp_video_path remains None, leaving an orphaned temporary file on disk. Setting temp_video_path before writing ensures it is cleaned up in the finally block.
  3. If video is a string path, we should check if the file exists to fail fast with a clear error.
            fps = generate_kwargs.pop("fps", 24)
            temp_video_path = None
            try:
                if isinstance(video, bytes):
                    with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as f:
                        temp_video_path = f.name
                        f.write(video)
                    video = temp_video_path
                elif isinstance(video, str):
                    if not os.path.exists(video):
                        raise FileNotFoundError(f"Driving video path does not exist: {video}")
                else:
                    raise TypeError("`video` must be video bytes or a local path")
References
  1. Avoid adding unused parameters to launch or configuration arguments if they are not directly consumed, as they can leak through **kwargs into downstream model construction. Instead, clean up stale or internal fields and overwrite only the required parameters.


self._process_progressor(generate_kwargs)
output = self._model(
image=image,
driving_video=video,
prompt=prompt,
**generate_kwargs,
)
finally:
if temp_video_path and os.path.exists(temp_video_path):
os.remove(temp_video_path)

return self._output_to_video(output, fps, response_format)

generate_kwargs["num_videos_per_prompt"] = n
fps = generate_kwargs.pop("fps", 10)

# process image
Expand Down
92 changes: 92 additions & 0 deletions xinference/model/video/model_spec.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -418,5 +418,97 @@
},
"updated_at": 1766385310,
"featured": false
},
{
"version": 2,
"model_name": "Wan2.2-Animate-2-14B",
"model_family": "WanAnimate2",
"model_ability": [
"image2video"
],
"default_model_config": {
"torch_dtype": "bfloat16"
},
"default_generate_config": {
"width": 640,
"height": 800,
"fps": 24,
"num_inference_steps": 40,
"guidance_scale": 3.0,
"flow_solver": "dpm"
},
"virtualenv": {
"packages": [
"diffusers @ git+https://github.com/huggingface/diffusers.git@refs/pull/14412/head",
"transformers>=4.57.6",
"accelerate>=1.13.0",
"flash-attn",
"decord==0.6.0",
"opencv-python",
"ftfy",
"imageio-ffmpeg",
"imageio",
"#system_numpy#"
],
"no_build_isolation": true
},
"model_src": {
"huggingface": {
"model_id": "Wan-AI/Wan2.2-Animate-2-14B-Diffusers",
"model_revision": "main"
},
"modelscope": {
"model_id": "Wan-AI/Wan2.2-Animate-2-14B-Diffusers",
"model_revision": "master"
}
},
"updated_at": 1786492826,
"featured": false
},
{
"version": 2,
"model_name": "Wan2.2-Animate-2-14B-Distilled",
"model_family": "WanAnimate2",
"model_ability": [
"image2video"
],
"default_model_config": {
"torch_dtype": "bfloat16"
},
"default_generate_config": {
"width": 640,
"height": 800,
"fps": 24,
"num_inference_steps": 10,
"guidance_scale": 1.0,
"flow_solver": "euler"
},
"virtualenv": {
"packages": [
"diffusers @ git+https://github.com/huggingface/diffusers.git@refs/pull/14412/head",
"transformers>=4.57.6",
"accelerate>=1.13.0",
"flash-attn",
"decord==0.6.0",
"opencv-python",
"ftfy",
"imageio-ffmpeg",
"imageio",
"#system_numpy#"
],
"no_build_isolation": true
},
"model_src": {
"huggingface": {
"model_id": "Wan-AI/Wan2.2-Animate-2-14B-Distilled-Diffusers",
"model_revision": "main"
},
"modelscope": {
"model_id": "Wan-AI/Wan2.2-Animate-2-14B-Distilled-Diffusers",
"model_revision": "master"
}
},
"updated_at": 1786492826,
"featured": false
}
]
Loading