feat: add Wan2.2 Animate 2 video models

Author: Minamiyama

xinference/api/restful_api.py

@@ -2474,6 +2474,7 @@ async def create_videos_from_images(
         request: Request,
         model: str = Form(...),
         image: UploadFile = File(media_type="application/octet-stream"),
+        video: Optional[UploadFile] = File(None, media_type="application/octet-stream"),
         prompt: Optional[Union[str, List[str]]] = Form(None),
         negative_prompt: Optional[Union[str, List[str]]] = Form(None),
         n: Optional[int] = Form(1),
@@ -2493,6 +2494,8 @@ async def create_videos_from_images(
                 parsed_kwargs = json.loads(kwargs)
             else:
                 parsed_kwargs = {}
+            if video is not None:
+                parsed_kwargs["video"] = await video.read()
             request_id = parsed_kwargs.get("request_id")
             self._add_running_task(request_id)
             video_list = await model_ref.image_to_video(

xinference/client/restful/async_restful_client.py

@@ -666,6 +666,7 @@ async def image_to_video(
         prompt: str,
         negative_prompt: Optional[str] = None,
         n: int = 1,
+        video: Optional[Union[str, bytes]] = None,
         **kwargs,
     ) -> "VideoList":
         """
@@ -681,6 +682,8 @@ async def image_to_video(
             The prompt or prompts not to guide the image generation.
         n: `int`, defaults to 1
             The number of videos to generate per prompt. Must be between 1 and 10.
+        video: `Union[str, bytes]`, optional
+            The driving video for character animation models.
         Returns
         -------
         VideoList
@@ -699,6 +702,18 @@ async def image_to_video(
         for key, value in params.items():
             files.append((key, (None, value)))
         files.append(("image", ("image", image, "application/octet-stream")))
+        if video is not None:
+            if isinstance(video, str):
+                with open(video, "rb") as f:
+                    video_data = f.read()
+            else:
+                video_data = video
+            files.append(
+                (
+                    "video",
+                    ("video", video_data, "application/octet-stream"),
+                )
+            )
         response = await self.session.post(url, data=files, headers=self.auth_headers)
         if response.status != 200:
             raise RuntimeError(

xinference/client/restful/restful_client.py

@@ -599,6 +599,7 @@ def image_to_video(
         prompt: str,
         negative_prompt: Optional[str] = None,
         n: int = 1,
+        video: Optional[Union[str, bytes]] = None,
         **kwargs,
     ) -> "VideoList":
         """
@@ -614,6 +615,8 @@ def image_to_video(
             The prompt or prompts not to guide the image generation.
         n: `int`, defaults to 1
             The number of videos to generate per prompt. Must be between 1 and 10.
+        video: `Union[str, bytes]`, optional
+            The driving video for character animation models.
         Returns
         -------
         VideoList
@@ -631,6 +634,18 @@ def image_to_video(
         for key, value in params.items():
             files.append((key, (None, value)))
         files.append(("image", ("image", image, "application/octet-stream")))
+        if video is not None:
+            if isinstance(video, str):
+                with open(video, "rb") as f:
+                    video_data = f.read()
+            else:
+                video_data = video
+            files.append(
+                (
+                    "video",
+                    ("video", video_data, "application/octet-stream"),
+                )
+            )
         response = self.session.post(url, files=files, headers=self.auth_headers)
         if response.status_code != 200:
             raise RuntimeError(

xinference/client/tests/test_async_client.py

@@ -23,6 +23,7 @@
 from ..restful.async_restful_client import (
     AsyncRESTfulChatModelHandle,
     AsyncRESTfulEmbeddingModelHandle,
+    AsyncRESTfulVideoModelHandle,
 )
 
 
@@ -42,9 +43,11 @@ async def wait_for_close(self):
 class _DummyAsyncSession:
     def __init__(self):
         self.last_json = None
+        self.last_data = None
 
-    async def post(self, url, json=None, headers=None):
+    async def post(self, url, json=None, headers=None, data=None):
         self.last_json = json
+        self.last_data = data
         return _DummyAsyncResponse()
 
     async def close(self):
@@ -713,6 +716,24 @@ async def test_async_restful_chat_enable_thinking_injected():
     handle.session = None
 
 
+@pytest.mark.asyncio
+async def test_async_restful_video_reads_video_path(tmp_path):
+    video_path = tmp_path / "driving.mp4"
+    video_path.write_bytes(b"video-content")
+    handle = AsyncRESTfulVideoModelHandle.__new__(AsyncRESTfulVideoModelHandle)
+    handle._model_uid = "test-model"
+    handle._base_url = "http://localhost"
+    handle.auth_headers = {}
+    dummy_session = _DummyAsyncSession()
+    handle.session = dummy_session
+
+    await handle.image_to_video(b"image-content", "prompt", video=str(video_path))
+
+    video_file = next(item for item in dummy_session.last_data if item[0] == "video")
+    assert video_file[1][1] == b"video-content"
+    handle.session = None
+
+
 @pytest.mark.asyncio
 async def test_add_model_replica_sends_empty_json_body():
     client = AsyncRESTfulClient.__new__(AsyncRESTfulClient)

xinference/client/tests/test_client.py

@@ -26,6 +26,7 @@
 from ..restful.restful_client import (
     RESTfulChatModelHandle,
     RESTfulEmbeddingModelHandle,
+    RESTfulVideoModelHandle,
     _get_error_string,
 )
 
@@ -40,9 +41,11 @@ def json(self):
 class _DummySession:
     def __init__(self):
         self.last_json = None
+        self.last_files = None
 
-    def post(self, url, json=None, stream=None, headers=None):
+    def post(self, url, json=None, stream=None, headers=None, files=None):
         self.last_json = json
+        self.last_files = files
         return _DummyResponse()
 
     def close(self):
@@ -638,6 +641,20 @@ def test_restful_chat_enable_thinking_injected():
     handle.session = None
 
 
+def test_restful_video_reads_video_path(tmp_path):
+    video_path = tmp_path / "driving.mp4"
+    video_path.write_bytes(b"video-content")
+    handle = RESTfulVideoModelHandle("test-model", "http://localhost", {})
+    dummy_session = _DummySession()
+    handle.session = dummy_session
+
+    handle.image_to_video(b"image-content", "prompt", video=str(video_path))
+
+    video_file = next(item for item in dummy_session.last_files if item[0] == "video")
+    assert video_file[1][1] == b"video-content"
+    handle.session = None
+
+
 def test_add_model_replica_sends_empty_json_body():
     client = RESTfulClient.__new__(RESTfulClient)
     client.base_url = "http://localhost:9997"

xinference/model/video/diffusers.py

@@ -18,6 +18,7 @@
 import logging
 import operator
 import os
+import tempfile
 import time
 import uuid
 from concurrent.futures import ThreadPoolExecutor
@@ -156,6 +157,12 @@ def load(self):
             pipeline = self._model = HunyuanVideoPipeline.from_pretrained(
                 self._model_path, transformer=transformer, **kwargs
             )
+        elif self.model_spec.model_family == "WanAnimate2":
+            from diffusers import WanAnimate2Pipeline
+
+            pipeline = self._model = WanAnimate2Pipeline.from_pretrained(
+                self._model_path, **kwargs
+            )
         elif self.model_spec.model_family == "Wan":
             from diffusers import AutoencoderKLWan, WanImageToVideoPipeline, WanPipeline
             from transformers import CLIPVisionModel
@@ -312,9 +319,50 @@ def image_to_video(
         assert callable(self._model)
         generate_kwargs = self._model_spec.default_generate_config.copy()
         generate_kwargs.update(kwargs)
-        generate_kwargs["num_videos_per_prompt"] = n
         if num_inference_steps:
             generate_kwargs["num_inference_steps"] = num_inference_steps
+
+        if self.model_spec.model_family == "WanAnimate2":
+            if n != 1:
+                raise ValueError(
+                    "Wan-Animate-2 only supports generating one video per request"
+                )
+
+            video = generate_kwargs.pop("video", None)
+            if video is None:
+                raise ValueError("`video` is required for Wan-Animate-2")
+
+            fps = generate_kwargs.get("fps", 24)
+            temp_video_path = None
+            output: Any
+            try:
+                if isinstance(video, bytes):
+                    with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as f:
+                        temp_video_path = f.name
+                        f.write(video)
+                    video = temp_video_path
+                elif isinstance(video, str):
+                    if not os.path.isfile(video):
+                        raise FileNotFoundError(
+                            f"Video path does not exist or is not a file: {video}"
+                        )
+                else:
+                    raise TypeError("`video` must be video bytes or a local path")
+
+                self._process_progressor(generate_kwargs)
+                output = self._model(
+                    image=image,
+                    driving_video=video,
+                    prompt=prompt,
+                    **generate_kwargs,
+                )
+            finally:
+                if temp_video_path and os.path.exists(temp_video_path):
+                    os.remove(temp_video_path)
+
+            return self._output_to_video(output, fps, response_format)
+
+        generate_kwargs["num_videos_per_prompt"] = n
         fps = generate_kwargs.pop("fps", 10)
 
         # process image

xinference/model/video/model_spec.json

@@ -418,5 +418,97 @@
     },
     "updated_at": 1766385310,
     "featured": false
+  },
+  {
+    "version": 2,
+    "model_name": "Wan2.2-Animate-2-14B",
+    "model_family": "WanAnimate2",
+    "model_ability": [
+      "image2video"
+    ],
+    "default_model_config": {
+      "torch_dtype": "bfloat16"
+    },
+    "default_generate_config": {
+      "width": 640,
+      "height": 800,
+      "fps": 24,
+      "num_inference_steps": 40,
+      "guidance_scale": 3.0,
+      "flow_solver": "dpm"
+    },
+    "virtualenv": {
+      "packages": [
+        "diffusers @ git+https://github.com/huggingface/diffusers.git@refs/pull/14412/head",
+        "transformers>=4.57.6",
+        "accelerate>=1.13.0",
+        "flash-attn",
+        "decord==0.6.0",
+        "opencv-python",
+        "ftfy",
+        "imageio-ffmpeg",
+        "imageio",
+        "#system_numpy#"
+      ],
+      "no_build_isolation": true
+    },
+    "model_src": {
+      "huggingface": {
+        "model_id": "Wan-AI/Wan2.2-Animate-2-14B-Diffusers",
+        "model_revision": "main"
+      },
+      "modelscope": {
+        "model_id": "Wan-AI/Wan2.2-Animate-2-14B-Diffusers",
+        "model_revision": "master"
+      }
+    },
+    "updated_at": 1786492826,
+    "featured": false
+  },
+  {
+    "version": 2,
+    "model_name": "Wan2.2-Animate-2-14B-Distilled",
+    "model_family": "WanAnimate2",
+    "model_ability": [
+      "image2video"
+    ],
+    "default_model_config": {
+      "torch_dtype": "bfloat16"
+    },
+    "default_generate_config": {
+      "width": 640,
+      "height": 800,
+      "fps": 24,
+      "num_inference_steps": 10,
+      "guidance_scale": 1.0,
+      "flow_solver": "euler"
+    },
+    "virtualenv": {
+      "packages": [
+        "diffusers @ git+https://github.com/huggingface/diffusers.git@refs/pull/14412/head",
+        "transformers>=4.57.6",
+        "accelerate>=1.13.0",
+        "flash-attn",
+        "decord==0.6.0",
+        "opencv-python",
+        "ftfy",
+        "imageio-ffmpeg",
+        "imageio",
+        "#system_numpy#"
+      ],
+      "no_build_isolation": true
+    },
+    "model_src": {
+      "huggingface": {
+        "model_id": "Wan-AI/Wan2.2-Animate-2-14B-Distilled-Diffusers",
+        "model_revision": "main"
+      },
+      "modelscope": {
+        "model_id": "Wan-AI/Wan2.2-Animate-2-14B-Distilled-Diffusers",
+        "model_revision": "master"
+      }
+    },
+    "updated_at": 1786492826,
+    "featured": false
   }
 ]