Merge pull request #55 from WuMingDao/dev

feat: add image-to-video generation

Author: WuMingDao

README.md

@@ -21,6 +21,7 @@ batch generation, and one-click deployment to Cloudflare Pages.
 ## Features
 
 - **Multiple AI Providers** - Gitee AI, HuggingFace Spaces, ModelScope
+- **Image-to-Video** - Generate videos from images (Gitee AI)
 - **Dark Mode UI** - Gradio-style with frosted glass effects
 - **Flexible Sizing** - Multiple aspect ratios (1:1, 16:9, 9:16, 4:3, etc.)
 - **4x Upscaling** - RealESRGAN integration

README.zh.md

@@ -21,6 +21,7 @@
 ## 功能特性
 
 - **多 AI 提供商** - Gitee AI、HuggingFace Spaces、ModelScope
+- **图片转视频** - 从图片生成视频 (Gitee AI)
 - **深色模式 UI** - Gradio 风格毛玻璃效果
 - **灵活尺寸** - 多种宽高比 (1:1, 16:9, 9:16, 4:3 等)
 - **4x 放大** - RealESRGAN 集成

apps/api/src/app.ts

@@ -26,6 +26,7 @@ import {
   TRANSLATION_CONFIG,
   type TranslateRequest,
   type TranslateResponse,
+  type VideoGenerateRequest,
   validateDimensions,
   validatePrompt,
   validateScale,
@@ -45,6 +46,7 @@ import {
   timeout,
 } from './middleware'
 import { getProvider, hasProvider } from './providers'
+import { createVideoTask, getVideoTaskStatus } from './providers/gitee'
 import { callGradioApi, formatDimensions, formatDuration } from './utils'
 
 export interface AppConfig {
@@ -88,13 +90,16 @@ export function createApp(config: AppConfig = {}) {
   app.use('/upscale', timeout(120000))
   app.use('/optimize', timeout(60000)) // 60 seconds for LLM
   app.use('/translate', timeout(30000)) // 30 seconds for translation
+  app.use('/video/generate', timeout(30000)) // 30 seconds for video task creation
+  app.use('/video/status/*', timeout(30000)) // 30 seconds for status check
 
   // Apply body limit (50KB for most endpoints)
   app.use('/generate', bodyLimit(50 * 1024))
   app.use('/generate-hf', bodyLimit(50 * 1024))
   app.use('/upscale', bodyLimit(50 * 1024))
   app.use('/optimize', bodyLimit(50 * 1024))
   app.use('/translate', bodyLimit(20 * 1024)) // 20KB for translation
+  app.use('/video/generate', bodyLimit(50 * 1024))
 
   // Health check
   app.get('/', (c) => {
@@ -522,6 +527,69 @@ export function createApp(config: AppConfig = {}) {
     }
   })
 
+  // Video generation - Create task
+  app.post('/video/generate', async (c) => {
+    let body: VideoGenerateRequest
+    try {
+      body = await c.req.json()
+    } catch {
+      return sendError(c, Errors.invalidParams('body', 'Invalid JSON body'))
+    }
+
+    const authToken = c.req.header('X-API-Key')
+
+    if (!authToken) {
+      return sendError(c, Errors.authRequired('Gitee AI'))
+    }
+
+    if (body.provider !== 'gitee') {
+      return c.json({ error: 'Use frontend direct call for HuggingFace' }, 400)
+    }
+
+    if (!body.imageUrl || !body.prompt) {
+      return sendError(c, Errors.invalidParams('body', 'imageUrl and prompt are required'))
+    }
+
+    const dimensionsValidation = validateDimensions(body.width, body.height)
+    if (!dimensionsValidation.valid) {
+      return sendError(
+        c,
+        Errors.invalidDimensions(dimensionsValidation.error || 'Invalid dimensions')
+      )
+    }
+
+    try {
+      const taskId = await createVideoTask(
+        body.imageUrl,
+        body.prompt,
+        body.width,
+        body.height,
+        authToken
+      )
+
+      return c.json({ taskId, status: 'pending' })
+    } catch (err) {
+      return sendError(c, err)
+    }
+  })
+
+  // Video generation - Query status
+  app.get('/video/status/:taskId', async (c) => {
+    const taskId = c.req.param('taskId')
+    const authToken = c.req.header('X-API-Key')
+
+    if (!authToken) {
+      return sendError(c, Errors.authRequired('Gitee AI'))
+    }
+
+    try {
+      const result = await getVideoTaskStatus(taskId, authToken)
+      return c.json(result)
+    } catch (err) {
+      return sendError(c, err)
+    }
+  })
+
   return app
 }
 

apps/api/src/providers/gitee.ts

@@ -2,10 +2,13 @@
  * Gitee AI Provider Implementation
  */
 
-import { Errors } from '@z-image/shared'
+import type { VideoTaskResponse } from '@z-image/shared'
+import { Errors, VIDEO_NEGATIVE_PROMPT } from '@z-image/shared'
 import type { ImageProvider, ProviderGenerateRequest, ProviderGenerateResult } from './types'
 
 const GITEE_API_URL = 'https://ai.gitee.com/v1/images/generations'
+const GITEE_VIDEO_API = 'https://ai.gitee.com/v1/async/videos/image-to-video'
+const GITEE_TASK_API = 'https://ai.gitee.com/api/v1/task'
 
 interface GiteeImageResponse {
   data: Array<{
@@ -115,3 +118,76 @@ export class GiteeProvider implements ImageProvider {
 }
 
 export const giteeProvider = new GiteeProvider()
+
+interface GiteeVideoTaskResponse {
+  task_id: string
+}
+
+interface GiteeTaskStatusResponse {
+  status: 'pending' | 'is_process' | 'success' | 'failure'
+  output?: {
+    file_url?: string
+    error?: string
+  }
+}
+
+export async function createVideoTask(
+  imageUrl: string,
+  prompt: string,
+  width: number,
+  height: number,
+  authToken: string
+): Promise<string> {
+  const formData = new FormData()
+  formData.append('image', imageUrl)
+  formData.append('prompt', prompt)
+  formData.append('negative_prompt', VIDEO_NEGATIVE_PROMPT)
+  formData.append('model', 'Wan2_2-I2V-A14B')
+  formData.append('num_inference_steps', '6')
+  formData.append('num_frames', '48')
+  formData.append('guidance_scale', '1')
+  formData.append('width', width.toString())
+  formData.append('height', height.toString())
+
+  const response = await fetch(GITEE_VIDEO_API, {
+    method: 'POST',
+    headers: {
+      Authorization: `Bearer ${authToken.trim()}`,
+    },
+    body: formData,
+  })
+
+  if (!response.ok) {
+    const errData = (await response.json().catch(() => ({}))) as GiteeErrorResponse
+    throw parseGiteeError(response.status, errData)
+  }
+
+  const data = (await response.json()) as GiteeVideoTaskResponse
+  return data.task_id
+}
+
+export async function getVideoTaskStatus(
+  taskId: string,
+  authToken: string
+): Promise<VideoTaskResponse> {
+  const response = await fetch(`${GITEE_TASK_API}/${taskId}`, {
+    headers: {
+      Authorization: `Bearer ${authToken.trim()}`,
+    },
+  })
+
+  if (!response.ok) {
+    const errData = (await response.json().catch(() => ({}))) as GiteeErrorResponse
+    throw parseGiteeError(response.status, errData)
+  }
+
+  const data = (await response.json()) as GiteeTaskStatusResponse
+
+  if (data.status === 'success') {
+    return { status: 'success', videoUrl: data.output?.file_url }
+  }
+  if (data.status === 'failure') {
+    return { status: 'failed', error: data.output?.error }
+  }
+  return { status: data.status === 'is_process' ? 'processing' : 'pending' }
+}

apps/web/src/components/feature/ImageResultCard.tsx

@@ -9,6 +9,7 @@ import {
   Info,
   Loader2,
   Trash2,
+  Video,
   X,
   ZoomIn,
   ZoomOut,
@@ -17,6 +18,7 @@ import { useCallback, useEffect, useRef, useState } from 'react'
 import { toast } from 'sonner'
 import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'
 import { ImageComparison } from '@/components/ui/ImageComparison'
+import { useVideoGenerator } from '@/hooks/useVideoGenerator'
 import { upscaleImage } from '@/lib/api'
 
 interface ImageResultCardProps {
@@ -27,6 +29,7 @@ interface ImageResultCardProps {
   isBlurred: boolean
   isUpscaled: boolean
   isUpscaling: boolean
+  giteeToken?: string
   setShowInfo: (v: boolean) => void
   setIsBlurred: (v: boolean) => void
   handleUpscale: () => void
@@ -42,6 +45,7 @@ export function ImageResultCard({
   isBlurred,
   isUpscaled,
   isUpscaling: externalIsUpscaling,
+  giteeToken,
   setShowInfo,
   setIsBlurred,
   handleUpscale: _externalHandleUpscale,
@@ -63,6 +67,10 @@ export function ImageResultCard({
   const dragStartRef = useRef({ x: 0, y: 0, posX: 0, posY: 0 })
   const imageContainerRef = useRef<HTMLDivElement>(null)
 
+  // Video generation state
+  const { videoState, generateVideo } = useVideoGenerator()
+  const [showVideo, setShowVideo] = useState(false)
+
   // Use display URL if set (after applying upscale), otherwise original
   const currentImageUrl = displayUrl || imageDetails?.url
 
@@ -119,6 +127,32 @@ export function ImageResultCard({
     setIsComparing(false)
   }, [])
 
+  // Handle video generation
+  const handleGenerateVideo = useCallback(async () => {
+    if (!currentImageUrl || !imageDetails) return
+
+    if (!giteeToken) {
+      toast.error('Please configure Gitee AI token first')
+      return
+    }
+
+    const width = Number.parseInt(imageDetails.dimensions.split('x')[0], 10) || 1024
+    const height = Number.parseInt(imageDetails.dimensions.split('x')[1], 10) || 1024
+
+    toast.info('Starting video generation...')
+    await generateVideo(currentImageUrl, imageDetails.prompt, width, height, 'gitee', giteeToken)
+  }, [currentImageUrl, imageDetails, generateVideo, giteeToken])
+
+  // Show video when generation succeeds
+  useEffect(() => {
+    if (videoState.status === 'success') {
+      setShowVideo(true)
+      toast.success('Video generated successfully!')
+    } else if (videoState.status === 'failed') {
+      toast.error(videoState.error || 'Video generation failed')
+    }
+  }, [videoState.status, videoState.error])
+
   // Zoom controls
   const handleZoomIn = useCallback(() => {
     setScale((s) => Math.min(s * 1.5, 8))
@@ -231,14 +265,25 @@ export function ImageResultCard({
           <div className="relative rounded-lg overflow-hidden bg-zinc-900 border border-zinc-800 group">
             {imageDetails ? (
               <>
-                {/* Normal image display with double-click to fullscreen */}
-                <img
-                  src={currentImageUrl || ''}
-                  alt="Generated"
-                  className={`w-full transition-all duration-300 cursor-pointer ${isBlurred ? 'blur-xl' : ''}`}
-                  onDoubleClick={handleDoubleClick}
-                  title="Double-click to view fullscreen"
-                />
+                {/* Video or Image display */}
+                {showVideo && videoState.videoUrl ? (
+                  <video
+                    src={videoState.videoUrl}
+                    controls
+                    autoPlay
+                    loop
+                    muted
+                    className="w-full"
+                  />
+                ) : (
+                  <img
+                    src={currentImageUrl || ''}
+                    alt="Generated"
+                    className={`w-full transition-all duration-300 cursor-pointer ${isBlurred ? 'blur-xl' : ''}`}
+                    onDoubleClick={handleDoubleClick}
+                    title="Double-click to view fullscreen"
+                  />
+                )}
 
                 {/* Floating Toolbar */}
                 <div className="absolute bottom-3 left-1/2 -translate-x-1/2 pointer-events-none">
@@ -271,6 +316,39 @@ export function ImageResultCard({
                       {isBlurred ? <EyeOff className="w-5 h-5" /> : <Eye className="w-5 h-5" />}
                     </button>
                     <div className="w-px h-5 bg-white/10" />
+                    {/* Generate Video */}
+                    <button
+                      type="button"
+                      onClick={
+                        videoState.status === 'success'
+                          ? () => setShowVideo(!showVideo)
+                          : handleGenerateVideo
+                      }
+                      disabled={
+                        videoState.status === 'generating' || videoState.status === 'polling'
+                      }
+                      title={
+                        videoState.status === 'generating' || videoState.status === 'polling'
+                          ? 'Generating video...'
+                          : videoState.status === 'success'
+                            ? showVideo
+                              ? 'Show image'
+                              : 'Show video'
+                            : 'Generate video'
+                      }
+                      className={`flex items-center justify-center w-10 h-10 rounded-xl transition-all ${
+                        videoState.status === 'success'
+                          ? 'text-green-400 bg-green-500/10'
+                          : 'text-white/70 hover:text-white hover:bg-white/10'
+                      } disabled:cursor-not-allowed disabled:opacity-50`}
+                    >
+                      {videoState.status === 'generating' || videoState.status === 'polling' ? (
+                        <Loader2 className="w-5 h-5 animate-spin text-orange-400" />
+                      ) : (
+                        <Video className="w-5 h-5" />
+                      )}
+                    </button>
+                    <div className="w-px h-5 bg-white/10" />
                     {/* Download */}
                     <button
                       type="button"