feat(tts): add voice upload API for Qwen3-TTS

examples/online_serving/qwen3_tts/README.md

@@ -82,12 +82,63 @@ curl http://localhost:8000/v1/audio/voices
 
 ## API Reference
 
-### Endpoint
+### Endpoints
+#### GET /v1/audio/voices
 
+List all available voices/speakers from the loaded model, including both built-in model voices and uploaded custom voices.
+
+**Response Example:**
+```json
+{
+  "voices": ["vivian", "ryan", "custom_voice_1"],
+  "uploaded_voices": [
+    {
+      "name": "custom_voice_1",
+      "consent": "user_consent_id",
+      "created_at": 1738660000,
+      "file_size": 1024000,
+      "mime_type": "audio/wav"
+    }
+  ]
+}
 ```
-POST /v1/audio/speech
+
+#### POST /v1/audio/voices
+
+Upload a new voice sample for voice cloning in Base task TTS requests.
+
+**Form Parameters:**
+- `audio_sample` (required): Audio file (max 10MB, supported formats: wav, mp3, flac, ogg, aac, webm, mp4)
+- `consent` (required): Consent recording ID
+- `name` (required): Name for the new voice
+
+**Response Example:**
+```json
+{
+  "success": true,
+  "voice": {
+    "name": "custom_voice_1",
+    "consent": "user_consent_id",
+    "file_path": "/tmp/voice_samples/custom_voice_1_user_consent_id_1738660000.wav",
+    "created_at": 1738660000,
+    "mime_type": "audio/wav",
+    "file_size": 1024000
+  }
+}
+```
+
+**Usage Example:**
+```bash
+curl -X POST http://localhost:8000/v1/audio/voices \
+  -F "audio_sample=@/path/to/voice_sample.wav" \
+  -F "consent=user_consent_id" \
+  -F "name=custom_voice_1"
 ```
 
+
+#### POST /v1/audio/speech
+
+
 This endpoint follows the [OpenAI Audio Speech API](https://platform.openai.com/docs/api-reference/audio/createSpeech) format with additional Qwen3-TTS parameters.
 
 ### Request Body

vllm_omni/entrypoints/openai/api_server.py

@@ -815,8 +815,80 @@ async def list_voices(raw_request: Request):
     if handler is None:
         return base(raw_request).create_error_response(message="The model does not support Speech API")
 
+    # Get all speakers (both model built-in and uploaded)
     speakers = sorted(handler.supported_speakers) if handler.supported_speakers else []
-    return JSONResponse(content={"voices": speakers})
+
+    # Get uploaded speakers details
+    uploaded_speakers = []
+    if hasattr(handler, 'uploaded_speakers'):
+        for voice_name, info in handler.uploaded_speakers.items():
+            uploaded_speakers.append({
+                "name": info.get("name", voice_name),
+                "consent": info.get("consent", ""),
+                "created_at": info.get("created_at", 0),
+                "file_size": info.get("file_size", 0),
+                "mime_type": info.get("mime_type", "")
+            })
+
+    return JSONResponse(content={
+        "voices": speakers,
+        "uploaded_voices": uploaded_speakers
+    })
+
+
+@router.post(
+    "/v1/audio/voices",
+    responses={
+        HTTPStatus.OK.value: {"model": dict},
+        HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
+        HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
+    },
+)
+async def upload_voice(
+    raw_request: Request,
+    audio_sample: UploadFile = File(...),
+    consent: str = Form(...),
+    name: str = Form(...),
+):
+    """Upload a new voice sample for voice cloning.
+
+    Uploads an audio file that can be used as a reference for voice cloning
+    in Base task TTS requests. The voice can then be referenced by name
+    in subsequent TTS requests.
+
+    Args:
+        audio_sample: Audio file (max 10MB)
+        consent: Consent recording ID
+        name: Name for the new voice
+        raw_request: Raw FastAPI request
+
+    Returns:
+        JSON response with voice information
+    """
+    handler = Omnispeech(raw_request)
+    if handler is None:
+        return base(raw_request).create_error_response(message="The model does not support Speech API")
+
+    try:
+        # Validate required parameters
+        if not consent:
+            return base(raw_request).create_error_response(message="consent is required")
+        if not name:
+            return base(raw_request).create_error_response(message="name is required")
+
+        # Upload the voice
+        result = await handler.upload_voice(audio_sample, consent, name)
+
+        return JSONResponse(content={
+            "success": True,
+            "voice": result
+        })
+
+    except ValueError as e:
+        return base(raw_request).create_error_response(message=str(e))
+    except Exception as e:
+        logger.exception(f"Failed to upload voice: {e}")
+        return base(raw_request).create_error_response(message=f"Failed to upload voice: {str(e)}")
 
 
 # Health and Model endpoints for diffusion mode

vllm_omni/entrypoints/openai/serving_speech.py

@@ -1,7 +1,11 @@
 import asyncio
+import json
+import os
+import time
+from pathlib import Path
 from typing import Any
 
-from fastapi import Request
+from fastapi import Request, UploadFile
 from fastapi.responses import Response
 from vllm.entrypoints.openai.engine.serving import OpenAIServing
 from vllm.logger import init_logger
@@ -40,9 +44,20 @@
 class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        # Initialize uploaded speakers storage
+        self.uploaded_speakers_dir = Path("/tmp/voice_samples")
+        self.uploaded_speakers_dir.mkdir(parents=True, exist_ok=True)
+        self.metadata_file = self.uploaded_speakers_dir / "metadata.json"
+
         # Load supported speakers
         self.supported_speakers = self._load_supported_speakers()
+        # Load uploaded speakers
+        self.uploaded_speakers = self._load_uploaded_speakers()
+        # Merge supported speakers with uploaded speakers
+        self.supported_speakers.update(self.uploaded_speakers.keys())
+
         logger.info(f"Loaded {len(self.supported_speakers)} supported speakers: {sorted(self.supported_speakers)}")
+        logger.info(f"Loaded {len(self.uploaded_speakers)} uploaded speakers")
 
     def _load_supported_speakers(self) -> set[str]:
         """Load supported speakers (case-insensitive) from the model configuration."""
@@ -62,6 +77,151 @@ def _load_supported_speakers(self) -> set[str]:
 
         return set()
 
+    def _load_uploaded_speakers(self) -> dict[str, dict]:
+        """Load uploaded speakers from metadata file."""
+        if not self.metadata_file.exists():
+            return {}
+
+        try:
+            with open(self.metadata_file, 'r') as f:
+                metadata = json.load(f)
+            return metadata.get("uploaded_speakers", {})
+        except Exception as e:
+            logger.warning(f"Could not load uploaded speakers metadata: {e}")
+            return {}
+
+    def _save_uploaded_speakers(self) -> None:
+        """Save uploaded speakers to metadata file."""
+        try:
+            metadata = {"uploaded_speakers": self.uploaded_speakers}
+            with open(self.metadata_file, 'w') as f:
+                json.dump(metadata, f, indent=2)
+        except Exception as e:
+            logger.error(f"Could not save uploaded speakers metadata: {e}")
+
+    def _get_uploaded_audio_data(self, voice_name: str) -> str | None:
+        """Get base64 encoded audio data for uploaded voice."""
+        voice_name_lower = voice_name.lower()
+        if voice_name_lower not in self.uploaded_speakers:
+            return None
+
+        speaker_info = self.uploaded_speakers[voice_name_lower]
+        file_path = Path(speaker_info["file_path"])
+
+        if not file_path.exists():
+            logger.warning(f"Audio file not found for voice {voice_name}: {file_path}")
+            return None
+
+        try:
+            import base64
+
+            # Read audio file
+            with open(file_path, 'rb') as f:
+                audio_bytes = f.read()
+
+            # Encode to base64
+            audio_b64 = base64.b64encode(audio_bytes).decode('utf-8')
+
+            # Get MIME type from file extension
+            mime_type = speaker_info.get("mime_type", "audio/wav")
+
+            # Return as data URL
+            return f"data:{mime_type};base64,{audio_b64}"
+        except Exception as e:
+            logger.error(f"Could not read audio file for voice {voice_name}: {e}")
+            return None
+
+    async def upload_voice(self, audio_file: UploadFile, consent: str, name: str) -> dict:
+        """Upload a new voice sample."""
+        # Validate file size (max 10MB)
+        MAX_FILE_SIZE = 10 * 1024 * 1024  # 10MB
+        audio_file.file.seek(0, 2)  # Seek to end
+        file_size = audio_file.file.tell()
+        audio_file.file.seek(0)  # Reset to beginning
+
+        if file_size > MAX_FILE_SIZE:
+            raise ValueError(f"File size exceeds maximum limit of 10MB. Got {file_size} bytes.")
+
+        # Detect MIME type from filename if content_type is generic
+        mime_type = audio_file.content_type
+        if mime_type == "application/octet-stream":
+            # Simple MIME type detection based on file extension
+            filename_lower = audio_file.filename.lower()
+            if filename_lower.endswith(".wav"):
+                mime_type = "audio/wav"
+            elif filename_lower.endswith((".mp3", ".mpeg")):
+                mime_type = "audio/mpeg"
+            elif filename_lower.endswith(".flac"):
+                mime_type = "audio/flac"
+            elif filename_lower.endswith(".ogg"):
+                mime_type = "audio/ogg"
+            elif filename_lower.endswith(".aac"):
+                mime_type = "audio/aac"
+            elif filename_lower.endswith(".webm"):
+                mime_type = "audio/webm"
+            elif filename_lower.endswith(".mp4"):
+                mime_type = "audio/mp4"
+            else:
+                mime_type = "audio/wav"  # Default
+
+        # Validate MIME type
+        allowed_mime_types = {
+            "audio/mpeg", "audio/wav", "audio/x-wav", "audio/ogg",
+            "audio/aac", "audio/flac", "audio/webm", "audio/mp4"
+        }
+
+        if mime_type not in allowed_mime_types:
+            raise ValueError(f"Unsupported MIME type: {mime_type}. Allowed: {allowed_mime_types}")
+
+        # Normalize voice name
+        voice_name_lower = name.lower()
+
+        # Check if voice already exists
+        if voice_name_lower in self.uploaded_speakers:
+            raise ValueError(f"Voice '{name}' already exists")
+
+        # Generate filename
+        timestamp = int(time.time())
+        file_ext = audio_file.filename.split('.')[-1] if '.' in audio_file.filename else "wav"
+        filename = f"{name}_{consent}_{timestamp}.{file_ext}"
+        file_path = self.uploaded_speakers_dir / filename
+
+        # Save audio file
+        try:
+            with open(file_path, 'wb') as f:
+                content = await audio_file.read()
+                f.write(content)
+        except Exception as e:
+            raise ValueError(f"Failed to save audio file: {e}")
+
+        # Update metadata
+        self.uploaded_speakers[voice_name_lower] = {
+            "name": name,
+            "consent": consent,
+            "file_path": str(file_path),
+            "created_at": timestamp,
+            "mime_type": mime_type,
+            "original_filename": audio_file.filename,
+            "file_size": file_size
+        }
+
+        # Update supported speakers
+        self.supported_speakers.add(voice_name_lower)
+
+        # Save metadata
+        self._save_uploaded_speakers()
+
+        logger.info(f"Uploaded new voice '{name}' with consent ID '{consent}'")
+
+        return {
+            "name": name,
+            "consent": consent,
+            "file_path": str(file_path),
+            "created_at": timestamp,
+            "mime_type": mime_type,
+            "file_size": file_size
+        }
+
     def _is_tts_model(self) -> bool:
         """Check if the current model is a supported TTS model."""
         stage_list = getattr(self.engine_client, "stage_list", None)
@@ -94,7 +254,7 @@ def _validate_tts_request(self, request: OpenAICreateSpeechRequest) -> str | Non
                 return f"Invalid speaker '{request.voice}'. Supported: {', '.join(sorted(self.supported_speakers))}"
 
         # Validate Base task requirements
-        if task_type == "Base":
+        if task_type == "Base" and request.voice is None:
             if request.ref_audio is None:
                 return "Base task requires 'ref_audio' for voice cloning"
             # Validate ref_audio format
@@ -155,6 +315,14 @@ def _build_tts_params(self, request: OpenAICreateSpeechRequest) -> dict[str, Any
         # Speaker (voice)
         if request.voice is not None:
             params["speaker"] = [request.voice]
+
+            # If voice is an uploaded speaker and no ref_audio provided, auto-set it
+            if request.voice.lower() in self.uploaded_speakers and request.ref_audio is None:
+                audio_data = self._get_uploaded_audio_data(request.voice)
+                if audio_data:
+                    params["ref_audio"] = [audio_data]
+                    params["x_vector_only_mode"] = [True]
+                    logger.info(f"Auto-set ref_audio for uploaded voice: {request.voice}")
         elif params["task_type"][0] == "CustomVoice":
             params["speaker"] = ["Vivian"]  # Default for CustomVoice