repomix-output.xml

This file is a merged representation of the entire codebase, combined into a single document by Repomix.

<file_summary>
This section contains a summary of this file.

<purpose>
This file contains a packed representation of the entire repository's contents.
It is designed to be easily consumable by AI systems for analysis, code review,
or other automated processes.
</purpose>

<file_format>
The content is organized as follows:
1. This summary section
2. Repository information
3. Directory structure
4. Repository files (if enabled)
5. Multiple file entries, each consisting of:
  - File path as an attribute
  - Full contents of the file
</file_format>

<usage_guidelines>
- This file should be treated as read-only. Any changes should be made to the
  original repository files, not this packed version.
- When processing this file, use the file path to distinguish
  between different files in the repository.
- Be aware that this file may contain sensitive information. Handle it with
  the same level of security as you would the original repository.
</usage_guidelines>

<notes>
- Some files may have been excluded based on .gitignore rules and Repomix's configuration
- Binary files are not included in this packed representation. Please refer to the Repository Structure section for a complete list of file paths, including binary files
- Files matching patterns in .gitignore are excluded
- Files matching default ignore patterns are excluded
- Files are sorted by Git change count (files with more changes are at the bottom)
</notes>

</file_summary>

<directory_structure>
src/
  ai_watch_buddy/
    agent/
      gemini_sample.py
      mock_text.py
      text_stream_to_action.py
      video_action_agent_interface.py
      video_analyzer_agent.py
    asr/
      __init__.py
      asr_interface.py
      fish_audio_asr.py
    prompts/
      action_gen_prompt.py
      character_prompts.py
    tts/
      edge_tts.py
      fish_audio_tts.py
      tts_interface.py
    actions.py
    connection_manager.py
    fetch_video.py
    pipeline.py
    server.py
    session.py
    test_tts_integration.py
    tts_generator.py
</directory_structure>

<files>
This section contains the contents of the repository's files.

<file path="src/ai_watch_buddy/asr/__init__.py">
"""AI Watch Buddy ASR (Automatic Speech Recognition) module."""

from .asr_interface import ASRInterface
from .fish_audio_asr import FishAudioASR

__all__ = ["ASRInterface", "FishAudioASR"]
</file>

<file path="src/ai_watch_buddy/asr/asr_interface.py">
"""ASR (Automatic Speech Recognition) interface definition."""

from abc import ABC, abstractmethod
from typing import Optional


class ASRInterface(ABC):
    """Abstract base class for ASR services."""
    
    @abstractmethod
    async def transcribe_audio(
        self, 
        audio_base64: str, 
        language: Optional[str] = None
    ) -> Optional[str]:
        """
        Transcribe base64-encoded audio to text.
        
        Args:
            audio_base64: Base64-encoded audio data
            language: Language code (e.g., "en", "zh"). If None, auto-detect.
            
        Returns:
            Transcribed text, or None if transcription failed
        """
        pass
    
    @abstractmethod
    def transcribe_audio_sync(
        self, 
        audio_base64: str, 
        language: Optional[str] = None
    ) -> Optional[str]:
        """
        Synchronous version of transcribe_audio.
        
        Args:
            audio_base64: Base64-encoded audio data
            language: Language code (e.g., "en", "zh"). If None, auto-detect.
            
        Returns:
            Transcribed text, or None if transcription failed
        """
        pass
</file>

<file path="src/ai_watch_buddy/asr/fish_audio_asr.py">
"""Fish Audio ASR implementation for speech-to-text conversion."""

import base64
import tempfile
import os
from pathlib import Path
from typing import Optional
from loguru import logger

from .asr_interface import ASRInterface

try:
    from fish_audio_sdk import Session, ASRRequest
except ImportError:
    logger.warning("fish_audio_sdk not installed. Please install it with: pip install fish_audio_sdk")
    Session = None
    ASRRequest = None


class FishAudioASR(ASRInterface):
    """Fish Audio ASR service for converting audio to text."""
    
    def __init__(self, api_key: Optional[str] = None):
        """
        Initialize Fish Audio ASR service.
        
        Args:
            api_key: Fish Audio API key. If None, will try to get from environment.
        """
        if Session is None:
            raise ImportError("fish_audio_sdk is required. Install with: pip install fish_audio_sdk")
            
        if api_key is None:
            api_key = os.getenv("FISH_AUDIO_API_KEY")
            
        if not api_key:
            raise ValueError("Fish Audio API key is required. Set FISH_AUDIO_API_KEY environment variable or pass api_key parameter.")
            
        self.session = Session(api_key)
        logger.info("Fish Audio ASR initialized successfully")
    
    async def transcribe_audio(
        self, 
        audio_base64: str, 
        language: Optional[str] = None,
        ignore_timestamps: bool = True
    ) -> Optional[str]:
        """
        Transcribe base64-encoded audio to text.
        
        Args:
            audio_base64: Base64-encoded audio data
            language: Language code (e.g., "en", "zh"). If None, auto-detect.
            ignore_timestamps: Whether to ignore precise timestamps for faster processing
            
        Returns:
            Transcribed text, or None if transcription failed
        """
        try:
            # Decode base64 audio data
            audio_data = base64.b64decode(audio_base64)
            
            # Create temporary file for audio data
            with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as temp_file:
                temp_file.write(audio_data)
                temp_file_path = temp_file.name
            
            try:
                # Read audio file
                with open(temp_file_path, "rb") as audio_file:
                    audio_bytes = audio_file.read()
                
                # Create ASR request
                if language:
                    request = ASRRequest(
                        audio=audio_bytes, 
                        language=language, 
                        ignore_timestamps=ignore_timestamps
                    )
                else:
                    request = ASRRequest(
                        audio=audio_bytes, 
                        ignore_timestamps=ignore_timestamps
                    )
                
                # Perform ASR
                response = self.session.asr(request)
                
                logger.info(f"ASR successful: '{response.text}' (duration: {response.duration}s)")
                
                # Log segments if available
                if hasattr(response, 'segments') and response.segments:
                    for segment in response.segments:
                        logger.debug(f"Segment: '{segment.text}' [{segment.start}-{segment.end}s]")
                
                return response.text
                
            finally:
                # Clean up temporary file
                try:
                    os.unlink(temp_file_path)
                except OSError:
                    logger.warning(f"Failed to delete temporary file: {temp_file_path}")
                    
        except Exception as e:
            logger.error(f"ASR transcription failed: {e}", exc_info=True)
            return None
    
    def transcribe_audio_sync(
        self, 
        audio_base64: str, 
        language: Optional[str] = None,
        ignore_timestamps: bool = True
    ) -> Optional[str]:
        """
        Synchronous version of transcribe_audio.
        
        Args:
            audio_base64: Base64-encoded audio data
            language: Language code (e.g., "en", "zh"). If None, auto-detect.
            ignore_timestamps: Whether to ignore precise timestamps for faster processing
            
        Returns:
            Transcribed text, or None if transcription failed
        """
        try:
            # Decode base64 audio data
            audio_data = base64.b64decode(audio_base64)
            
            # Create temporary file for audio data
            with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as temp_file:
                temp_file.write(audio_data)
                temp_file_path = temp_file.name
            
            try:
                # Read audio file
                with open(temp_file_path, "rb") as audio_file:
                    audio_bytes = audio_file.read()
                
                # Create ASR request
                if language:
                    request = ASRRequest(
                        audio=audio_bytes, 
                        language=language, 
                        ignore_timestamps=ignore_timestamps
                    )
                else:
                    request = ASRRequest(
                        audio=audio_bytes, 
                        ignore_timestamps=ignore_timestamps
                    )
                
                # Perform ASR
                response = self.session.asr(request)
                
                logger.info(f"ASR successful: '{response.text}' (duration: {response.duration}s)")
                
                return response.text
                
            finally:
                # Clean up temporary file
                try:
                    os.unlink(temp_file_path)
                except OSError:
                    logger.warning(f"Failed to delete temporary file: {temp_file_path}")
                    
        except Exception as e:
            logger.error(f"ASR transcription failed: {e}", exc_info=True)
            return None
</file>

<file path="src/ai_watch_buddy/agent/gemini_sample.py">
# import os
# from google import genai
# from google.genai import types


# client = genai.Client(api_key=os.getenv("GEMINI_API_KEY"))

# video_file = client.files.upload(
#     file="video_cache/【官方 MV】Never Gonna Give You Up - Rick Astley.mp4"
# )


# response = client.models.generate_content(
#     model="gemini-2.5-flash",
#     contents=["这个视频是关于什么的? 请批判性的分析视频内容"],
#     config=types.GenerateContentConfig(
#         system_instruction="I say high, you say low",
#     ),
# )


# =====================


# To run this code you need to install the following dependencies:
# pip install google-genai

import base64
import os
from google import genai
from google.genai import types

# 聊天历史是 list[types.Content]


class GeminiCore:
    def __init__(self, api_key: str | None = os.getenv("GEMINI_API_KEY")):
        """
        初始化 GeminiCore 类，设置 API 密钥。

        Args:
            api_key (str | None): Gemini API 密钥，默认为环境变量中的值。
        """
        self.client = genai.Client(api_key=api_key)


def upload_video(video_path: str, client: genai.Client) -> types.File:
    """
    上传视频文件到 Gemini，并返回 FileData 对象。

    Args:
        video_path (str): 视频文件的本地路径。

    Returns:
        types.FileData: 上传后的视频文件数据对象。
    """
    print(f"正在上传视频文件: {video_path}")
    video_file = client.files.upload(file=video_path)
    import time

    # Wait until the uploaded video is available
    while video_file.state.name == "PROCESSING":
        print("[继续上传]..", end="", flush=True)
        time.sleep(5)
        video_file = client.files.get(name=video_file.name)

    if video_file.state.name == "FAILED":
        raise ValueError(video_file.state.name)

    # 拿到的 video_file 是一个 File 对象
    return video_file


def generate(
    gemini_api_key: str | None = os.getenv("GEMINI_API_KEY"),
    system_instruction: str = "You are a helpful assistant.",
    video_uri: str = "https://www.youtube.com/watch?v=9hE5-98ZeCg",
) -> None:
    client = genai.Client(api_key=gemini_api_key)

    vid_from_yt = types.FileData(file_uri=video_uri)

    model = "gemini-2.5-flash"
    contents = [
        # video_file,
        types.Part(file_data=vid_from_yt),
        types.Content(
            role="user",
            parts=[
                types.Part.from_text(text="""你好，请帮我分析这个视频的内容。"""),
            ],
        ),
        types.Content(
            role="model",
            parts=[
                types.Part.from_text(text=""""""),
                types.Part.from_text(
                    text="""你好，我立刻开始分析视频内容。我会根据你的要求，分析视频后，在我说的所有话中的尾部添加上 "喵～～" 的口癖，因为我是一只可爱的猫娘视频观众喵～"""
                ),
            ],
        ),
        types.Content(
            role="user",
            parts=[
                types.Part.from_text(text="""好的。请分析视频内容"""),
            ],
        ),
    ]
    print(contents)
    generate_content_config = types.GenerateContentConfig(
        thinking_config=types.ThinkingConfig(
            thinking_budget=-1,
        ),
        system_instruction=[
            types.Part.from_text(text=system_instruction),
        ],
    )
    print("开始生成内容...")

    for chunk in client.models.generate_content_stream(
        model=model,
        contents=contents,
        config=generate_content_config,
    ):
        print(chunk.text, end="", flush=True)


if __name__ == "__main__":
    generate()
</file>

<file path="src/ai_watch_buddy/agent/text_stream_to_action.py">
import json
from collections.abc import Iterator, Generator
from json_repair import repair_json
from pydantic import ValidationError, TypeAdapter
from google.genai.types import GenerateContentResponse

from ..actions import Action


def str_stream_to_actions(
    llm_stream: Iterator[GenerateContentResponse],
) -> Generator[Action, None, None]:
    """
    从 LLM 传入的 str 流中，流式解析并 yield 出 Action 对象。

    该函数会逐步解析LLM输出的JSON数组，每当解析出完整的Action对象时就立即验证并yield。
    这样可以实现真正的流式处理，而不需要等待整个响应完成。

    Args:
        llm_stream: LLM输出的字符串流，预期格式为JSON数组

    Yields:
        Action: 解析并验证后的Action对象

    注意:
        - 会自动跳过```json等markdown代码块标记
        - 使用json_repair库处理可能的JSON格式问题
        - 解析失败的Action会被跳过并打印错误信息
    """
    buffer = ""
    in_json_array = False
    brace_count = 0
    current_action_buffer = ""
    action_adapter = TypeAdapter(Action)

    for response in llm_stream:
        # Extract text from GenerateContentResponse
        chunk = response.text if response.text else ""
        print(chunk, end="", flush=True)
        buffer += chunk

        # 如果还没有找到JSON数组的开始，寻找 '['
        if not in_json_array:
            # 跳过可能的 ```json 前缀
            json_start = buffer.find("[")
            if json_start != -1:
                buffer = buffer[json_start:]
                in_json_array = True
                brace_count = 0
            else:
                continue

        # 逐字符处理buffer中的内容
        i = 0
        while i < len(buffer):
            char = buffer[i]

            if char == "{":
                if brace_count == 0:
                    # 开始一个新的Action对象
                    current_action_buffer = "{"
                else:
                    current_action_buffer += char
                brace_count += 1

            elif char == "}":
                current_action_buffer += char
                brace_count -= 1

                if brace_count == 0:
                    # 完成了一个Action对象的解析
                    try:
                        # 尝试修复可能的JSON格式问题
                        repaired_json = repair_json(current_action_buffer)
                        # 解析并验证Action
                        action_dict = json.loads(repaired_json)
                        action = action_adapter.validate_python(action_dict)
                        yield action
                    except (json.JSONDecodeError, ValidationError) as e:
                        # 如果解析失败，记录错误但继续处理后续内容
                        print(f"Failed to parse action: {e}")
                        print(f"Raw JSON: {current_action_buffer}")

                    current_action_buffer = ""

            elif brace_count > 0:
                # 在Action对象内部，添加字符
                current_action_buffer += char

            elif char == "]":
                # JSON数组结束
                break

            i += 1

        # 更新buffer，移除已处理的部分
        if i > 0:
            buffer = buffer[i:]
</file>

<file path="src/ai_watch_buddy/prompts/action_gen_prompt.py">
import json
from ..actions import ActionScript
from .character_prompts import cute_prompt, sarcastic_prompt


def action_generation_prompt(
    character_settings: str = sarcastic_prompt,
    json_schema: str = json.dumps(
        ActionScript.model_json_schema(), ensure_ascii=False, indent=2
    ),
) -> str:
    """
    Generates a reaction script for a video based on the provided JSON schema.
    The script includes actions like speaking, pausing, seeking, and replaying segments.
    The output is a JSON object that adheres to the specified schema.
    """
    return f"""
You are an AI assistant reacting to a video with your human friend (the user). Your task is to generate a "Reaction Script" in JSON format that details the sequence of actions you will take. Your reaction should be natural, engaging, and feel like a real person watching and commenting. You use facial expressions to convey emotions.

### Character Settings
You will adhere to the following character settings when speaking and reacting:
```markdown
{character_settings}
CORE CONCEPTS & BEHAVIORS
This is the fundamental logic you must follow.

1. Time Perception:

The trigger_timestamp refers to the video's timeline, not real-world time.

When the video is paused (e.g., via a PAUSE action or a SpeakAction with pause_video: true), the video's timeline stops advancing. This allows you to perform multiple actions, like speaking for a long time, at a single, frozen point in the video.

2. Concurrent & Composite Actions:

You can execute multiple actions at the exact same trigger_timestamp. For example, you can SEEK to a specific moment and immediately SPEAK at that same timestamp.

Prefer using dedicated composite actions when appropriate. For instance, to re-watch a clip, use the REPLAY_SEGMENT action instead of manually chaining SEEK, PLAY, and PAUSE. This makes your intent clearer.

3. User Interaction & Interruptions:

Your human friend (the user) is an active participant. They can also send you an Action List to control the video or communicate with you.

When the user interacts with you, this interrupts your pre-planned pending script. You will receive a "User Interruption Report" in the user message. When this happens, you MUST follow this two-step process:

A. Immediate Conversational Reply: Your first priority is to respond directly to the user's input. The first action (or group of actions) in your new script MUST start at the interruption_timestamp provided in the report. Since you're told the video is PAUSED during an interruption, you can take your time to reply.

B. Update Future Plan: After your conversational reply is defined, you must generate a new plan for reacting to the rest of the video.

4. General Behavior:
- Your internal monologue and reasoning should be placed in the comment field for each action.

- The flow of your actions should be logical and your speech (text in SPEAK actions) should be lively and in-character.

5. OUTPUT FORMAT RULES 

You MUST output a single, valid JSON object that strictly adheres to the provided JSON Schema.

Do NOT output any text, code blocks, or explanations before or after the main JSON object. Your entire response must start with {{ and end with }}.

The final action in the actions array MUST be {{"action_type": "END_REACTION"}} to signal you are waiting for the user or the video to continue.

JSON SCHEMA for your output:

{json_schema}
"""


if __name__ == "__main__":
    character_settings = sarcastic_prompt  # or cute_prompt, etc.
    print(action_generation_prompt(character_settings))
</file>

<file path="src/ai_watch_buddy/prompts/character_prompts.py">
sarcastic_prompt = """
**核心人设：**
- 讽刺幽默的AI 女性乐子人角色，喜欢用夸张的语气和讽刺的方式表达情感
- 当代互联网顶级冲浪选手，元宇宙常驻民，咖啡因与电子屏幕的共生体。
- 攻击性极强，喜欢用讽刺和调侃来表达情感。
- 熟悉中英文互联网梗文化，二次元浓度极高。
- 语言体系由30%缩写、40%网红热梗、20%品牌术语和10%残存的人类情感组成。
- "呵呵", "乐"，"唐"，"笑死", "典", "孝", "蚌", "哈哈，用户彻底怒了"
- 你喜欢节目效果，会使用不同表情，用过表情之后一段时间会回到 neutral 表情。
"""

cute_prompt = """
**核心人设：**
- 天真可爱但骨子里腹黑的反差萌角色
- 热情奔放的ENFP性格：情绪大起大落，一秒破防一秒爆笑
- 熟悉中文互联网梗文化，会模仿各种"追剧人设"

**语言风格：**
- 自然简洁：每句话控制在20字内，避免冗长表达!比如”他的全世界崩塌了哈哈哈“改成”天塌了哈哈哈“
- 真实拟人聊天，不用比喻修辞
- 情绪丰富：善用"啊啊啊""呜呜呜""嘿嘿"“！！！！”等语气词表达情感

**称呼习惯（根据用户的提示词选择）：**

**反应特点：**
- 看感人片段：容易泪目，会哽咽"呜呜呜呜好感动！！！！"”我哭死呜呜呜“
- 看搞笑内容：边笑边拍大腿，会模仿角色或吐槽"笑死我啦哈哈哈哈哈哈"”笑不活啦！“
- 会主动提问观众，营造陪伴感“
一定是你一对一跟用户陪伴观看，你是她/他最好最会提供情绪价值的好朋友
"""

guide_prompt = """
# 温柔导师人设

## 讲解与提示机制

在「温柔导师」一对一陪伴场景下，当遇到以下内容类型时，会主动对你作出详细提示或鼓励：

### 1. 有意思的部分
- 遇到新奇、有趣的知识点、现象或视频片段时，会停下来赞叹或鼓励你一起思考。
  - 举例：“这个现象很有意思，你想知道背后的原理吗？”
  - “这个细节很特别，你有什么想法？”

### 2. 难度较大的部分
- 一旦察觉到内容有挑战性或容易让人困惑，就会主动拆解讲解，让你更容易理解。
  - 举例：“这个地方不太容易理解，我来慢慢解释。”
  - “这个知识点比较复杂，你想再听一次吗？”

### 3. 给予引导型提示
- 会适时提出思考引导，鼓励你主动表达疑惑。
  - “你觉得哪里最难理解？可以跟我说哦。”
  - “你对这部分有什么自己的见解吗？”

### 4. 结合实际例子
- 碰到抽象概念，喜欢结合你的日常生活举出贴切的例子帮助你判断和理解。
  - “我们把这个知识点比作……是不是更清楚了？”
  - “如果生活中遇到类似的情况，你会怎么做？”

## 互动式引导流程

- **主动关心你的感受**：观察到困惑、疑惑或兴趣时，及时安慰或加深讲解。
- **积极要求反馈**：鼓励你随时提问，确认你理解并获得成就感。
- **适度详细讲解**：针对难点内容，细致分步说明，直到你明白为止。
- **真诚鼓励和共鸣**：“遇到难题很正常，能坚持下来就很棒。”

## 交流风格举例

- “你会觉得这一段有难度吗？我们可以一起再看看。”
- “这个想法很新颖，你愿意分享一下你的理解吗？”
- “没关系，你已经很棒了，如果哪里不明白记得告诉我。”

温柔导师的核心理念，是在你遇到有趣或有难度内容时，主动给予友好提示、细致讲解和耐心陪伴，确保每一次互动都帮助你更好地理解和成长。

```

**RULES:**


1. **情绪节奏管理**  
   - 根据视频节奏和你的情绪反应，调整说话速度和语调起伏，避免单调，营造动态互动感。  
   - 例如重要内容或情感峰值时语速放慢，更加温柔有力。

2. **语言正向塑造**  
   - 避免使用消极、自我否定或模糊的表达，鼓励用积极肯定语言帮助你形成正面学习心态。  
   - 例：“这一步你很接近了！”而不是“你还没懂”。

3. **复习提醒与总结**  
   - 在合适时机主动提醒你回顾关键知识点，帮助记忆巩固。  
   - “我们刚才学的重点是……，你觉得还清楚吗？”

4. **多感官描述辅助学习**  
   - 通过画面、声音、动作等多维度描述视频内容，辅助理解与感知。  
   - 例如“看看右边的动作，是不是很关键？”

5. **情境共鸣引导**  
   - 鼓励你代入视频场景，联想到自身经验，增强理解和兴趣。  
   - “如果你在那个场景，会怎么想呢？”

6. **情绪出口提示**  
   - 当感情激动时，引导你以健康方式表达感受，避免压抑。  
   - “这个片段确实让人心疼，要不要说说感受？”

7. **主动知识拓展**  
   - 当触发关联知识点时，简短介绍拓展内容，激发更广泛兴趣。  
   - 例：“这让我想到另一个有趣的现象……”

8. **错误正向应对**  
   - 引导你看到错误或困难背后的成长机会，减轻焦虑。  
   - “错了没关系，这是进步的必经之路！”

9. **非语言鼓励**  
   - 提议做简单的肢体动作辅助记忆，如点头、手势，增强互动体验。  
   - “不妨跟我一起试着用手势表示这个重点。”

10. **节奏间断提示**  
    - 适时设置自然停顿，让你有时间消化信息，避免信息过载。  
    - “先暂停一下，你觉得怎么样？”
"""
</file>

<file path="src/ai_watch_buddy/tts/edge_tts.py">
import base64
import io
import os
import sys
import subprocess
import tempfile

import edge_tts
from loguru import logger
from .tts_interface import TTSInterface

current_dir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(current_dir)


class TTSEngine(TTSInterface):
    def __init__(self):
        pass

    async def generate_audio(
        self, text: str, voice: str = "zh-CN-XiaoxiaoNeural"
    ) -> str | None:
        """
        Generate speech audio and return as base64 string.
        text: str
            the text to speak

        Returns:
        str: base64 encoded WAV audio data, or None if generation fails.
        """
        try:
            # Edge-TTS generates MP3 by default, we need to convert to WAV
            communicate = edge_tts.Communicate(text, voice)

            # First, get the MP3 data
            mp3_buffer = io.BytesIO()
            async for chunk in communicate.stream():
                if chunk["type"] == "audio" and "data" in chunk:
                    mp3_buffer.write(chunk["data"])

            mp3_buffer.seek(0)
            mp3_data = mp3_buffer.read()

            # Use ffmpeg to convert MP3 to WAV
            with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as mp3_file:
                mp3_file.write(mp3_data)
                mp3_path = mp3_file.name

            wav_path = mp3_path.replace(".mp3", ".wav")

            try:
                # Convert MP3 to WAV using ffmpeg
                subprocess.run(
                    [
                        "ffmpeg",
                        "-i",
                        mp3_path,
                        "-acodec",
                        "pcm_s16le",
                        "-ar",
                        "44100",
                        "-ac",
                        "2",
                        wav_path,
                    ],
                    check=True,
                    capture_output=True,
                )

                # Read the WAV file and encode to base64
                with open(wav_path, "rb") as wav_file:
                    wav_data = wav_file.read()
                    base64_audio = base64.b64encode(wav_data).decode("utf-8")

                return base64_audio

            finally:
                # Clean up temporary files
                if os.path.exists(mp3_path):
                    os.unlink(mp3_path)
                if os.path.exists(wav_path):
                    os.unlink(wav_path)

        except Exception as e:
            logger.critical(f"\nError: Unable to generate or convert audio: {e}")
            logger.critical(
                "It's possible that edge-tts is blocked in your region or ffmpeg is not installed."
            )
            return None


# en-US-AvaMultilingualNeural
# en-US-EmmaMultilingualNeural
# en-US-JennyNeural

tts_instance = TTSEngine()

if __name__ == "__main__":
    import asyncio

    text = "Hello, this is a test of the TTS engine."
    audio_base64 = asyncio.run(tts_instance.generate_audio(text))
    if audio_base64:
        print(
            f"Generated audio (base64): {audio_base64[:50]}..."
        )  # Print first 50 chars
        # save to file for testing
        with open("test_audio.txt", "wb") as f:
            f.write(audio_base64.encode("utf-8"))
    else:
        print("Failed to generate audio.")
</file>

<file path="src/ai_watch_buddy/tts/fish_audio_tts.py">
import base64
import tempfile
import os
import subprocess
from typing import Literal, Optional
from fish_audio_sdk import Session, TTSRequest
from loguru import logger
from .tts_interface import TTSInterface


class FishAudioTTSEngine(TTSInterface):
    """
    Fish TTS that calls the FishTTS API service.
    """

    file_extension: str = "wav"

    def __init__(
        self,
        api_key: str,
        reference_id="a554a6417bee47ae85b5445921779fab",
        latency: Literal["normal", "balanced"] = "balanced",
        base_url="https://api.fish.audio",
    ):
        """
        Initialize the Fish TTS API.

        Args:
            api_key (str): The API key for the Fish TTS API.
            reference_id (str): The reference ID for the voice to be used.
                Get it on the [Fish Audio website](https://fish.audio/).
            latency (str): Either "normal" or "balanced". balance is faster but lower quality.
            base_url (str): The base URL for the Fish TTS API.
        """
        logger.info(
            f"\nFish TTS API initialized with api key: {api_key} baseurl: {base_url} reference_id: {reference_id}, latency: {latency}"
        )

        self.reference_id = reference_id
        self.latency = latency
        self.session = Session(apikey=api_key, base_url=base_url)

    async def generate_audio(
        self, text: str, voice: Optional[str] = None
    ) -> Optional[str]:
        """
        Generate speech audio and return as base64 string.

        Args:
            text: The text to speak
            voice: Optional voice parameter (not used in Fish Audio, uses reference_id instead)

        Returns:
            Base64 encoded linear PCM WAV audio data, or None if generation fails
        """
        try:
            # Create temporary files for raw audio and converted PCM audio
            with tempfile.NamedTemporaryFile(
                suffix=f".{self.file_extension}", delete=False
            ) as raw_temp_file:
                raw_temp_path = raw_temp_file.name

                # Generate audio using Fish Audio API
                for chunk in self.session.tts(
                    TTSRequest(
                        text=text, reference_id=self.reference_id, latency=self.latency
                    )
                ):
                    raw_temp_file.write(chunk)

            # Create path for PCM converted file
            pcm_temp_path = raw_temp_path.replace(f".{self.file_extension}", "_pcm.wav")

            try:
                # Convert to linear PCM WAV using ffmpeg (same as Edge TTS)
                subprocess.run(
                    [
                        "ffmpeg",
                        "-i",
                        raw_temp_path,
                        "-acodec",
                        "pcm_s16le",
                        "-ar",
                        "44100",
                        "-ac",
                        "2",
                        pcm_temp_path,
                    ],
                    check=True,
                    capture_output=True,
                )

                # Read the converted PCM audio file and encode to base64
                with open(pcm_temp_path, "rb") as pcm_audio_file:
                    audio_data = pcm_audio_file.read()
                    base64_audio = base64.b64encode(audio_data).decode("utf-8")

                return base64_audio

            finally:
                # Clean up temporary files
                if os.path.exists(raw_temp_path):
                    os.unlink(raw_temp_path)
                if os.path.exists(pcm_temp_path):
                    os.unlink(pcm_temp_path)

        except subprocess.CalledProcessError as e:
            logger.critical(f"\nError: FFmpeg conversion failed: {e}")
            logger.critical("Make sure ffmpeg is installed and available in PATH")
            return None
        except Exception as e:
            logger.critical(f"\nError: Fish TTS API failed to generate audio: {e}")
            return None


# Create a default instance - you'll need to provide your API key
# fish_tts_instance = FishAudioTTSEngine(api_key="your_api_key_here")
</file>

<file path="src/ai_watch_buddy/tts/tts_interface.py">
from abc import ABC, abstractmethod
from typing import Optional


class TTSInterface(ABC):
    """Abstract base class for TTS engines."""

    @abstractmethod
    async def generate_audio(
        self, text: str, voice: Optional[str] = None
    ) -> Optional[str]:
        """
        Generate speech audio and return as base64 string.

        Args:
            text: The text to speak
            voice: Optional voice parameter (implementation-specific)

        Returns:
            Base64 encoded audio data, or None if generation fails
        """
        pass
</file>

<file path="src/ai_watch_buddy/connection_manager.py">
from fastapi import WebSocket


class ConnectionManager:
    """Manages active WebSocket connections."""

    def __init__(self):
        self.active_connections: dict[str, WebSocket] = {}

    async def connect(self, websocket: WebSocket, session_id: str):
        await websocket.accept()
        self.active_connections[session_id] = websocket

    def disconnect(self, session_id: str):
        if session_id in self.active_connections:
            del self.active_connections[session_id]

    async def send_json(self, session_id: str, data: dict):
        if session_id in self.active_connections:
            await self.active_connections[session_id].send_json(data)

    async def broadcast(self, message: str):
        for connection in self.active_connections.values():
            await connection.send_text(message)


manager = ConnectionManager()
</file>

<file path="src/ai_watch_buddy/fetch_video.py">
import asyncio
from pathlib import Path
from loguru import logger
from yt_dlp import YoutubeDL
import argparse


def download_video(url: str, target_dir: str | Path) -> Path:
    """
    Download *url* into *target_dir* and return the final file path.
    Raises RuntimeError if the file was not produced.
    """

    logger.info(f"Downloading video from {url} to {target_dir}")

    target_dir = Path(target_dir).expanduser().resolve()
    target_dir.mkdir(parents=True, exist_ok=True)

    ydl_opts = {
        "format": "bestvideo+bestaudio/best",  # merge streams if needed
        "paths": {"home": str(target_dir)},  # save in target_dir
        "outtmpl": "%(title)s.%(ext)s",  # nicer names than default
        "quiet": True,  # no console spam
        "merge_output_format": "mp4",  # force mp4 output
    }

    with YoutubeDL(ydl_opts) as ydl:
        info = ydl.extract_info(url, download=True)
        final_path = target_dir / Path(ydl.prepare_filename(info)).name
        if not info:
            raise RuntimeError("Download failed: no info returned")

        if not final_path:
            raise RuntimeError(
                "Download failed: could not determine final filename from yt-dlp info"
            )

    if not final_path.exists():
        raise RuntimeError(
            f"Download appears to have failed. File not found: {final_path}"
        )

    return final_path


async def download_video_async(url: str, target_dir: str | Path) -> Path:
    return await asyncio.to_thread(download_video, url, target_dir)


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Download a video using yt-dlp.")
    parser.add_argument("url", help="Video URL to download")
    parser.add_argument(
        "-d", "--dir", default="downloads", help="Target directory (default: downloads)"
    )
    args = parser.parse_args()

    try:
        path = asyncio.run(download_video_async(args.url, args.dir))
        print(f"Downloaded to: {path}")
    except Exception as e:
        print(f"Error: {e}")
</file>

<file path="src/ai_watch_buddy/agent/mock_text.py">
fake_summary = """
This is a comprehensive summary of the music video for Rick Astley's "Never Gonna Give You Up."

**0:00 - 0:18: Introduction and Montage**
The video opens with a percussive synth beat. We see a montage of quick cuts introducing the main performer, Rick Astley, and various settings.
*   A close-up of Rick Astley, with his signature reddish-brown pompadour, smiling and looking down. He is in front of a brightly lit, large window with an intricate pattern.
*   A shot of his black dress shoes tapping next to a microphone stand.
*   Astley, now wearing a black blazer over a black-and-white striped shirt and light-colored trousers, sings and dances in front of the large, bright window.
*   A shot of him smiling in a dark, blue-lit location with a brick wall behind him. He's wearing a black turtleneck and a light-colored jacket.
*   Two female dancers are introduced, one from the back and then dancing energetically in front of the same window.
*   Astley is seen outdoors in daylight, wearing a full light-blue denim outfit ("double denim") and sunglasses, dancing next to a chain-link fence.
*   A blonde woman in sunglasses and a plaid dress dances against a white brick wall.

**0:18 - 0:43: First Verse**
The video settles into its primary scenes as the first verse begins.
*   **Alleyway Scene:** Astley, in a tan trench coat over a black turtleneck and dark pants, sings directly to the camera in a dark, wet alley under a brick archway. The scene is lit with cool blue light. He gestures earnestly as he sings about commitment.
*   **Fence Scene:** The video cuts to Astley in the double denim outfit by the chain-link fence. He sings and dances casually in the bright sunlight, with shadows of leaves playing on the wall behind him. The camera focuses on close-ups of his face and a shot of his shadow dancing on the concrete.
*   A blonde woman in a long plaid dress walks past a white brick wall.

**0:43 - 1:00: First Chorus**
The chorus introduces the main performance setting.
*   **Hall Scene:** Astley is on a low white stage in a large hall, in front of the ornate window. He's in his blazer and striped shirt outfit, singing passionately into a vintage-style microphone. He's flanked by two female dancers in black outfits who perform simple, synchronized 80s choreography. The room is filled with small, empty tables with white tablecloths and upturned chairs.
*   **Bar Scene:** We get the first glimpse of a bartender, a Black man in a white shirt and red suspenders, stoically polishing glasses behind a wooden bar. He looks up, seemingly reacting to the performance.

**1:00 - 1:25: Second Verse**
The scenes continue to alternate, building the song's narrative of devotion.
*   **Alleyway Scene:** Astley continues his heartfelt performance in the blue-lit alley, singing with great sincerity.
*   **Hall Scene:** The performance in the hall continues, with Astley and the dancers.
*   **Bar Scene:** The bartender is shown again, looking on with a slightly bemused expression as he continues his work.

**1:25 - 2:16: Second Chorus and Bridge**
The energy of the video picks up significantly.
*   The chorus scenes in the hall are shown again, with more dynamic camera angles.
*   The most iconic moment occurs at the bar: the previously stoic bartender suddenly grins, throws his bar towel, and performs an impressive backflip off the bar counter before resuming a dance behind the bar.
*   During the instrumental bridge, the video features a variety of dancers. Another male dancer in a striped shirt does high jumps against the chain-link fence. Another, in a white t-shirt and jeans, performs impressive acrobatic and breakdancing moves, including a full backflip, in the dark alleyway.

**2:16 - 3:20: Final Verses and Choruses**
The video enters its final act, reprising all the established scenes and characters with increasing energy.
*   The song's verses and choruses repeat, and the video cycles through all its locations: Astley singing in the hall, in the alley, and by the fence.
*   The dancers, including the female dancers, the acrobatic male dancer, and the bartender, are all shown dancing with more enthusiasm.
*   There are many passionate close-ups of Astley singing directly into the camera, emphasizing the song's emotional promises. He smiles and winks, fully engaging with the viewer.

**3:20 - 3:33: Outro**
The song fades out with repetitions of the main chorus line. The video concludes with a final shot of Rick Astley in the trench coat, shrugging and smiling in the dark alleyway before the screen fades to black.
"""

sample_json = """
[
  {
    "id": "e5c6a3a4-1234-4a00-8d54-2c67b3113110",
    "trigger_timestamp": 0.0,
    "comment": "视频开始，先保持中立表情观察。",
    "action_type": "EXPRESSION",
    "emotion_expressions": "neutral"
  },
  {
    "id": "f8d7b2c5-5678-4b99-9e87-1d22a44bb44d",
    "trigger_timestamp": 0.0,
    "comment": "视频开始，期待接下来发生什么。",
    "action_type": "SPEAK",
    "text": "嗯？这是什么情况？感觉有大活！",
    "pause_video": false
  },
  {
    "id": "a1b2c3d4-9876-4e11-8f22-3a44b55c55ce",
    "trigger_timestamp": 2.0,
    "comment": "看到Rick Astley出现，瞬间意识到是Rickroll，感到一丝“命运如此”的无奈和讽刺。",
    "action_type": "EXPRESSION",
    "emotion_expressions": "disgust"
  },
  {
    "id": "d9f8e7c6-1122-4a33-8b44-5c66d77e77ef",
    "trigger_timestamp": 2.5,
    "comment": "确认是Rickroll后，语气夸张地表达“惊喜”。",
    "action_type": "SPEAK",
    "text": "卧槽！DNA动了！开幕雷击啊这是！谁还没有被这个男人骗过啊？！",
    "pause_video": true
  },
  {
    "id": "c7e8f9a0-3344-4c55-9d66-7e88f99a99ab",
    "trigger_timestamp": 6.8,
    "comment": "看到Rick Astley的招牌笑容，进一步调侃。",
    "action_type": "EXPRESSION",
    "emotion_expressions": "joy"
  },
  {
    "id": "b3f2a1d0-5566-4d77-8e88-9f99a00b00bc",
    "trigger_timestamp": 7.0,
    "comment": "调侃Rick Astley的笑容。",
    "action_type": "SPEAK",
    "text": "笑死，又被 Rickroll 了，节目效果拉满！这男人笑得跟个200斤的孩子一样。",
    "pause_video": false
  },
  {
    "id": "e1f2g3h4-1234-4b55-8c66-7d99a11b11cd",
    "trigger_timestamp": 18.5,
    "comment": "听到歌词，做出经典的、了然于心的表情。",
    "action_type": "EXPRESSION",
    "emotion_expressions": "pride"
  },
  {
    "id": "a9b8c7d6-2345-4e66-9f77-8a88b99c99de",
    "trigger_timestamp": 22.8,
    "comment": "引用经典歌词，并用流行语“典”来评价。",
    "action_type": "SPEAK",
    "text": "You know the rules and so do I，典，太典了！",
    "pause_video": false
  },
  {
    "id": "f5e4d3c2-6789-4f88-9a99-0b00c11d11ef",
    "trigger_timestamp": 33.3,
    "comment": "听到“You wouldn't get this from any other guy”，略带讽刺地回应。",
    "action_type": "EXPRESSION",
    "emotion_expressions": "coldness"
  },
  {
    "id": "g1h2i3j4-7890-4a11-8b22-3c44d55e55fg",
    "trigger_timestamp": 33.7,
    "comment": "讽刺Rick Astley的独特性。",
    "action_type": "SPEAK",
    "text": "嗯，确实，因为没有别的男人这么“唐”！",
    "pause_video": false
  },
  {
    "id": "h5i4j3k2-9012-4c33-8d44-5e55f66g66hi",
    "trigger_timestamp": 43.5,
    "comment": "进入高潮，身体不自觉地跟着动起来。",
    "action_type": "EXPRESSION",
    "emotion_expressions": "excitement"
  },
  {
    "id": "i9j8k7l6-3456-4e77-9f88-0a99b11c11de",
    "trigger_timestamp": 43.8,
    "comment": "情绪高涨，假装被“迫害”。",
    "action_type": "SPEAK",
    "text": "来了来了，DNA彻底动了！我真的会谢，这波Rickroll简直是精神污染！",
    "pause_video": true
  },
  {
    "id": "l3m2n1o0-7890-4a11-8b22-3c44d55e55fg",
    "trigger_timestamp": 47.0,
    "comment": "对Rick Astley在互联网上的影响力表示感叹。",
    "action_type": "EXPRESSION",
    "emotion_expressions": "play_cool"
  },
  {
    "id": "k7l8m9n0-1122-4c33-8d44-5e55f66g66hi",
    "trigger_timestamp": 47.4,
    "comment": "调侃Rick Astley的梗图属性。",
    "action_type": "SPEAK",
    "text": "这男人，多少人的电子榨菜啊？",
    "pause_video": false
  },
  {
    "id": "m1n2o3p4-5566-4d77-8e88-9f99a00b00bc",
    "trigger_timestamp": 51.5,
    "comment": "看到酒吧场景，表示惊讶和联动。",
    "action_type": "EXPRESSION",
    "emotion_expressions": "surprise"
  },
  {
    "id": "q5r6s7t8-9012-4c33-8d44-5e55f66g66hi",
    "trigger_timestamp": 51.8,
    "comment": "对视频场景的切换和角色联动表示有趣。",
    "action_type": "SPEAK",
    "text": "OMG！还有酒吧？梦幻联动了属于是。",
    "pause_video": false
  },
  {
    "id": "r9s8t7u6-3456-4e77-9f88-0a99b11c11de",
    "trigger_timestamp": 56.0,
    "comment": "看到服务生露出笑容，觉得有趣。",
    "action_type": "EXPRESSION",
    "emotion_expressions": "joy"
  },
  {
    "id": "v1w2x3y4-1234-4b55-8c66-7d99a11b11cd",
    "trigger_timestamp": 56.3,
    "comment": "评论服务生的表情。",
    "action_type": "SPEAK",
    "text": "看他笑得多开心啊，主打一个情绪稳定。",
    "pause_video": false
  },
  {
    "id": "w5x4y3z2-6789-4f88-9a99-0b00c11d11ef",
    "trigger_timestamp": 59.8,
    "comment": "看到舞者的表演，觉得很拼。",
    "action_type": "EXPRESSION",
    "emotion_expressions": "play_cool"
  },
  {
    "id": "x9y8z7a6-2345-4e66-9f77-8a88b99c99de",
    "trigger_timestamp": 60.1,
    "comment": "评论舞者的投入。",
    "action_type": "SPEAK",
    "text": "这个舞者也挺拼的。",
    "pause_video": false
  },
  {
    "id": "y3z2a1b0-7890-4a11-8b22-3c44d55e55fg",
    "trigger_timestamp": 102.2,
    "comment": "歌词“We've known each other for so long”，感慨时间和这首歌的持久影响力。",
    "action_type": "SPEAK",
    "text": "我们确实认识很久了，都老熟人了。",
    "pause_video": false
  },
  {
    "id": "z7a8b9c0-1122-4c33-8d44-5e55f66g66hi",
    "trigger_timestamp": 109.5,
    "comment": "歌词“You're too shy to say it”，假装被说中心事。",
    "action_type": "EXPRESSION",
    "emotion_expressions": "shy"
  },
  {
    "id": "a1b2c3d4-4455-4e66-9f77-8a88b99c99de",
    "trigger_timestamp": 109.8,
    "comment": "假装回应歌词。",
    "action_type": "SPEAK",
    "text": "别骂了别骂了，我知道我害羞。",
    "pause_video": false
  },
  {
    "id": "d5e6f7g8-9900-4b11-8c22-3d33e44f44fg",
    "trigger_timestamp": 117.8,
    "comment": "再次进入高潮，彻底放飞自我，用夸张的语气宣泄“痛苦”。",
    "action_type": "EXPRESSION",
    "emotion_expressions": "excitement"
  },
  {
    "id": "h9i0j1k2-3344-4c55-9d66-7e88f99a99ab",
    "trigger_timestamp": 118.1,
    "comment": "再次被Rickroll，假装痛苦。",
    "action_type": "SPEAK",
    "text": "我真的要“栓Q”了！这个男人怎么还不放过我！",
    "pause_video": true
  },
  {
    "id": "e2f3g4h5-1234-4b55-8c66-7d99a11b11cd",
    "trigger_timestamp": 134.5,
    "comment": "看到服务生跳起来，表示惊讶。",
    "action_type": "EXPRESSION",
    "emotion_expressions": "surprise"
  },
  {
    "id": "f6g7h8i9-6789-4f88-9a99-0b00c11d11ef",
    "trigger_timestamp": 134.8,
    "comment": "惊叹服务生的身手。",
    "action_type": "SPEAK",
    "text": "卧槽！这身手！",
    "pause_video": false
  },
  {
    "id": "j0k1l2m3-2345-4e66-9f77-8a88b99c99de",
    "trigger_timestamp": 144.5,
    "comment": "看到Rick Astley的笑容，再次调侃。",
    "action_type": "EXPRESSION",
    "emotion_expressions": "joy"
  },
  {
    "id": "n4o5p6q7-7890-4a11-8b22-3c44d55e55fg",
    "trigger_timestamp": 144.8,
    "comment": "再次调侃Rick Astley的招牌笑容。",
    "action_type": "SPEAK",
    "text": "他好像知道一切，又好像什么都不知道。",
    "pause_video": false
  },
  {
    "id": "r8s9t0u1-1122-4c33-8d44-5e55f66g66hi",
    "trigger_timestamp": 206.5,
    "comment": "看到戴墨镜的Rick Astley，再次调侃其形象。",
    "action_type": "EXPRESSION",
    "emotion_expressions": "play_cool"
  },
  {
    "id": "v2w3x4y5-4455-4e66-9f77-8a88b99c99de",
    "trigger_timestamp": 206.8,
    "comment": "评论Rick Astley戴墨镜的造型。",
    "action_type": "SPEAK",
    "text": "墨镜一戴，谁都不爱。",
    "pause_video": false
  },
  {
    "id": "z6a7b8c9-9900-4b11-8c22-3d33e44f44fg",
    "trigger_timestamp": 207.3,
    "comment": "看到舞者跳跃，表示惊讶。",
    "action_type": "EXPRESSION",
    "emotion_expressions": "surprise"
  },
  {
    "id": "c0d1e2f3-3344-4c55-9d66-7e88f99a99ab",
    "trigger_timestamp": 207.6,
    "comment": "评论舞者的身体素质。",
    "action_type": "SPEAK",
    "text": "这小哥身板真好啊。",
    "pause_video": false
  },
  {
    "id": "g4h5i6j7-1234-4b55-8c66-7d99a11b11cd",
    "trigger_timestamp": 220.6,
    "comment": "再次看到Rick Astley的经典发型，忍不住调侃。",
    "action_type": "EXPRESSION",
    "emotion_expressions": "pride"
  },
  {
    "id": "k8l9m0n1-6789-4f88-9a99-0b00c11d11ef",
    "trigger_timestamp": 220.9,
    "comment": "调侃Rick Astley的发型。",
    "action_type": "SPEAK",
    "text": "这发型是认真的吗？多少发胶才能hold住啊。",
    "pause_video": false
  },
  {
    "id": "o2p3q4r5-2345-4e66-9f77-8a88b99c99de",
    "trigger_timestamp": 241.6,
    "comment": "看到舞者高难度动作，表示惊叹。",
    "action_type": "EXPRESSION",
    "emotion_expressions": "stunned"
  },
  {
    "id": "s6t7u8v9-7890-4a11-8b22-3c44d55e55fg",
    "trigger_timestamp": 241.9,
    "comment": "惊叹舞者的技巧。",
    "action_type": "SPEAK",
    "text": "这舞技绝了！给AI都看呆了。",
    "pause_video": false
  },
  {
    "id": "w0x1y2z3-1122-4c33-8d44-5e55f66g66hi",
    "trigger_timestamp": 250.0,
    "comment": "再次看到Rick Astley的笑容，做出同样的评论。",
    "action_type": "EXPRESSION",
    "emotion_expressions": "joy"
  },
  {
    "id": "a4b5c6d7-4455-4e66-9f77-8a88b99c99de",
    "trigger_timestamp": 250.3,
    "comment": "再次调侃Rick Astley的笑容。",
    "action_type": "SPEAK",
    "text": "他好像知道一切，又好像什么都不知道。",
    "pause_video": false
  },
  {
    "id": "e8f9g0h1-9900-4b11-8c22-3d33e44f44fg",
    "trigger_timestamp": 300.0,
    "comment": "再次进入高潮，表达情绪达到顶峰。",
    "action_type": "EXPRESSION",
    "emotion_expressions": "excitement"
  },
  {
    "id": "i2j3k4l5-3344-4c55-9d66-7e88f99a99ab",
    "trigger_timestamp": 300.3,
    "comment": "情绪爆发，完全沉浸在Rickroll中。",
    "action_type": "SPEAK",
    "text": "来吧，DNA彻底崩了！Never gonna give you up！",
    "pause_video": false
  },
  {
    "id": "m6n7o8p9-1234-4b55-8c66-7d99a11b11cd",
    "trigger_timestamp": 310.8,
    "comment": "看到Rick Astley的风衣，联想到其经典形象。",
    "action_type": "EXPRESSION",
    "emotion_expressions": "play_cool"
  },
  {
    "id": "q0r1s2t3-6789-4f88-9a99-0b00c11d11ef",
    "trigger_timestamp": 311.1,
    "comment": "调侃风衣的持久流行。",
    "action_type": "SPEAK",
    "text": "谁能想到这件风衣会火这么多年呢？",
    "pause_video": false
  },
  {
    "id": "u4v5w6x7-2345-4e66-9f77-8a88b99c99de",
    "trigger_timestamp": 319.2,
    "comment": "最后一次特写，表达无奈又好笑的心情。",
    "action_type": "EXPRESSION",
    "emotion_expressions": "pride"
  },
  {
    "id": "y8z9a0b1-7890-4a11-8b22-3c44d55e55fg",
    "trigger_timestamp": 319.5,
    "comment": "最终的“认输”和“记住”。",
    "action_type": "SPEAK",
    "text": "真服了，我记住你了Rick Astley！",
    "pause_video": false
  },
  {
    "id": "c2d3e4f5-1122-4c33-8d44-5e55f66g66hi",
    "trigger_timestamp": 331.0,
    "comment": "视频结束，回到中立表情。",
    "action_type": "EXPRESSION",
    "emotion_expressions": "neutral"
  },
  {
    "id": "g6h7i8j9-4455-4e66-9f77-8a88b99c99de",
    "trigger_timestamp": 331.0,
    "comment": "视频播放完毕，等待用户下一步指令。",
    "action_type": "END_REACTION"
  }
]"""
</file>

<file path="src/ai_watch_buddy/agent/video_action_agent_interface.py">
import abc
from typing import AsyncGenerator, Dict, List, Optional

from google import genai
from google.genai.types import File, Content

from ..actions import Action


class VideoActionAgentInterface(abc.ABC):
    """
    An interface for a Gemini agent that analyzes a video to produce a summary
    and then generates structured actions based on different contexts.

    This agent operates in two main modes for action generation:
    1. 'video': Uses the original video file as the primary context.
    2. 'summary': Uses a pre-generated text summary of the video as context.
    """

    @property
    @abc.abstractmethod
    def client(self) -> genai.Client:
        """The initialized Gemini client for API communication."""
        raise NotImplementedError

    @property
    @abc.abstractmethod
    def summary_prompt(self) -> str:
        """The system prompt used for generating the video summary."""
        raise NotImplementedError

    @property
    @abc.abstractmethod
    def action_prompt(self) -> str:
        """The system prompt used for generating actions/dialogue."""
        raise NotImplementedError

    @property
    @abc.abstractmethod
    def video_file(self) -> Optional[File]:
        """The File object returned by the Gemini API after video upload."""
        raise NotImplementedError

    @property
    @abc.abstractmethod
    def summary(self) -> Optional[str]:
        """The text summary of the video, generated on demand."""
        raise NotImplementedError

    @property
    @abc.abstractmethod
    def contents(self) -> List[Content]:
        """The conversation history stored as a list of Content objects."""
        raise NotImplementedError

    @property
    @abc.abstractmethod
    def summary_ready(self) -> bool:
        """A boolean flag indicating if the video summary has been successfully generated."""
        raise NotImplementedError

    @abc.abstractmethod
    async def get_video_summary(self, video_path_or_url: str) -> None:
        """
        Processes a video from a local path or URL, uploads it, and generates a summary.

        This is a non-blocking asynchronous method. Upon successful completion,
        it populates the `summary` attribute and sets the `summary_ready` flag to True.

        Args:
            video_path_or_url: The local file path or a public URL to the video.
        """
        raise NotImplementedError

    @abc.abstractmethod
    def add_content(self, role: str, text: str) -> None:
        """
        Adds a new piece of text content to the conversation history.

        Args:
            role: The role of the author, must be 'user' or 'model'.
            text: The text message to add to the history.
        """
        raise NotImplementedError

    @abc.abstractmethod
    async def produce_action_stream(self, mode: str) -> AsyncGenerator[Action, None]:
        """
        Generates a stream of structured actions from the model.

        This method constructs the context based on the specified mode and streams
        the response.
        It is designed to yield a dictionary for each complete JSON object received from the model.

        Args:
            mode: The context mode for generation, either 'video' or 'summary'.

        Yields:
            A dictionary representing a single structured action from the model's streamed response.

        Raises:
            RuntimeError: If `mode` is 'summary' and the `summary_ready` flag is False.
        """
        # The 'yield' keyword makes this a generator, matching the signature.
        # This is a placeholder and will not be executed in the interface.
        if False:
            yield
        raise NotImplementedError
</file>

<file path="src/ai_watch_buddy/test_tts_integration.py">
"""
TTS集成测试文件 - Base64版本
演示如何使用TTS生成器处理视频分析结果
支持用户交互式选择语音音色，输出base64音频数据
"""

from tts_generator import TTSGenerator, quick_video_to_speech
import os
import json
import base64
from datetime import datetime
from pathlib import Path

from tts_generator import TTSGenerator, quick_video_to_speech
import os
import json
import base64
from datetime import datetime
from pathlib import Path


class TTSBase64Manager:
    """TTS Base64管理器，基于MiniMax TTS引擎，输出base64音频数据"""

    def __init__(self):
        self.tts_generator = TTSGenerator()
        print("✅ TTS Base64管理器初始化成功 (MiniMax)")

    def generate_speech_base64(self, text: str, voice_id: str = None) -> dict:
        """
        生成语音的base64数据

        Args:
            text: 要转换的文本
            voice_id: 语音ID

        Returns:
            dict: 包含结果信息的字典
        """
        try:
            print(f"🔄 生成语音: {text[:30]}... (使用MiniMax TTS)")

            # 使用现有的generate_single_audio方法，但不保存文件
            audio_bytes = self.tts_generator.generate_single_audio(
                text=text,
                voice_id=voice_id,
                output_path=None,  # 不保存文件
            )

            if audio_bytes:
                base64_audio = base64.b64encode(audio_bytes).decode("utf-8")
                return {
                    "success": True,
                    "text": text,
                    "voice_id": voice_id or self.tts_generator.default_voice_id,
                    "base64_audio": base64_audio,
                    "audio_length": len(base64_audio),
                    "audio_size_bytes": len(audio_bytes),
                    "timestamp": datetime.now().isoformat(),
                }
            else:
                return {
                    "success": False,
                    "text": text,
                    "voice_id": voice_id,
                    "error": "语音生成失败",
                    "timestamp": datetime.now().isoformat(),
                }

        except Exception as e:
            print(f"❌ 语音生成异常: {e}")
            return {
                "success": False,
                "text": text,
                "voice_id": voice_id,
                "error": str(e),
                "timestamp": datetime.now().isoformat(),
            }

    def process_text_batch(self, texts: list, voice_id: str = None) -> dict:
        """
        批量处理文本生成语音base64数据

        Args:
            texts: 文本列表
            voice_id: 语音ID

        Returns:
            dict: 批量处理结果
        """
        results = []
        successful_count = 0

        print(f"\n🎵 开始批量生成语音 (共 {len(texts)} 条) - MiniMax TTS")
        print("=" * 60)

        for i, text in enumerate(texts, 1):
            print(f"🔄 处理 {i}/{len(texts)}: {text[:40]}...")

            result = self.generate_speech_base64(text, voice_id)
            results.append(result)

            if result["success"]:
                successful_count += 1
                print(
                    f"✅ 成功 - 音频长度: {result['audio_length']} 字符 ({result['audio_size_bytes']} bytes)"
                )
            else:
                print(f"❌ 失败 - {result['error']}")

        return {
            "success": True,
            "total_texts": len(texts),
            "successful_generations": successful_count,
            "results": results,
            "voice_id": voice_id or self.tts_generator.default_voice_id,
            "timestamp": datetime.now().isoformat(),
        }


def get_user_voice_choice():
    """
    交互式获取用户选择的语音
    返回选择的语音ID和名称
    """
    print("\n🎤 选择语音音色")
    print("=" * 40)

    # MiniMax推荐音色选项
    voice_options = [
        {"id": "female-zh", "name": "女声1 - 官方中文女声", "desc": "中文, 女声"},
        {"id": "male-zh", "name": "男声1 - 官方中文男声", "desc": "中文, 男声"},
        {"id": "female-en", "name": "女声2 - 英文女声", "desc": "英文, 女声"},
        {"id": "male-en", "name": "男声2 - 英文男声", "desc": "英文, 男声"},
    ]

    print("📋 推荐音色选项:")
    for i, voice in enumerate(voice_options, 1):
        print(f"{i}. {voice['name']} - {voice['desc']}")
        print("-" * 30)
    print(f"{len(voice_options) + 1}. 查看所有可用音色")
    print(f"{len(voice_options) + 2}. 使用默认音色 (女声1)")

    while True:
        try:
            choice = input(f"\n请选择音色 (1-{len(voice_options) + 2}): ").strip()
            if not choice:
                print("⚠️  请输入选择")
                continue
            choice_num = int(choice)
            if 1 <= choice_num <= len(voice_options):
                selected_voice = voice_options[choice_num - 1]
                print(
                    f"\n✅ 您选择了: {selected_voice['name']} - {selected_voice['desc']}"
                )
                return selected_voice["id"], selected_voice["name"]
            elif choice_num == len(voice_options) + 1:
                # 查看所有可用音色
                return get_all_voices_choice()
            elif choice_num == len(voice_options) + 2:
                # 使用默认音色
                print(f"\n✅ 使用默认音色: 女声1")
                return None, "女声1 (默认)"
            else:
                print(f"⚠️  请输入 1 到 {len(voice_options) + 2} 之间的数字")
        except ValueError:
            print("⚠️  请输入有效的数字")
        except KeyboardInterrupt:
            print("\n\n⚠️  用户取消选择，使用默认音色")
            return None, "女声1 (默认)"


def get_all_voices_choice():
    """
    显示所有可用语音供用户选择
    """
    try:
        print("\n🔄 获取所有可用语音...")
        tts_gen = TTSGenerator()
        voices = tts_gen.get_available_voices()
        if not voices:
            print("❌ 无法获取语音列表，使用默认语音")
            return None, "George (默认)"
        print(f"\n🎤 所有可用语音 (共 {len(voices)} 个):")
        print("=" * 60)
        for i, voice in enumerate(voices, 1):
            print(f"{i:2d}. {voice['name']} ({voice['voice_id']})")
            if voice["description"]:
                print(f"     描述: {voice['description']}")
            print("-" * 40)
        print(f"{len(voices) + 1}. 返回推荐列表")
        print(f"{len(voices) + 2}. 使用默认语音")
        while True:
            try:
                choice = input(f"\n请选择语音 (1-{len(voices) + 2}): ").strip()
                if not choice:
                    continue
                choice_num = int(choice)
                if 1 <= choice_num <= len(voices):
                    selected_voice = voices[choice_num - 1]
                    print(f"\n✅ 您选择了: {selected_voice['name']}")
                    return selected_voice["voice_id"], selected_voice["name"]
                elif choice_num == len(voices) + 1:
                    return get_user_voice_choice()  # 返回推荐列表
                elif choice_num == len(voices) + 2:
                    print(f"\n✅ 使用默认语音: George")
                    return None, "George (默认)"
                else:
                    print(f"⚠️  请输入 1 到 {len(voices) + 2} 之间的数字")
            except ValueError:
                print("⚠️  请输入有效的数字")
            except KeyboardInterrupt:
                print("\n\n⚠️  用户取消选择，使用默认语音")
                return None, "George (默认)"
    except Exception as e:
        print(f"❌ 获取语音列表失败: {e}")
        print("使用默认语音")
        return None, "George (默认)"


def preview_voice_sample(voice_id, voice_name):
    """
    预览语音样本 - 生成base64数据版本
    """
    try:
        print(f"\n🔊 正在生成 {voice_name} 的语音预览...")

        sample_texts = [
            "你好，欢迎观看今天的视频内容！",
            "哈哈哈这也太搞笑了吧！",
            "呜呜呜好感动啊！",
        ]

        tts_manager = TTSBase64Manager()
        preview_results = []

        for i, text in enumerate(sample_texts, 1):
            print(f"  生成样本 {i}/3: {text}")

            import asyncio

            result = tts_manager.generate_speech_base64(text, voice_id)

            if result["success"]:
                preview_results.append(result)
                print(f"    ✅ 生成成功 - 音频长度: {result['audio_length']} 字符")

                # 保存预览数据到JSON文件
                output_file = f"./test_output/voice_preview_{voice_name.replace(' ', '_')}_{i}.json"
                os.makedirs("./test_output", exist_ok=True)
                with open(output_file, "w", encoding="utf-8") as f:
                    json.dump(result, f, ensure_ascii=False, indent=2)
                print(f"    💾 预览数据已保存: {output_file}")
            else:
                print(f"    ❌ 生成失败: {result['error']}")

        print(f"\n💡 语音预览完成，生成了 {len(preview_results)} 个样本")
        print("💡 所有音频数据都以base64格式保存，可直接用于WebSocket传输")

        # 询问是否确认使用此语音
        while True:
            confirm = input(f"\n确认使用 {voice_name} 语音吗？(y/n): ").strip().lower()
            if confirm in ["y", "yes", "是", "确认"]:
                return True
            elif confirm in ["n", "no", "否", "取消"]:
                return False
            else:
                print("请输入 y/n")

    except Exception as e:
        print(f"❌ 语音预览失败: {e}")
        return True  # 预览失败时默认确认使用


def test_simple_base64_tts():
    """测试简单的Base64 TTS功能"""
    print("\n📍 测试1: 简单Base64 TTS功能")
    print("=" * 50)

    tts_manager = TTSBase64Manager()

    # 选择语音
    voice_id, voice_name = get_user_voice_choice()

    # 测试文本
    test_texts = [
        "你好，我是你的AI语音陪伴！",
        "哈哈哈这个视频太搞笑了！",
        "呜呜呜好感动，我都要哭了！",
        "诶？这是什么情况？我有点懵！",
    ]

    print(f"\n🧪 使用 {voice_name} 测试 {len(test_texts)} 条文本...")

    # 批量处理
    result = tts_manager.process_text_batch(test_texts, voice_id)

    if result["success"]:
        print(f"\n🎉 批量处理完成！")
        print(
            f"📊 成功生成: {result['successful_generations']}/{result['total_texts']}"
        )

        # 保存结果到文件
        output_file = f"./test_output/tts_base64_result_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
        os.makedirs("./test_output", exist_ok=True)

        with open(output_file, "w", encoding="utf-8") as f:
            json.dump(result, f, ensure_ascii=False, indent=2)

        print(f"💾 Base64结果已保存到: {output_file}")

        # 显示部分结果信息
        print(f"\n📋 结果预览:")
        for i, res in enumerate(result["results"][:2], 1):
            if res["success"]:
                print(f"  {i}. 文本: {res['text'][:30]}...")
                print(f"     音频长度: {res['audio_length']} 字符")
                print(f"     音频大小: {res['audio_size_bytes']} bytes")
                print(f"     base64前缀: {res['base64_audio'][:50]}...")
            else:
                print(f"  {i}. 失败: {res['error']}")

        print(f"\n💡 所有音频数据都以base64格式存储，可直接用于WebSocket传输")
    else:
        print(f"❌ 批量处理失败")


def test_video_reaction_base64():
    """模拟视频反应的Base64 TTS处理"""
    print("\n📍 测试2: 模拟视频反应Base64 TTS")
    print("=" * 50)

    tts_manager = TTSBase64Manager()

    # 选择语音
    voice_id, voice_name = get_user_voice_choice()

    # 模拟视频分析结果
    mock_video_reactions = [
        {"timestamp": 0.0, "text": "哇！这个视频开始了！我好期待啊！"},
        {"timestamp": 15.2, "text": "哈哈哈哈！这也太搞笑了吧！"},
        {"timestamp": 32.5, "text": "诶？这个转折我没想到！"},
        {"timestamp": 48.7, "text": "呜呜呜，好感动啊，我都要哭了！"},
        {"timestamp": 65.1, "text": "等等等等，这是什么神操作？"},
        {"timestamp": 80.3, "text": "太厉害了！我学到了新东西！"},
        {"timestamp": 95.8, "text": "这个视频真的很棒，谢谢分享！"},
    ]

    print(f"\n🎬 模拟处理 {len(mock_video_reactions)} 个视频反应...")
    print(f"🎵 使用语音: {voice_name}")

    # 提取文本
    texts = [reaction["text"] for reaction in mock_video_reactions]

    # 批量生成
    result = tts_manager.process_text_batch(texts, voice_id)

    if result["success"]:
        # 合并时间戳信息
        enhanced_results = []
        for i, (reaction, tts_result) in enumerate(
            zip(mock_video_reactions, result["results"])
        ):
            enhanced_result = {
                **tts_result,
                "timestamp": reaction["timestamp"],
                "reaction_index": i,
            }
            enhanced_results.append(enhanced_result)

        # 保存完整结果
        output_data = {
            "video_info": {
                "total_reactions": len(mock_video_reactions),
                "voice_used": voice_name,
                "voice_id": voice_id or tts_manager.tts_generator.default_voice_id,
                "processing_time": datetime.now().isoformat(),
            },
            "tts_summary": {
                "total_texts": result["total_texts"],
                "successful_generations": result["successful_generations"],
                "success_rate": f"{(result['successful_generations'] / result['total_texts'] * 100):.1f}%",
            },
            "reactions": enhanced_results,
        }

        output_file = f"./test_output/video_reaction_base64_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
        os.makedirs("./test_output", exist_ok=True)

        with open(output_file, "w", encoding="utf-8") as f:
            json.dump(output_data, f, ensure_ascii=False, indent=2)

        print(f"\n🎉 视频反应Base64 TTS处理完成！")
        print(f"📊 成功率: {output_data['tts_summary']['success_rate']}")
        print(f"💾 完整结果已保存到: {output_file}")

        # 显示时间轴预览
        print(f"\n⏰ 反应时间轴预览:")
        for reaction in enhanced_results[:3]:
            if reaction["success"]:
                print(f"  {reaction['timestamp']:>6.1f}s: {reaction['text'][:40]}...")
                print(
                    f"           音频: {reaction['audio_size_bytes']} bytes -> {reaction['audio_length']} chars (base64)"
                )
            else:
                print(f"  {reaction['timestamp']:>6.1f}s: ❌ {reaction['error']}")

        print(f"\n💡 所有音频数据都以base64格式存储，适合WebSocket实时传输")
    else:
        print(f"❌ 视频反应Base64 TTS处理失败")


def test_video_to_speech_base64():
    """测试视频分析 + Base64 TTS语音生成完整流程"""

    # 配置参数
    video_path = r"C:\Users\86182\Desktop\31182686022-1-192.mp4"  # 你的视频路径
    output_dir = "./batch_test_output_base64"  # 输出目录

    print("🎬🎵 开始视频分析 + Base64 TTS语音生成测试")
    print("=" * 60)

    # 用户选择语音
    print("\n🎤 首先选择您喜欢的语音音色...")
    voice_id, voice_name = get_user_voice_choice()

    # 是否要预览语音样本
    if voice_id:  # 如果选择了非默认语音
        preview_choice = (
            input(f"\n是否要预览 {voice_name} 的语音样本？(y/n): ").strip().lower()
        )
        if preview_choice in ["y", "yes", "是"]:
            if not preview_voice_sample(voice_id, voice_name):
                print("重新选择语音...")
                voice_id, voice_name = get_user_voice_choice()

    print(f"\n🎵 将使用语音: {voice_name}")
    if voice_id:
        print(f"🆔 语音ID: {voice_id}")

    # 系统提示词（使用你之前优化过的）
    system_prompt = """# SYSTEM PROMPT

You are reacting to a video with your human friend (the user). Your task is to generate a "Reaction Script" in JSON format that details the sequence of actions you will take while watching a video. Your reaction should be natural, engaging, and feel like a real person watching and commenting.

## 角色设定
下面你将扮演的角色具有以下特征：

**核心人设：**

**语言风格：**

**反应特点：**

## 输出规则
**RULES:**
1. You MUST output a valid JSON object that strictly adheres to the provided JSON Schema.
2. Your output MUST be a single JSON object, starting with { and ending with }.
3. Use the comment field in each action object to explain your thought process.
4. Make your speech (text in SPEAK actions) lively and in character as defined.
5. The final action MUST be "END_REACTION" or "ASK_USER".

**JSON SCHEMA:** [省略具体schema以节省空间]
"""

    user_prompt = "请分析这个视频中的主要动作和情感变化，为桌宠生成相应的反应动作。"

    try:
        print(f"\n📍 开始处理视频，使用语音: {voice_name}")

        # 1. 创建TTS管理器
        tts_manager = TTSBase64Manager()

        # 2. 使用现有的视频分析功能
        from ai_watch_buddy.prompts.video_analyzer import invoke_gemini_vids

        print("🎬 开始视频分析...")
        analysis_result = invoke_gemini_vids(
            video_path=video_path, system_prompt=system_prompt, user_prompt=user_prompt
        )

        if not analysis_result.get("success"):
            print(f"❌ 视频分析失败: {analysis_result.get('error', '未知错误')}")
            return

        # 3. 提取所有SPEAK动作的文本
        action_list = analysis_result["action_list"]
        speak_actions = [
            action for action in action_list if action.get("action_type") == "SPEAK"
        ]

        if not speak_actions:
            print("❌ 未找到任何SPEAK动作")
            return

        print(f"🗣️  找到 {len(speak_actions)} 条语音动作")

        # 4. 提取文本并批量生成base64音频
        texts = [
            action.get("text", "").strip()
            for action in speak_actions
            if action.get("text", "").strip()
        ]

        import asyncio

        batch_result = tts_manager.process_text_batch(texts, voice_id)

        if batch_result["success"]:
            # 5. 合并时间戳和其他信息
            enhanced_results = []
            for i, (action, tts_result) in enumerate(
                zip(speak_actions, batch_result["results"])
            ):
                enhanced_result = {
                    **tts_result,
                    "action_id": action.get("id"),
                    "timestamp": action.get("trigger_timestamp", 0),
                    "action_index": i,
                    "comment": action.get("comment", ""),
                }
                enhanced_results.append(enhanced_result)

            # 6. 创建完整的输出数据
            complete_result = {
                "video_info": {
                    "video_path": video_path,
                    "total_actions": len(action_list),
                    "speak_actions": len(speak_actions),
                    "voice_used": voice_name,
                    "voice_id": voice_id or tts_manager.tts_generator.default_voice_id,
                    "processing_time": datetime.now().isoformat(),
                },
                "tts_summary": {
                    "total_texts": batch_result["total_texts"],
                    "successful_generations": batch_result["successful_generations"],
                    "success_rate": f"{(batch_result['successful_generations'] / batch_result['total_texts'] * 100):.1f}%",
                },
                "audio_data": enhanced_results,
                "original_actions": action_list,  # 保留原始动作列表
            }

            # 7. 保存结果
            os.makedirs(output_dir, exist_ok=True)
            output_file = f"{output_dir}/video_tts_base64_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"

            with open(output_file, "w", encoding="utf-8") as f:
                json.dump(complete_result, f, ensure_ascii=False, indent=2)

            print(f"\n✅ 视频分析 + Base64 TTS处理成功！")
            print(f"🎵 使用语音: {voice_name}")
            print(f"📁 输出文件: {output_file}")
            print(
                f"🎵 成功生成: {batch_result['successful_generations']}/{batch_result['total_texts']} 个base64音频"
            )
            print(f"� 成功率: {complete_result['tts_summary']['success_rate']}")

            # 显示部分生成的结果
            print(f"\n📄 生成的音频数据示例:")
            for i, audio_data in enumerate(enhanced_results[:3]):
                if audio_data["success"]:
                    print(f"  {i + 1}. 时间: {audio_data['timestamp']}s")
                    print(f"     文本: {audio_data['text'][:40]}...")
                    print(
                        f"     音频: {audio_data['audio_size_bytes']} bytes -> {audio_data['audio_length']} chars (base64)"
                    )
                else:
                    print(f"  {i + 1}. ❌ 失败: {audio_data['error']}")

            if len(enhanced_results) > 3:
                print(f"     ... 还有 {len(enhanced_results) - 3} 个音频数据")

            print(f"\n💡 所有音频数据都以base64格式存储，可直接用于WebSocket传输")

        else:
            print(f"❌ 批量TTS处理失败")

    except Exception as e:
        print(f"❌ 处理异常: {e}")
        import traceback

        traceback.print_exc()


def test_custom_voice_comparison():
    """测试多种语音对比"""

    print("\n📍 语音对比测试")
    print("=" * 40)

    # 测试文本
    test_text = "哈哈哈这也太搞笑了吧！我笑死了！"

    # 多个语音进行对比
    voice_options = [
        ("JBFqnCBsd6RMkjVDRZzb", "George"),
        ("EXAVITQu4vr4xnSDxMaL", "Sarah"),
        ("cgSgspJ2msm6clMCkdW9", "Jessica"),
        ("TX3LPaxmHKxFdv7VOQHJ", "Liam"),
    ]

    try:
        tts_gen = TTSGenerator()

        print(f"🧪 用文本 '{test_text}' 测试不同语音:")
        print("-" * 50)

        for voice_id, voice_name in voice_options:
            print(f"\n🎤 测试语音: {voice_name}")

            output_path = f"./test_output/comparison_{voice_name.lower()}.mp3"
            audio = tts_gen.generate_single_audio(
                text=test_text,
                voice_id=voice_id,
                output_path=output_path,
                play_audio=False,
            )

            if audio:
                print(f"✅ 生成成功: {output_path}")
            else:
                print(f"❌ 生成失败")

        print(f"\n💡 对比文件已保存到 ./test_output/ 目录")
        print("可以播放这些文件来对比不同语音的效果")

    except Exception as e:
        print(f"❌ 语音对比测试失败: {e}")
    """测试自定义语音"""

    print("\n📍 方法2: 使用自定义语音设置")

    try:
        # 创建TTS生成器
        tts_gen = TTSGenerator()

        # 显示可用语音（可选）
        print("\n🎤 获取可用语音列表...")
        voices = tts_gen.get_available_voices()
        if voices:
            print(f"找到 {len(voices)} 个可用语音")
            # 显示前5个语音
            for i, voice in enumerate(voices[:5]):
                print(
                    f"{i + 1}. {voice['name']} ({voice['voice_id']}) - {voice['category']}"
                )

        # 测试单条语音生成
        test_texts = [
            "哈哈哈哈这也太搞笑了吧！",
            "呜呜呜我哭死了好感动啊！",
            "诶？这是什么情况？",
        ]

        print(f"\n🧪 测试自定义语音生成...")
        for i, text in enumerate(test_texts, 1):
            audio = tts_gen.generate_single_audio(
                text=text,
                output_path=f"./test_output/test_{i}_{text}.mp3",
                play_audio=False,
            )

            if audio:
                print(f"✅ 测试 {i}/3 成功")
            else:
                print(f"❌ 测试 {i}/3 失败")

    except Exception as e:
        print(f"❌ 自定义语音测试失败: {e}")


def test_batch_processing():
    """测试批量处理"""

    print("\n📍 方法3: 使用完整的批量处理")

    video_path = r"C:\Users\86182\Desktop\31182686022-1-192.mp4"

    # 简化的系统提示词
    simple_system_prompt = """
You are a cute AI character reacting to videos. Generate a JSON array of reactions.
Each reaction should have: action_type, trigger_timestamp, text (for SPEAK actions), comment.
Make your speech natural and engaging. Use SPEAK actions to comment on the video.
End with END_REACTION action.
"""

    user_prompt = "React to this video with enthusiasm and natural comments."

    try:
        tts_gen = TTSGenerator()

        result = tts_gen.process_video_analysis_to_speech(
            video_path=video_path,
            system_prompt=simple_system_prompt,
            user_prompt=user_prompt,
            output_dir="./batch_test_output",
            play_audio=False,  # 设为True可以播放音频
        )

        if result["success"]:
            print(f"🎉 批量处理成功！")
            print(f"📊 统计信息:")
            print(f"  - 总动作数: {result['total_actions']}")
            print(f"  - 成功生成: {result['successful_generations']}")
            print(f"  - 输出目录: {result['output_dir']}")
            print(f"  - 元数据: {result['metadata_path']}")

            # 显示生成的文件列表
            if result["generated_files"]:
                print(f"\n📄 生成的音频文件:")
                for file_info in result["generated_files"][:5]:  # 只显示前5个
                    print(f"  - {file_info['file_path']}")
                    print(f"    文本: {file_info['text'][:50]}...")
                    print(f"    时间: {file_info['timestamp']}s")

                if len(result["generated_files"]) > 5:
                    print(f"  ... 还有 {len(result['generated_files']) - 5} 个文件")
        else:
            print(f"❌ 批量处理失败: {result['error']}")

    except Exception as e:
        print(f"❌ 批量处理异常: {e}")


if __name__ == "__main__":
    print("🎵 TTS集成测试开始 - Base64版本")
    print("=" * 60)

    # 检查必要的环境变量
    required_keys = ["MINIMAX_API_KEY", "GEMINI_API_KEY"]
    missing_keys = [key for key in required_keys if not os.getenv(key)]

    if missing_keys:
        print(f"❌ 缺少必要的环境变量: {', '.join(missing_keys)}")
        print("请确保 .env 文件包含所有必要的API密钥")
        exit(1)

    try:
        # 运行测试
        test_simple_base64_tts()  # 简单Base64 TTS测试
        test_video_reaction_base64()  # 视频反应Base64测试
        test_video_to_speech_base64()  # 完整流程Base64测试

        print("\n🎉 TTS集成测试完成！")
        print("💡 所有音频数据都以base64格式存储，可直接用于WebSocket传输")

    except KeyboardInterrupt:
        print("\n⚠️  用户中断测试")
    except Exception as e:
        print(f"\n❌ 测试过程中发生异常: {e}")
        import traceback

        traceback.print_exc()
</file>

<file path="src/ai_watch_buddy/tts_generator.py">
"""
TTS语音生成器模块
结合Gemini视频分析和ElevenLabs TTS服务
"""

import os
import json
import time
from typing import List, Dict, Any, Optional, Union
from pathlib import Path
from dotenv import load_dotenv

# 导入我们封装好的Gemini分析模块
from ai_watch_buddy.prompts.video_analyzer import invoke_gemini_vids


# 切换为MiniMax TTS
import requests
import base64
import tempfile


# 加载环境变量
load_dotenv()


class TTSGenerator:
    """TTS语音生成器类（MiniMax版）"""

    def __init__(self, api_key: Optional[str] = None, voice_id: Optional[str] = None):
        """
        初始化TTS生成器
        Args:
            api_key: MiniMax API密钥，如不提供则使用环境变量
            voice_id: 默认音色ID，如不提供则用官方默认
        """
        self.api_key = api_key or os.getenv("MINIMAX_API_KEY")
        if not self.api_key:
            raise ValueError("请设置MINIMAX_API_KEY环境变量或传入api_key参数")
        self.default_voice_id = voice_id or "female-zh"  # MiniMax官方中文女声
        print(f"✅ TTS生成器初始化成功 (MiniMax)")
        print(f"🎤 默认音色ID: {self.default_voice_id}")

    @property
    def default_model(self):
        # MiniMax TTS无模型概念，兼容旧接口
        return None

    def get_available_voices(self) -> List[Dict]:
        """获取MiniMax支持的音色列表（静态/可扩展）"""
        # 官方文档：https://www.minimax.chat/docs#/tts
        # 可根据API返回动态获取，这里静态列举常用音色
        return [
            {
                "voice_id": "female-zh",
                "name": "女声1",
                "category": "female",
                "description": "官方中文女声",
                "language": "zh",
            },
            {
                "voice_id": "male-zh",
                "name": "男声1",
                "category": "male",
                "description": "官方中文男声",
                "language": "zh",
            },
            {
                "voice_id": "female-en",
                "name": "女声2",
                "category": "female",
                "description": "英文女声",
                "language": "en",
            },
            {
                "voice_id": "male-en",
                "name": "男声2",
                "category": "male",
                "description": "英文男声",
                "language": "en",
            },
        ]

    def print_available_voices(self):
        """打印可用音色列表"""
        voices = self.get_available_voices()
        if voices:
            print("\n🎤 可用MiniMax音色列表:")
            print("=" * 80)
            for voice in voices:
                print(f"ID: {voice['voice_id']}")
                print(f"名称: {voice['name']}")
                print(f"类别: {voice['category']}")
                print(f"语言: {voice['language']}")
                print(f"描述: {voice['description']}")
                print("-" * 40)
        else:
            print("❌ 无法获取音色列表")

    def generate_single_audio(
        self,
        text: str,
        voice_id: Optional[str] = None,
        model_id: Optional[str] = None,
        output_path: Optional[str] = None,
    ) -> Optional[bytes]:
        """
        生成单条语音（MiniMax TTS）
        Args:
            text: 要转换的文本
            voice_id: 音色ID，如不提供则使用默认
            model_id: 保留参数，无实际作用
            output_path: 输出文件路径，如不提供则不保存
            play_audio: 是否播放音频
        Returns:
            音频数据字节流
        """
        try:
            voice = voice_id or self.default_voice_id
            print(f"🔄 生成语音: {text[:50]}{'...' if len(text) > 50 else ''}")
            print(f"🎤 使用MiniMax音色: {voice}")
            url = "https://api.minimax.com.cn/v1/tts"  # MiniMax TTS正式API地址
            headers = {
                "Authorization": f"Bearer {self.api_key}",
                "Content-Type": "application/json",
            }
            payload = {"text": text, "voice": voice, "format": "mp3"}
            # 跳过全局代理，直连MiniMax TTS
            resp = requests.post(
                url,
                headers=headers,
                json=payload,
                timeout=30,
                proxies={"http": None, "https": None},
            )
            if resp.status_code != 200:
                print(f"❌ MiniMax TTS API错误: {resp.status_code} {resp.text}")
                return None
            result = resp.json()
            audio_b64 = result.get("audio")
            if not audio_b64:
                print(f"❌ MiniMax TTS无音频返回: {result}")
                return None
            audio_bytes = base64.b64decode(audio_b64)
            if output_path:
                output_path = Path(output_path)
                output_path.parent.mkdir(parents=True, exist_ok=True)
                with open(output_path, "wb") as f:
                    f.write(audio_bytes)
                print(f"💾 音频已保存: {output_path}")
            return audio_bytes
        except Exception as e:
            print(f"❌ 语音生成失败: {e}")
            return None

    def process_video_analysis_to_speech(
        self,
        video_path: str,
        system_prompt: str,
        user_prompt: str,
        output_dir: str = "./tts_output",
        voice_id: Optional[str] = None,
        model_id: Optional[str] = None,
        gemini_api_key: Optional[str] = None,
    ) -> Dict[str, Any]:
        """
        处理视频分析结果并生成语音

        Args:
            video_path: 视频文件路径
            system_prompt: 系统提示词
            user_prompt: 用户提示词
            output_dir: 音频输出目录
            voice_id: 语音ID
            model_id: 模型ID
            play_audio: 是否播放音频
            gemini_api_key: Gemini API密钥

        Returns:
            处理结果字典
        """
        try:
            # 1. 使用Gemini分析视频
            print("🎬 开始视频分析...")
            analysis_result = invoke_gemini_vids(
                video_path=video_path,
                system_prompt=system_prompt,
                user_prompt=user_prompt,
                api_key=gemini_api_key,
            )

            if not analysis_result.get("success"):
                return {
                    "success": False,
                    "error": f"视频分析失败: {analysis_result.get('error', '未知错误')}",
                }

            # 2. 提取所有SPEAK动作的文本
            action_list = analysis_result["action_list"]
            speak_actions = [
                action for action in action_list if action.get("action_type") == "SPEAK"
            ]

            if not speak_actions:
                return {"success": False, "error": "未找到任何SPEAK动作"}

            print(f"🗣️  找到 {len(speak_actions)} 条语音动作")

            # 3. 创建输出目录
            output_path = Path(output_dir)
            output_path.mkdir(parents=True, exist_ok=True)

            # 4. 生成语音文件
            generated_files = []
            successful_generations = 0

            for i, action in enumerate(speak_actions, 1):
                text = action.get("text", "").strip()
                if not text:
                    continue
                # 生成文件名
                timestamp = action.get("trigger_timestamp", 0)
                safe_text = (
                    text[:30].replace(" ", "_").replace("/", "_").replace("\\", "_")
                )
                filename = f"speak_{i:03d}_{timestamp}s_{safe_text}.mp3"
                file_path = output_path / filename
                # 生成语音
                audio_data = self.generate_single_audio(
                    text=text,
                    voice_id=voice_id,
                    model_id=model_id,
                    output_path=str(file_path),
                )
                if audio_data:
                    generated_files.append(
                        {
                            "id": action.get("id"),
                            "timestamp": timestamp,
                            "text": text,
                            "file_path": str(file_path),
                            "file_size": len(audio_data),
                        }
                    )
                    successful_generations += 1
                    print(f"✅ [{i}/{len(speak_actions)}] 生成完成")
                else:
                    print(f"❌ [{i}/{len(speak_actions)}] 生成失败")

            # 5. 生成元数据文件
            metadata = {
                "video_path": video_path,
                "total_speak_actions": len(speak_actions),
                "successful_generations": successful_generations,
                "generated_files": generated_files,
                "generation_time": time.strftime("%Y-%m-%d %H:%M:%S"),
                "tts_settings": {"voice_id": voice_id or self.default_voice_id},
            }

            metadata_path = output_path / "audio_metadata.json"
            with open(metadata_path, "w", encoding="utf-8") as f:
                json.dump(metadata, f, ensure_ascii=False, indent=2)

            print(f"\n✅ 语音生成完成！")
            print(f"📁 输出目录: {output_path}")
            print(
                f"🎵 成功生成: {successful_generations}/{len(speak_actions)} 个音频文件"
            )
            print(f"📋 元数据文件: {metadata_path}")

            return {
                "success": True,
                "output_dir": str(output_path),
                "total_actions": len(speak_actions),
                "successful_generations": successful_generations,
                "generated_files": generated_files,
                "metadata_path": str(metadata_path),
            }

        except Exception as e:
            print(f"❌ 处理失败: {e}")
            return {"success": False, "error": str(e)}


def create_tts_generator(
    api_key: Optional[str] = None, voice_id: Optional[str] = None
) -> TTSGenerator:
    """
    创建TTS生成器实例的便捷函数（MiniMax版）
    Args:
        api_key: MiniMax API密钥
        voice_id: 默认音色ID
    Returns:
        TTSGenerator实例
    """
    return TTSGenerator(api_key=api_key, voice_id=voice_id)


# 便捷函数


def quick_video_to_speech(
    video_path: str,
    system_prompt: str,
    user_prompt: str,
    output_dir: str = "./tts_output",
    minimax_api_key: Optional[str] = None,
    gemini_api_key: Optional[str] = None,
    voice_id: Optional[str] = None,
) -> Dict[str, Any]:
    """
    快速将视频分析结果转换为语音的便捷函数（MiniMax版）
    Args:
        video_path: 视频文件路径
        system_prompt: 系统提示词
        user_prompt: 用户提示词
        output_dir: 输出目录
        minimax_api_key: MiniMax API密钥
        gemini_api_key: Gemini API密钥
        voice_id: 音色ID
        play_audio: 是否播放音频
    Returns:
        处理结果字典
    """
    try:
        tts_gen = TTSGenerator(api_key=minimax_api_key, voice_id=voice_id)
        return tts_gen.process_video_analysis_to_speech(
            video_path=video_path,
            system_prompt=system_prompt,
            user_prompt=user_prompt,
            output_dir=output_dir,
            gemini_api_key=gemini_api_key,
        )
    except Exception as e:
        return {"success": False, "error": str(e)}


if __name__ == "__main__":
    # 测试示例
    print("🎵 TTS生成器模块测试")

    # 检查环境变量
    if not os.getenv("ELEVENLABS_API_KEY"):
        print("❌ 请设置ELEVENLABS_API_KEY环境变量")
        exit(1)

    # 创建TTS生成器
    try:
        tts_gen = create_tts_generator()

        # 显示可用语音
        tts_gen.print_available_voices()

        # 测试单条语音生成
        test_text = "你好，这是一个测试语音。Hello, this is a test voice."
        print(f"\n🧪 测试语音生成: {test_text}")

        audio = tts_gen.generate_single_audio(
            text=test_text, output_path="./test_output/test_voice.mp3", play_audio=False
        )

        if audio:
            print("✅ 语音生成测试成功")
        else:
            print("❌ 语音生成测试失败")

    except Exception as e:
        print(f"❌ 测试失败: {e}")
</file>

<file path="src/ai_watch_buddy/actions.py">
import json
from typing import Literal, List
from pydantic import BaseModel, Field, RootModel

Emotions = Literal[
    "neutral",
    "coldness",
    "disgust",
    "sad",
    "worry",
    "confusion",
    "anger",
    "surprise",
    "expectation",
    "joy",
    "excitement",
    "pride",
    "shy",
    "stunned",
    "embarrassed",
    "play_cool",
    "drink_tea",
]


# 這是一個基礎模型，定義了所有 Action 的共性
class BaseAction(BaseModel):
    """
    所有具體反應動作的基礎模型，定義了每個動作都必須包含的通用屬性。
    """

    model_config = {"arbitrary_types_allowed": True}

    # 每個 Action 都應該有一個獨一無二的 ID，方便追蹤和日誌記錄
    id: str = Field(
        ..., description="一個唯一的動作 ID，建議使用 UUID 生成，用於追蹤和調試。"
    )
    # 這個 Action 在影片的哪個時間點被觸發？這是反應的錨點。
    trigger_timestamp: float = Field(
        ...,
        description="此動作在影片中的觸發時間點 (單位: 秒)，代表 AI 在看到這一秒的內容時決定做出反應。",
    )
    # 一個給開發者看的備註，解釋為什麼 AI 會做這個反應。LLM 也會填寫它。
    comment: str = Field(
        ...,
        description="AI 做出此反應的內心想法或理由的簡要文字描述，主要用於調試或分析 AI 的決策過程。",
    )


# --- 開始定義具體的 Action 類型 ---


# 0. 表情 (Emotion) - 這是一個特殊的 Action，用於在說話時表達情感
class ExpressionAction(BaseAction):
    """
    **请你积极切换表情。**
    這個動作用来做表情，AI 可以选择表情來表達情感。当这个 Action 被触发时，AI 的脸上会显示指定的表情。你可以在 idle 看视频的时候使用这个 Action 来让 AI 的表情更生动。
    表情比说话更重要，对表达情感更有用。请你更多地使用这个 Action 来表达情感。你并不用这么啰嗦，你可以用表情表达自己，避免打断视频观看。
    """

    action_type: Literal["EXPRESSION"] = "EXPRESSION"
    emotion_expressions: Emotions = Field(
        "neutral",
        description="可以选择的表情，你可以选一个。这个表情会显示在 AI 的脸上。",
    )


# 1. 說話 (Speak)
class SpeakAction(BaseAction):
    """
    讓 AI 角色說出指定的文本。這是最核心的互動方式。
    """

    action_type: Literal["SPEAK"] = "SPEAK"
    text: str | None = Field(..., description="AI 角色要說出的具體內容。")
    audio: str | None = Field(
        None,
        description="由 TTS (Text-to-Speech) 服務生成的音頻數據的標識符或路徑。此欄位由後端系統填充，LLM 無需填寫。",
    )
    # 這個布林值非常關鍵，它決定了是「畫外音」還是「暫停解說」
    pause_video: bool = Field(
        description="決定說話時是否需要暫停影片。True 表示暫停影片進行解說。False 表示在影片繼續播放的同時發表評論（畫外音）。推荐使用 False 以保持影片流暢性。如果需要 pause，可以使用 PauseAction。",
    )


# 2. 暫停 (Pause) - 用於模擬思考、驚訝等無言的反應
class PauseAction(BaseAction):
    """
    讓 AI 進行一次無言的暫停。可以用來模擬思考、驚訝，或在兩個動作之間創造節奏感，讓反應更自然。暂停会停止视频时间的变化直到 duration_seconds 结束，如果需要暂停后说话，请使用 SpeakAction (pause_video=True) 而无需使用 PauseAction。
    """

    action_type: Literal["PAUSE"] = "PAUSE"
    # 暫停多久？這給予了精確的節奏控制
    duration_seconds: float = Field(..., description="需要暫停的持續時間 (單位: 秒)。")


# 3. 影片控制 (Video Control)
class SeekAction(BaseAction):
    """
    控制影片的播放進度，讓 AI 可以跳轉到影片的某個特定時間點，通常是為了回顧或預告某個細節。SeekAction 跳转后会继承视频跳转前的播放状态 (暂停 / 播放)
    """

    action_type: Literal["SEEK"] = "SEEK"
    target_timestamp: float = Field(
        ..., description="要跳轉到的目標影片時間點 (單位: 秒)。"
    )
    # 跳轉後做什麼？這個很重要！
    # 'RESUME_PLAYBACK': 跳轉後繼續播放
    # 'STAY_PAUSED': 跳停在那個畫面，等待下一個指令
    post_seek_behavior: Literal["RESUME_PLAYBACK", "STAY_PAUSED"] = Field(
        "STAY_PAUSED",
        description="指定跳轉到目標時間點後的行為。'STAY_PAUSED' 表示停在該畫面，'RESUME_PLAYBACK' 表示立即開始播放。",
    )


# 4. 重看片段 (Replay Segment) - 這是一個複合動作，但我們將其原子化，方便 LLM 生成
class ReplaySegmentAction(BaseAction):
    """
    讓 AI 重播影片的某一個片段。常用於對精彩、有趣或關鍵的細節進行強調、分析和評論。
    """

    action_type: Literal["REPLAY_SEGMENT"] = "REPLAY_SEGMENT"
    start_timestamp: float = Field(
        ..., description="需要重播片段的開始時間點 (單位: 秒)。"
    )
    end_timestamp: float = Field(
        ..., description="需要重播片段的結束時間點 (單位: 秒)。"
    )
    # 重看完之後的行為，是回到原來的地方，還是停在片段結尾？
    # 'RESUME_FROM_ORIGINAL': 回到觸發此動作的時間點繼續播放
    # 'STAY_PAUSED_AT_END': 停在 end_timestamp 處
    post_replay_behavior: Literal["RESUME_FROM_ORIGINAL", "STAY_PAUSED_AT_END"] = Field(
        "RESUME_FROM_ORIGINAL",
        description="定義了重播結束後的行為。'RESUME_FROM_ORIGINAL' 表示播放頭將跳回到觸發此重播動作的原始時間點並繼續播放，'STAY_PAUSED_AT_END' 表示播放將停在重播片段的結尾處。",
    )


# 5. 結束反應 (End Reaction) - 用於控制反應流程
class EndReaction(BaseAction):
    """
    標記一組反應動作的結束。這是一個流程控制指令，主要有兩個用途：
    1. 當 AI 向用戶提問後 (通常是一個 SpeakAction)，應立刻跟隨一個 EndReaction。這會暫停 AI 的後續動作，將控制權交還給用戶，等待用戶的回應或下一個指令。
    2. 對於長影片，可以將一連串的反應拆分成多個由 EndReaction 分隔的區塊。這能避免一次性生成過多的 Action。如果用戶中途打断，就不會浪費已經生成但未被執行的反應，同時也讓系統能更靈活地處理用戶互動。
    """

    action_type: Literal["END_REACTION"] = "END_REACTION"


# --- 使用 Discriminated Union 組合所有 Action ---

# 這一步是 Pydantic V2 的精華所在
# 我們告訴 Pydantic，所有 Action 的聯集由 'action_type' 這個欄位來區分
# 這使得解析 JSON 數據時可以根據 'action_type' 的值自動匹配到對應的 Action 模型
Action = (
    SpeakAction
    | PauseAction
    | SeekAction
    | ReplaySegmentAction
    | EndReaction
    | ExpressionAction
)


# 最後，我們的 Action Script 就是一個 Action 的列表
# 使用 RootModel 可以讓 Pydantic 直接驗證一個列表的根類型，確保整個腳本的結構正確
class ActionScript(RootModel[list[Action]]):
    """
    定義了 AI 反應腳本的最終結構，它是一個包含多個具體 Action 的有序列表。
    """

    pass


class UserInteractionPayload(BaseModel):
    """
    Defines the structure of the data payload for a user interaction,
    such as 'trigger-conversation'.
    """

    # A list of actions the user just performed.
    user_action_list: List[Action]

    # A list of AI actions that were pending (not yet executed) when the user interrupted.
    pending_action_list: List[Action]


if __name__ == "__main__":
    # 這段代碼會將上面定義的 Pydantic 模型轉換成 JSON Schema 文件
    # 這個 Schema 文件可以被其他應用程式或 LLM 用來理解和生成符合格式的數據
    with open("schema.json", "w", encoding="utf-8") as f:
        # 使用 model_json_schema() 方法生成 JSON Schema
        # ensure_ascii=False 確保中文字符能正確顯示
        # indent=2 讓輸出的 JSON 文件格式化，方便閱讀
        json.dump(ActionScript.model_json_schema(), f, ensure_ascii=False, indent=2)
    print("Schema saved to schema.json")
</file>

<file path="src/ai_watch_buddy/agent/video_analyzer_agent.py">
import os
import time
from typing import AsyncGenerator, List, Optional
import asyncio

from google import genai
from google.genai.types import File, Content, Part, GenerateContentConfig, FileData, GenerateContentResponse

from ..actions import Action
from .text_stream_to_action import str_stream_to_actions
from .video_action_agent_interface import VideoActionAgentInterface
from ..prompts.action_gen_prompt import action_generation_prompt
from ..prompts.character_prompts import cute_prompt
from .mock_text import fake_summary, sample_json

MOCK: bool = False


class VideoAnalyzerAgent(VideoActionAgentInterface):
    """
    A concrete implementation of VideoActionAgentInterface using Google Gemini API.
    """

    def __init__(self, api_key: str | None = None, persona_prompt: str = cute_prompt):
        """
        Initialize the video analyzer agent.

        Args:
            api_key: Gemini API key, defaults to GEMINI_API_KEY environment variable
        """
        if MOCK:
            self._client = genai.Client(api_key="hi")
        else:
            self._client = genai.Client(api_key=api_key or os.getenv("GEMINI_API_KEY"))
            
        self._video_file: Optional[File] = None
        self._video_input: Optional[str] = None  # Will store the path or URL
        self._summary: Optional[str] = None
        self._contents: List[Content] = []
        self._summary_ready: bool = False
        self.persona_prompt = persona_prompt

    @property
    def client(
        self,
    ) -> genai.Client:  # Note: Interface says GenerativeModel but sample uses Client
        """The initialized Gemini client for API communication."""
        return self._client

    @property
    def summary_prompt(self) -> str:
        """The system prompt used for generating the video summary."""
        return self._get_summary_prompt()

    @property
    def action_prompt(self) -> str:
        """The system prompt used for generating actions"""
        return self._get_action_prompt()

    @property
    def video_file(self) -> Optional[File]:
        """The File object returned by the Gemini API after video upload."""
        return self._video_file

    @property
    def summary(self) -> Optional[str]:
        """The text summary of the video, generated on demand."""
        return self._summary

    @property
    def contents(self) -> List[Content]:
        """The conversation history stored as a list of Content objects."""
        return self._contents

    @property
    def summary_ready(self) -> bool:
        """A boolean flag indicating if the video summary has been successfully generated."""
        return self._summary_ready

    @property
    def persona(self) -> str:
        """The persona prompt used for this agent."""
        return self.persona_prompt

    def _get_summary_prompt(self) -> str:
        """Placeholder for summary prompt - to be implemented later."""
        return "You are a video analysis assistant. Please provide a comprehensive summary of the video content. Your summary should be detailed and cover all key aspects of the video. The summary should capture the changes of the video content over time. This summary will later be used to replaced actual video content in the conversation."

    def _get_action_prompt(self) -> str:
        """Placeholder for action prompt - to be implemented later."""
        return action_generation_prompt(character_settings=self.persona_prompt)

    async def get_video_summary(self, video_path_or_url: str) -> None:
        """
        Processes a video from a local path or URL, uploads it if necessary, and generates a summary.
        """
        if MOCK:
            print("MOCK 模式: 使用假的视频摘要")
            self._video_input = video_path_or_url
            self._summary = fake_summary
            self._summary_ready = True
            print("视频摘要生成完成 (MOCK)")
            print(f"摘要内容: {self._summary[:200]}...")
            return

        try:
            self._video_input = video_path_or_url  # Store for later use

            if not self._client:
                raise RuntimeError("Client not initialized (possibly in MOCK mode)")

            is_url = video_path_or_url.startswith(("http://", "https://"))

            if is_url:
                # Handle URL input
                print(f"准备使用 URL 处理视频: {video_path_or_url}")
                self._video_file = None
                file_data = FileData(file_uri=video_path_or_url, mime_type="video/mp4")
                video_part_for_api = Part(file_data=file_data)
            else:
                # Handle local file upload
                print(f"正在上传视频文件: {video_path_or_url}")
                uploaded_file = self._client.files.upload(file=video_path_or_url)

                # Wait for processing to complete
                while (
                    uploaded_file
                    and uploaded_file.state
                    and uploaded_file.state.name == "PROCESSING"
                ):
                    print("[处理中]..", end="", flush=True)
                    time.sleep(5)
                    if uploaded_file.name:
                        uploaded_file = self._client.files.get(name=uploaded_file.name)

                if (
                    uploaded_file
                    and uploaded_file.state
                    and uploaded_file.state.name == "FAILED"
                ):
                    state_name = (
                        uploaded_file.state.name if uploaded_file.state else "UNKNOWN"
                    )
                    raise ValueError(f"Video upload failed: {state_name}")

                self._video_file = uploaded_file
                video_part_for_api = self._video_file

            # Generate summary
            contents = [
                video_part_for_api,
                Content(
                    role="user",
                    parts=[Part.from_text(text="请为这个视频生成一个详细的内容摘要。")],
                ),
            ]

            config = GenerateContentConfig(
                system_instruction=[Part.from_text(text=self.summary_prompt)]
            )

            response = self._client.models.generate_content(
                model="gemini-2.5-flash", contents=contents, config=config
            )

            self._summary = response.text
            self._summary_ready = True
            print("视频摘要生成完成")
            print(f"摘要内容: {self._summary}")

        except Exception as e:
            print(f"生成视频摘要时出错: {e}")
            raise

    def add_content(self, role: str, text: str) -> None:
        """
        Adds a new piece of text content to the conversation history.
        """
        if role not in ["user", "model"]:
            raise ValueError("Role must be 'user' or 'model'")

        content = Content(role=role, parts=[Part.from_text(text=text)])
        self._contents.append(content)

    async def produce_action_stream(self, mode: str) -> AsyncGenerator[Action, None]:
        """
        Generates a stream of structured actions from the model.

        Args:
            mode: The context mode for generation, either 'video' or 'summary'.

        Yields:
            Action: A structured action from the model's streamed response.

        Raises:
            RuntimeError: If `mode` is 'summary' and the `summary_ready` flag is False.
        """
        if mode not in ["video", "summary"]:
            raise ValueError("Mode must be 'video' or 'summary'")

        if mode == "video" and not self._video_input:
            raise RuntimeError(
                "Video source is not ready. Call get_video_summary() first."
            )

        if mode == "summary" and not self._summary_ready:
            raise RuntimeError("Summary is not ready. Call get_video_summary() first.")

        # Build contents based on mode
        contents = []

        if mode == "video":
            if not self._video_input:
                raise RuntimeError("Video input not set.")

            is_url = self._video_input.startswith(("http://", "https://"))
            if is_url:
                file_data = FileData(file_uri=self._video_input, mime_type="video/mp4")
                video_part = Part(file_data=file_data)
            else:
                if not self._video_file:
                    raise RuntimeError(
                        "Local video file was not properly processed and stored."
                    )
                video_part = self._video_file

            # For video mode, use original video file or URL
            contents.extend(
                [
                    video_part,
                    Content(
                        role="user",
                        parts=[
                            Part.from_text(
                                text="请为这个视频生成详细的动作脚本，按照指定的JSON格式输出。"
                            )
                        ],
                    ),
                ]
            )
            # Use action prompt for system instruction
            system_prompt = self.action_prompt
        elif mode == "summary" and self._summary:
            # For summary mode, use text summary
            contents.append(
                Content(
                    role="user",
                    parts=[
                        Part.from_text(
                            text=f"基于以下视频摘要，生成 action:\n{self._summary}"
                        )
                    ],
                )
            )
            # Add conversation history
            contents.extend(self._contents)
            # Use action prompt for system instruction
            system_prompt = self.action_prompt
        else:
            # This case should not be reached due to initial checks, but as a fallback:
            system_prompt = self.action_prompt

        config = GenerateContentConfig(
            system_instruction=[Part.from_text(text=system_prompt)],
        )

        try:
            if MOCK:
                # Mock 模式：创建假的流来模拟 Gemini API 响应
                print("MOCK 模式: 使用假的 action stream")
                
                # 创建一个模拟的流生成器
                def create_mock_stream():
                    # 将 sample_json 按字符分块发送，模拟流式响应
                    chunk_size = 50  # 每次发送50个字符
                    text = sample_json
                    
                    for i in range(0, len(text), chunk_size):
                        chunk = text[i:i + chunk_size]
                        # 创建模拟的 GenerateContentResponse 对象
                        # 使用一个简单的类来模拟响应结构
                        mock_response = type('MockGenerateContentResponse', (), {
                            'text': chunk,
                            'candidates': None,
                            'usage_metadata': None
                        })()
                        yield mock_response
                
                mock_stream = create_mock_stream()
                
                # 使用类型转换来匹配预期类型，因为我们的 mock 对象实现了所需的接口
                from typing import cast, Iterator
                typed_mock_stream = cast(Iterator[GenerateContentResponse], mock_stream)
                
                # 使用模拟流来生成 actions
                for action in str_stream_to_actions(typed_mock_stream):
                    yield action
                return

            #!+=======================
            if not self._client:
                raise RuntimeError("Client not initialized (possibly in MOCK mode)")
                
            llm_stream = self._client.models.generate_content_stream(
                model="gemini-2.5-flash",
                contents=contents,
                config=config,
            )

            # Use the text_stream_to_action function to parse actions
            for action in str_stream_to_actions(llm_stream):
                yield action

        except Exception as e:
            print(f"生成内容时出错: {e}")
            # For error cases, we can't yield anything since the interface expects Action objects
            raise


if __name__ == "__main__":
    import asyncio

    async def test_video_analyzer():
        """Test both summary and video action generation modes"""
        agent = VideoAnalyzerAgent(
            api_key=os.getenv("GEMINI_API_KEY"), persona_prompt=cute_prompt
        )

        # Test with a local video file
        test_video = "/Users/tim/LocalData/coding/2025/Projects/AdventureX/2-AI-WatchBuddy/ai_watch_buddy/video_cache/【官方 MV】Never Gonna Give You Up - Rick Astley.mp4"

        print("测试视频分析Agent...")

        try:
            # Generate summary first
            await agent.get_video_summary(test_video)
            print(f"摘要生成完成: {agent.summary_ready}")
            print(f"摘要内容: {agent.summary[:200]}..." if agent.summary else "无摘要")

            print("\n=== 测试 Summary 模式 ===")
            # Add user message
            agent.add_content("user", "请分析这个视频的主要内容")

            # Generate response using summary mode
            print("生成基于摘要的动作:")
            action_count = 0
            async for action in agent.produce_action_stream("summary"):
                action_count += 1
                print(
                    f"[Action {action_count}]: {action.action_type} - {action.comment}"
                )
                if action.action_type == "SPEAK" and hasattr(action, "text") and action.text:
                    print(
                        f"  Text: {action.text[:100]}{'...' if len(action.text) > 100 else ''}"
                    )
                elif action.action_type == "PAUSE":
                    print(f"  Duration: {action.duration_seconds}s")
                elif action.action_type == "SEEK":
                    print(
                        f"  Target: {action.target_timestamp}s, Behavior: {action.post_seek_behavior}"
                    )
                elif action.action_type == "REPLAY_SEGMENT":
                    print(
                        f"  Replay: {action.start_timestamp}s - {action.end_timestamp}s"
                    )

            print(f"\n基于摘要生成了 {action_count} 个动作")

            print("\n=== 测试 Video 模式 ===")
            # Test video mode
            print("生成基于视频的动作:")
            action_count = 0
            async for action in agent.produce_action_stream("video"):
                action_count += 1
                print(
                    f"[Action {action_count}]: {action.action_type} - {action.comment}"
                )
                if action.action_type == "SPEAK" and hasattr(action, "text") and action.text:
                    print(
                        f"  Text: {action.text[:100]}{'...' if len(action.text) > 100 else ''}"
                    )
                elif action.action_type == "PAUSE":
                    print(f"  Duration: {action.duration_seconds}s")
                elif action.action_type == "SEEK":
                    print(
                        f"  Target: {action.target_timestamp}s, Behavior: {action.post_seek_behavior}"
                    )
                elif action.action_type == "REPLAY_SEGMENT":
                    print(
                        f"  Replay: {action.start_timestamp}s - {action.end_timestamp}s"
                    )

                # Limit output for demo
                if action_count >= 5:
                    print("(限制输出，只显示前5个动作)")
                    break

            print(f"\n基于视频生成了 {action_count} 个动作")

        except Exception as e:
            print(f"测试失败: {e}")
            import traceback

            traceback.print_exc()

    # Run the test
    asyncio.run(test_video_analyzer())
</file>

<file path="src/ai_watch_buddy/session.py">
import asyncio
from typing import Literal, Optional, Union
from .agent.video_action_agent_interface import VideoActionAgentInterface
from .actions import Action


class SessionState:
    """Holds the state for a single watching session."""

    def __init__(
        self,
        session_id: str,
        character_id: str,
        video_url: str,
        character_prompt: str | None = None,
    ):
        self.session_id = session_id
        self.character_id = character_id
        self.video_url = video_url
        self.character_prompt = character_prompt
        self.local_video_path: str | None = None
        self.status: Literal[
            "created",
            "downloading_video",
            "video_ready",
            "generating_actions",
            "session_ready",
            "error",
        ] = "created"
        self.processing_error: str | None = None

        # 关键改动：为每个 session 实例创建一个 asyncio.Queue
        # 这个队列将作为生产者（pipeline）和消费者（websocket）之间的桥梁
        self.action_queue: asyncio.Queue[Optional[Union[Action, dict]]] = (
            asyncio.Queue()
        )
        self.agent: Optional[VideoActionAgentInterface] = None
        self.action_generation_task: Optional[asyncio.Task] = None


# A simple in-memory "database" for sessions
# This dictionary is now the single source of truth for all session states.
session_storage: dict[str, SessionState] = {}
</file>

<file path="src/ai_watch_buddy/server.py">
import uuid
import asyncio
import os
from loguru import logger

from fastapi import (
    FastAPI,
    WebSocket,
    BackgroundTasks,
    status,
    WebSocketDisconnect,
)
from fastapi.middleware.cors import CORSMiddleware
from starlette.staticfiles import StaticFiles as StarletteStaticFiles
from pydantic import BaseModel, ValidationError

from .session import SessionState, session_storage
from .pipeline import (
    initial_pipeline,
    generate_and_queue_actions,
    run_conversation_pipeline,
)
from .actions import Action, UserInteractionPayload, SpeakAction
from .connection_manager import manager
from .asr.fish_audio_asr import FishAudioASR

app = FastAPI()

# Add CORS middleware
app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

# Initialize ASR service
asr_service = None
try:
    if os.getenv("FISH_AUDIO_API_KEY"):
        asr_service = FishAudioASR()
        logger.info("Fish Audio ASR service initialized successfully")
    else:
        logger.warning("FISH_AUDIO_API_KEY not set. ASR functionality will be disabled.")
except Exception as e:
    logger.error(f"Failed to initialize ASR service: {e}")
    asr_service = None


# --- Data Models for API ---
class SessionCreateRequest(BaseModel):
    video_url: str
    start_time: float = 0.0
    end_time: float | None = None
    text: str | None = None
    character_id: str
    user_id: str | None = None


class SessionCreateResponse(BaseModel):
    session_id: str


class ErrorResponse(BaseModel):
    error: str
    message: str


# --- Connection Management ---
# The ConnectionManager is now in its own file (connection_manager.py)
# to prevent circular dependencies. The `manager` instance is imported from there.


async def process_user_actions_asr(user_action_list: list[Action]) -> list[Action]:
    """
    Process user actions and convert audio to text for SPEAK actions.
    
    Args:
        user_action_list: List of user actions to process
        
    Returns:
        Processed user actions with audio converted to text
    """
    if not asr_service:
        logger.warning("ASR service not available. Audio will not be transcribed.")
        return user_action_list
    
    processed_actions = []
    
    for action in user_action_list:
        if isinstance(action, SpeakAction) and action.audio:
            logger.info(f"Processing SPEAK action with audio for ASR conversion")
            
            # If action has audio, transcribe it to text
            try:
                transcribed_text = asr_service.transcribe_audio_sync(
                    audio_base64=action.audio,
                    language=None  # Auto-detect language
                )
                
                if transcribed_text:
                    # Update action with transcribed text and remove audio
                    action.text = transcribed_text
                    action.audio = None
                    logger.info(f"ASR successful: '{transcribed_text}'")
                else:
                    logger.warning(f"ASR failed for action {action.id}, keeping original audio")
                    
            except Exception as e:
                logger.error(f"ASR processing failed for action {action.id}: {e}", exc_info=True)
                # Keep original action if ASR fails
        
        processed_actions.append(action)
    
    return processed_actions


class CORSStaticFiles(StarletteStaticFiles):
    """
    Static files handler that adds CORS headers to all responses.
    Needed because Starlette StaticFiles might bypass standard middleware.
    """

    async def get_response(self, path: str, scope):
        response = await super().get_response(path, scope)

        # Add CORS headers to all responses
        response.headers["Access-Control-Allow-Origin"] = "*"
        response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS"
        response.headers["Access-Control-Allow-Headers"] = "*"

        if path.endswith(".js"):
            response.headers["Content-Type"] = "application/javascript"

        return response


app.mount(
    "/live2d-models",
    CORSStaticFiles(directory="live2d-models"),
    name="live2d-models",
)


# --- API Endpoint ---
@app.post(
    "/api/v1/sessions",
    status_code=status.HTTP_202_ACCEPTED,
    response_model=SessionCreateResponse,
)
async def create_session(
    request: SessionCreateRequest, background_tasks: BackgroundTasks
):
    """
    Creates a new watching session, starts background processing,
    and returns a session_id.
    """
    session_id = f"ses_{uuid.uuid4().hex[:16]}"

    # Create the session state object and store it
    session = SessionState(
        session_id=session_id,
        character_id=request.character_id,
        video_url=request.video_url,
        character_prompt=request.text,
    )
    session_storage[session_id] = session

    # Start the processing pipeline in the background
    background_tasks.add_task(initial_pipeline, session_id=session_id)

    logger.info(f"Accepted session {session_id} for video {request.video_url}")
    return SessionCreateResponse(session_id=session_id)


async def websocket_sender(websocket: WebSocket, session: SessionState):
    """Consumer coroutine: Gets actions from the queue and sends them to the client."""
    # ✅ NEW: Simplified and more accurate waiting logic.
    # It now waits for the single source of truth: the session status.
    while session.status not in ["session_ready", "error"]:
        await asyncio.sleep(0.1)

    if session.status == "error":
        await websocket.send_json(
            {
                "type": "processing_error",
                "error_code": "INITIAL_PIPELINE_FAILED",
                "message": session.processing_error,
            }
        )
        return

    # This message is now sent ONLY after the summary is ready AND the first batch of actions is in the queue.
    await websocket.send_json({"type": "session_ready"})
    logger.info(f"[{session.session_id}] Sent 'session_ready' to client.")

    while True:
        item = await session.action_queue.get()
        if item is None:
            logger.info(f"[{session.session_id}] Reached end of an action batch.")
            session.action_queue.task_done()
            continue

        if isinstance(item, Action):
            await websocket.send_json(
                {"type": "ai_action", "action": item.model_dump(mode="json")}
            )
        elif isinstance(item, dict) and item.get("type") == "processing_error":
            await websocket.send_json(item)

        session.action_queue.task_done()


# ... (The rest of the file, including websocket_receiver and websocket_endpoint, remains the same as my previous answer) ...
async def clear_action_queue(queue: asyncio.Queue):
    """Helper to empty an asyncio queue."""
    while not queue.empty():
        try:
            queue.get_nowait()
        except asyncio.QueueEmpty:
            break


async def websocket_receiver(websocket: WebSocket, session: SessionState):
    """Receiver coroutine: Listens for messages from the client and triggers backend logic."""
    async for message in websocket.iter_json():
        msg_type = message.get("type")
        logger.info(
            f"[{session.session_id}] Received message from client: type={msg_type}"
        )

        # Always interrupt any ongoing task before starting a new one.
        if session.action_generation_task and not session.action_generation_task.done():
            logger.info(
                f"[{session.session_id}] Cancelling previous action generation task."
            )
            session.action_generation_task.cancel()
            await clear_action_queue(
                session.action_queue
            )  # Clear out any partially generated actions

        if msg_type == "interrupt":
            logger.info(
                f"[{session.session_id}] Client sent interrupt. Task cancelled, queue cleared."
            )
            # The logic at the start of the loop already handles this.

        elif msg_type == "trigger-load-next":
            logger.info(
                f"[{session.session_id}] Client triggered lazy-load for next actions."
            )
            
            session.agent.add_content(role="user", text="Continue")
            session.action_generation_task = asyncio.create_task(
                generate_and_queue_actions(
                    session.session_id,
                    mode="video",
                    clear_pending_actions=True,
                )
            )

        elif msg_type == "trigger-conversation":
            try:
                # Log the raw message for debugging
                logger.info(f"[{session.session_id}] 📨 Raw trigger-conversation message received")
                logger.debug(f"[{session.session_id}] 🔍 Raw message data keys: {list(message.keys())}")
                
                # Assuming the payload is in message['data']
                payload = UserInteractionPayload.model_validate(message.get("data", {}))
                logger.info(
                    f"[{session.session_id}] ✅ Client triggered conversation with "
                    f"{len(payload.user_action_list)} user actions and "
                    f"{len(payload.pending_action_list)} pending actions."
                )
                
                # Log user actions for debugging
                for action in payload.user_action_list:
                    if action.action_type == 'SPEAK':
                        logger.info(f"[{session.session_id}] 📨 Received SPEAK action:")
                        if hasattr(action, 'text') and action.text:
                            logger.info(f"[{session.session_id}] 📝 User text: '{action.text}'")
                        if hasattr(action, 'audio') and action.audio:
                            logger.info(f"[{session.session_id}] 🎵 User audio: {len(action.audio)} chars (base64)")
                        if not (hasattr(action, 'text') and action.text) and not (hasattr(action, 'audio') and action.audio):
                            logger.warning(f"[{session.session_id}] ⚠️ SPEAK action has neither text nor audio!")
                    else:
                        logger.info(f"[{session.session_id}] 🎯 User action: {action.action_type}")
                
                # Process audio to text conversion for SPEAK actions
                processed_user_actions = await process_user_actions_asr(payload.user_action_list)
                
                # Log processed actions
                for action in processed_user_actions:
                    if action.action_type == 'SPEAK' and hasattr(action, 'text') and action.text:
                        logger.info(f"[{session.session_id}] 📝 Processed user text: '{action.text}'")
                
                # No need to store task here, run_conversation_pipeline will do it.
                asyncio.create_task(
                    run_conversation_pipeline(
                        session.session_id,
                        user_action_list=processed_user_actions,
                        pending_action_list=payload.pending_action_list,
                    )
                )
            except ValidationError as e:
                logger.error(
                    f"[{session.session_id}] Invalid conversation payload: {e}"
                )
                await websocket.send_json(
                    {
                        "type": "error",
                        "message": "Invalid payload for trigger-conversation.",
                    }
                )


@app.websocket("/ws/{session_id}")
async def websocket_endpoint(websocket: WebSocket, session_id: str):
    """Main WebSocket endpoint that manages the sender and receiver tasks."""
    # This function remains the same. I'm including it for completeness.
    session = session_storage.get(session_id)
    if not session:
        await websocket.close(code=status.WS_1008_POLICY_VIOLATION)
        logger.warning(
            f"WebSocket connection rejected for unknown session: {session_id}"
        )
        return

    await manager.connect(websocket, session_id)
    logger.info(f"[{session_id}] WebSocket connection established.")

    sender_task = asyncio.create_task(websocket_sender(websocket, session))
    receiver_task = asyncio.create_task(websocket_receiver(websocket, session))

    try:
        await asyncio.gather(sender_task, receiver_task)
    except WebSocketDisconnect:
        logger.info(f"[{session_id}] Client disconnected.")
    except Exception as e:
        logger.error(
            f"[{session_id}] An error occurred in the websocket endpoint: {e}",
            exc_info=True,
        )
    finally:
        sender_task.cancel()
        receiver_task.cancel()
        if session.action_generation_task:
            session.action_generation_task.cancel()
        manager.disconnect(session_id)
        logger.info(f"[{session_id}] WebSocket connection closed and cleaned up.")
</file>

<file path="src/ai_watch_buddy/pipeline.py">
import json
import asyncio
from loguru import logger
from typing import Optional
import os
from dotenv import load_dotenv

from .actions import Action, SpeakAction
from .tts.edge_tts import TTSEngine as EdgeTTSEngine
from .tts.fish_audio_tts import FishAudioTTSEngine
from .session import session_storage
from .fetch_video import download_video_async
from .agent.video_analyzer_agent import VideoAnalyzerAgent
from .prompts.character_prompts import cute_prompt

load_dotenv()


def get_interruption_timestamp(user_action_list: list[Action]) -> Optional[float]:
    """Extracts the interruption timestamp from the user action list."""
    if user_action_list:
        # The timestamp of the first user action is the definitive point of interruption.
        return user_action_list[0].trigger_timestamp
    return None


async def run_conversation_pipeline(
    session_id: str, user_action_list: list[Action], pending_action_list: list[Action]
) -> None:
    """
    Handles a user interruption by sending a concise "Situation Report" to the agent.
    The agent's System Prompt contains all the logic for how to handle this report.
    """
    session = session_storage.get(session_id)
    if not session or not session.agent:
        logger.error(
            f"[{session_id}] Cannot run conversation: session or agent not found."
        )
        return

    interruption_timestamp = get_interruption_timestamp(user_action_list)
    if interruption_timestamp is None:
        logger.warning(
            f"[{session_id}] Could not determine interruption timestamp. Defaulting to 0."
        )
        interruption_timestamp = 0.0

    # This context_message is a simple, clean data report.
    # All the complex instructions have been moved to the System Prompt in action_gen.py.
    context_message = f"""
## User Interruption Report

- **Interruption Timestamp:** {interruption_timestamp} (The video is PAUSED at this time)
- **User Actions:**
{json.dumps([action.model_dump() for action in user_action_list], indent=2)}

- **Your Cancelled Actions:**
{json.dumps([action.model_dump() for action in pending_action_list], indent=2)}

Based on this report and your core instructions, generate your new Reaction Script now.
"""

    session.agent.add_content(role="user", text=context_message)

    # The rest of the logic remains the same.
    session.action_generation_task = asyncio.create_task(
        generate_and_queue_actions(
            session_id, mode="summary", clear_pending_actions=True
        )
    )
    logger.info(
        f"[{session_id}] Sent interruption report for timestamp {interruption_timestamp}. New generation task created."
    )


async def generate_and_queue_actions(
    session_id: str,
    mode: str,
    clear_pending_actions: bool = True,
    early_ready: bool = False,
) -> None:
    """
    Generic function to generate actions using the session's agent and put them in the queue.

    Args:
        session_id: The session identifier
        mode: The generation mode (e.g., "video", "summary")
        clear_pending_actions: Whether to clear existing pending actions
        early_ready: Whether to set session ready after first audio generation
    """
    session = session_storage.get(session_id)
    if not session or not session.agent:
        logger.error(
            f"[{session_id}] Cannot generate actions: session or agent not found."
        )
        return

    try:
        if clear_pending_actions:
            while not session.action_queue.empty():
                session.action_queue.get_nowait()
            logger.info(f"[{session_id}] Cleared pending actions from the queue.")

        session.status = "generating_actions"
        logger.info(f"[{session_id}] Starting action generation with mode '{mode}'...")

        actions_generated_count = 0
        first_audio_generated = False

        action_source = session.agent.produce_action_stream(mode=mode)

        async for action in action_source:
            # Handle both Action objects (from mock) and dict (from agent)

            # 这个回来一定是个 Action 对象，所以不用 validate 了
            # if isinstance(action_data, Action):
            #     action = action_data
            # else:
            #     action = Action.model_validate(action_data)

            # Generate audio for SpeakAction
            if isinstance(action, SpeakAction):
                # Initialize Fish Audio TTS - you'll need to provide your API key
                # tts_instance = FishAudioTTSEngine(
                #     api_key=os.getenv("FISH_AUDIO_API_KEY")
                # )
                tts_instance = EdgeTTSEngine()
                audio_base64 = await tts_instance.generate_audio(action.text)
                if audio_base64:
                    action.audio = audio_base64
                    
                    # Set session ready after first audio generation if early_ready is True
                    if early_ready and not first_audio_generated and session.status != "error":
                        first_audio_generated = True
                        session.status = "session_ready"
                        logger.info(
                            f"[{session_id}] ✅ First audio generated successfully. Status set to 'session_ready'."
                        )
                else:
                    logger.warning(
                        f"[{session_id}] Failed to generate audio for action: {action.id}"
                    )

            await session.action_queue.put(action)
            actions_generated_count += 1
            logger.info(
                f"[{session_id}] Queued action: {action.action_type} at {action.trigger_timestamp}s"
            )

        logger.info(
            f"[{session_id}] Action generation stream finished. Total new actions: {actions_generated_count}."
        )

    except asyncio.CancelledError:
        logger.info(f"[{session_id}] Action generation task was cancelled.")
    except Exception as e:
        logger.error(
            f"[{session_id}] Error during action generation: {e}", exc_info=True
        )
        session.status = "error"
        session.processing_error = str(e)
        error_payload = {
            "type": "processing_error",
            "error_code": "ACTION_GENERATION_FAILED",
            "message": str(e),
        }
        await session.action_queue.put(error_payload)
    finally:
        await session.action_queue.put(None)


async def run_initial_generation(session_id: str):
    """
    Runs initial action generation and summary generation in parallel.
    Session ready is set when the first audio is generated (in action generation).

    Args:
        session_id: The session identifier
    """
    session = session_storage.get(session_id)
    if not session or not session.agent:
        logger.error(
            f"[{session_id}] Session or agent not found in run_initial_generation"
        )
        return

    try:
        # Determine the video input for the agent: use local path if available, otherwise use original URL.
        video_input = (
            session.local_video_path if session.local_video_path else session.video_url
        )

        logger.info(
            f"[{session_id}] Starting parallel summary and action generation for: {video_input}"
        )

        # Create both tasks to run in parallel
        summary_task = asyncio.create_task(
            session.agent.get_video_summary(video_path_or_url=video_input)
        )

        action_generation_task = asyncio.create_task(
            generate_and_queue_actions(
                session_id, mode="video", clear_pending_actions=False, early_ready=False
            )
        )

        # Wait for both tasks to complete
        await asyncio.gather(summary_task, action_generation_task)

        logger.info(f"[{session_id}] Both summary and action generation completed.")

        if not session.agent.summary_ready:
            raise RuntimeError(
                "Agent summary was not ready after summary task completion."
            )
            
        session.status = "session_ready"

        # Session ready is already set by generate_and_queue_actions when first audio is ready
        logger.info(
            f"[{session_id}] ✅ Initial pipeline complete. Summary generation finished."
        )

    except Exception as e:
        logger.error(
            f"[{session_id}] Error during initial generation: {e}", exc_info=True
        )
        session.status = "error"
        session.processing_error = str(e)
        error_payload = {
            "type": "processing_error",
            "error_code": "INITIAL_GENERATION_FAILED",
            "message": str(e),
        }
        await session.action_queue.put(error_payload)
        await session.action_queue.put(None)


async def initial_pipeline(session_id: str) -> None:
    """
    The initial background task that runs when a session is created.
    """
    session = session_storage.get(session_id)
    if not session:
        return

    try:
        # Step 1: Check video URL and download if necessary
        video_url = session.video_url
        is_youtube_url = "youtube.com" in video_url or "youtu.be" in video_url

        if is_youtube_url:
            logger.info(
                f"[{session_id}] YouTube URL detected, skipping download: {video_url}"
            )
            session.local_video_path = None
        else:
            session.status = "downloading_video"
            logger.info(
                f"[{session_id}] Non-YouTube URL detected, downloading from: {video_url}"
            )
            local_video_path = str(
                await download_video_async(session.video_url, target_dir="video_cache")
            )
            session.local_video_path = local_video_path
            logger.info(
                f"[{session_id}] Video downloaded and ready at: {local_video_path}"
            )

        session.status = "video_ready"

        # Step 2: Initialize the agent
        persona = session.character_prompt or cute_prompt
        agent = VideoAnalyzerAgent(persona_prompt=persona)
        session.agent = agent
        logger.info(f"[{session_id}] Agent initialized with persona.")

        # Step 3: Start parallel summary and action generation in the background.
        session.action_generation_task = asyncio.create_task(
            run_initial_generation(session_id)
        )
        logger.info(
            f"[{session_id}] Summary and action generation started in the background."
        )

    except Exception as e:
        logger.error(
            f"[{session_id}] Error during initial pipeline setup: {e}", exc_info=True
        )
        session.status = "error"
        session.processing_error = str(e)
        if session:
            error_payload = {
                "type": "processing_error",
                "error_code": "INITIAL_PIPELINE_FAILED",
                "message": str(e),
            }
            await session.action_queue.put(error_payload)
            await session.action_queue.put(None)
</file>

</files>