merge: bring camera realtime fixes into main

Tonnodoubt · Tonnodoubt · commit 90a33f6087c4 · 2026-04-03T11:04:55.000+08:00
diff --git a/app/(tabs)/main.tsx b/app/(tabs)/main.tsx
@@ -314,6 +314,7 @@ const MainUIScreen: React.FC<MainUIScreenProps> = () => {
   const sessionStartedResolverRef = useRef<((value: boolean) => void) | null>(null);
   const sessionTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
   const pendingSessionPromiseRef = useRef<Promise<boolean> | null>(null);
+  const activeSessionModeRef = useRef<'text' | 'audio' | null>(null);
 
   // Agent Backend 管理（传入 openPanel 以支持动态刷新）
   const { agent, onAgentChange, refreshAgentState } = useLive2DAgentBackend({
@@ -392,9 +393,13 @@ const MainUIScreen: React.FC<MainUIScreenProps> = () => {
         console.log('✅ 收到 session_started，input_mode:', inputMode);
         if (inputMode === 'text') {
           setIsTextSessionActive(true);
+          activeSessionModeRef.current = 'text';
         } else if (inputMode === 'audio') {
           // audio session 启动意味着 text session 已被替换，重置状态
           setIsTextSessionActive(false);
+          activeSessionModeRef.current = 'audio';
+        } else {
+          activeSessionModeRef.current = null;
         }
         if (sessionTimeoutRef.current) {
           clearTimeout(sessionTimeoutRef.current);
@@ -412,6 +417,7 @@ const MainUIScreen: React.FC<MainUIScreenProps> = () => {
       if (parsedMsg?.type === 'session_failed') {
         console.log('❌ 收到 session_failed，input_mode:', parsedMsg.input_mode);
         setIsTextSessionActive(false);
+        activeSessionModeRef.current = null;
         if (sessionTimeoutRef.current) {
           clearTimeout(sessionTimeoutRef.current);
           sessionTimeoutRef.current = null;
@@ -428,6 +434,7 @@ const MainUIScreen: React.FC<MainUIScreenProps> = () => {
       if (parsedMsg?.type === 'session_ended_by_server') {
         console.log('⚠️ 收到 session_ended_by_server，input_mode:', parsedMsg.input_mode);
         setIsTextSessionActive(false);
+        activeSessionModeRef.current = null;
         return;
       }
 
@@ -570,6 +577,7 @@ const MainUIScreen: React.FC<MainUIScreenProps> = () => {
         }
         // 连接断开时重置 text session 状态
         setIsTextSessionActive(false);
+        activeSessionModeRef.current = null;
       }
     }
   });
@@ -851,7 +859,39 @@ const MainUIScreen: React.FC<MainUIScreenProps> = () => {
     }
   }, [mainManager]);
 
-  const handleToggleScreen = useCallback(async (next: boolean) => {
+  const ensureRealtimeVisionSession = useCallback(async (): Promise<boolean> => {
+    const getActiveSessionMode = (): 'text' | 'audio' | null => activeSessionModeRef.current;
+
+    if (!audio.isConnected) {
+      return false;
+    }
+
+    if (getActiveSessionMode() === 'audio' || audio.isRecording) {
+      return true;
+    }
+
+    console.log('📤 为实时摄像头准备 audio session');
+    audio.sendMessage({
+      action: 'start_session',
+      input_type: 'audio',
+      audio_format: 'PCM_48000HZ_MONO_16BIT',
+      new_session: false,
+    });
+
+    const start = Date.now();
+    const timeoutMs = 5000;
+    while (Date.now() - start < timeoutMs) {
+      if (getActiveSessionMode() === 'audio') {
+        return true;
+      }
+      await new Promise(resolve => setTimeout(resolve, 100));
+    }
+
+    console.warn('⚠️ 实时摄像头切换 audio session 超时');
+    return false;
+  }, [audio.isConnected, audio.isRecording, audio.sendMessage]);
+
+  const handleToggleCamera = useCallback(async (next: boolean) => {
     if (!next) {
       // 停止摄像头
       cameraStream.stopStreaming();
@@ -864,22 +904,35 @@ const MainUIScreen: React.FC<MainUIScreenProps> = () => {
     if (!hasPermission) return;
 
     // 显示选择对话框
+    const startCameraStream = async (selectedFacing: 'front' | 'back') => {
+      statusToastRef.current?.show('正在准备实时视觉...', 2000);
+      const sessionReady = await ensureRealtimeVisionSession();
+      if (!sessionReady) {
+        statusToastRef.current?.show('实时视觉会话准备失败', 3000);
+        return;
+      }
+
+      cameraStream.startStreaming(selectedFacing);
+      statusToastRef.current?.show(
+        selectedFacing === 'front' ? '前置摄像头已开启' : '后置摄像头已开启',
+        2000
+      );
+    };
+
     Alert.alert(
       '选择摄像头',
       '请选择要使用的摄像头',
       [
         {
           text: '前置摄像头',
           onPress: () => {
-            cameraStream.startStreaming('front');
-            statusToastRef.current?.show('前置摄像头已开启', 2000);
+            void startCameraStream('front');
           },
         },
         {
           text: '后置摄像头',
           onPress: () => {
-            cameraStream.startStreaming('back');
-            statusToastRef.current?.show('后置摄像头已开启', 2000);
+            void startCameraStream('back');
           },
         },
         {
@@ -888,7 +941,7 @@ const MainUIScreen: React.FC<MainUIScreenProps> = () => {
         },
       ]
     );
-  }, [cameraStream.startStreaming, cameraStream.stopStreaming, cameraStream.checkAndRequestPermission]);
+  }, [cameraStream.startStreaming, cameraStream.stopStreaming, cameraStream.checkAndRequestPermission, ensureRealtimeVisionSession]);
 
   const handleGoodbye = useCallback(() => {
     // 如果麦克风正在录音，先停止
@@ -1042,6 +1095,13 @@ const MainUIScreen: React.FC<MainUIScreenProps> = () => {
       return false;
     }
 
+    // 连续摄像头需要 realtime/audio session；切回文本前先暂停它，避免图片堆进文本会话队列。
+    if (cameraStream.isStreaming) {
+      console.log('📹 切换到文本会话，先暂停实时摄像头');
+      cameraStream.stopStreaming();
+      statusToastRef.current?.show('发送文本时已暂停实时摄像头', 2500);
+    }
+
     // 如果当前正在录音（语音模式），先停止录音并等待服务端清理旧 session，
     // 避免 start_session(text) 与正在启动/活跃的 audio session 产生竞态
     if (audio.isRecording) {
@@ -1095,7 +1155,7 @@ const MainUIScreen: React.FC<MainUIScreenProps> = () => {
 
     pendingSessionPromiseRef.current = promise;
     return promise;
-  }, [isTextSessionActive, audio.isConnected, audio.isRecording, audio.sendMessage, audio.toggleRecording]);
+  }, [isTextSessionActive, audio.isConnected, audio.isRecording, audio.sendMessage, audio.toggleRecording, cameraStream.isStreaming, cameraStream.stopStreaming]);
 
   // 图片消息服务
   const imageMessageService = useMemo(() => new ImageMessageService(), []);
@@ -1273,7 +1333,7 @@ const MainUIScreen: React.FC<MainUIScreenProps> = () => {
         - 详见：docs/strategy/cross-platform-components.md
         
         功能包括：
-        - 麦克风/屏幕共享切换
+        - 麦克风/摄像头切换
         - Agent 设置面板
         - Settings 面板
         - 设置菜单（Live2D设置、API密钥、角色管理等）
@@ -1285,7 +1345,7 @@ const MainUIScreen: React.FC<MainUIScreenProps> = () => {
           right={isMobile ? 12 : 24}
           top={isMobile ? screenHeight * 0.05 : 24}
           micEnabled={toolbarMicEnabled}
-          screenEnabled={cameraStream.isStreaming}
+          cameraEnabled={cameraStream.isStreaming}
           goodbyeMode={toolbarGoodbyeMode}
           openPanel={toolbarOpenPanel}
           onOpenPanelChange={setToolbarOpenPanel}
@@ -1294,7 +1354,7 @@ const MainUIScreen: React.FC<MainUIScreenProps> = () => {
           agent={agent}
           onAgentChange={handleToolbarAgentChange}
           onToggleMic={handleToggleMic}
-          onToggleScreen={handleToggleScreen}
+          onToggleCamera={handleToggleCamera}
           onGoodbye={handleGoodbye}
           onReturn={handleReturn}
           onSettingsMenuClick={handleSettingsMenuClick}
diff --git a/assets/icons/README.md b/assets/icons/README.md
@@ -7,7 +7,7 @@
 请从主项目的 `static/icons/` 目录复制以下图标文件到此目录：
 
 - `mic_icon_off.png` - 麦克风图标
-- `screen_icon_off.png` - 屏幕分享图标
+- `screen_icon_off.png` - 摄像头按钮当前复用的历史图标资源
 - `Agent_off.png` - Agent 工具图标
 - `set_off.png` - 设置图标
 - `rest_off.png` - 离开/返回图标
diff --git a/docs/features/realtime-vision-guide.md b/docs/features/realtime-vision-guide.md
@@ -172,7 +172,7 @@ export interface CameraStreamConfig {
 **关键实现**：
 
 - `setCameraRef(ref)` — 绑定 CameraView ref
-- `start()` — 启动 `setInterval` 帧捕获循环，立即捕获第一帧
+- `start()` — 启动帧捕获循环，首帧延迟约 500ms 以提升 CameraView 启动稳定性
 - `stop()` — 清除 interval，状态回到 `idle`
 - `pause()` / `resume()` — 后台暂停/恢复
 - `captureAndSend()` — 内部方法：
@@ -256,7 +256,7 @@ const cameraStream = useCameraStream({
 **(c) 替换 toolbarScreenEnabled 状态 (line 220)**
 
 - 删除 `const [toolbarScreenEnabled, setToolbarScreenEnabled] = useState(false);`
-- 把 toolbar 的 `screenEnabled` prop 改为 `screenEnabled={cameraStream.isStreaming}`
+- 把 toolbar 的 `cameraEnabled` prop 接到 `cameraStream.isStreaming`
 
 **(d) 重写 handleToggleScreen (line 772-775)**
 
@@ -303,7 +303,7 @@ iOS `NSCameraUsageDescription` 也做同样更新。
 | `services/imageCompression.ts` | 直接复用 `compress()` 方法，无需改动 |
 | `services/AudioService.ts` | 通过 hook 层的 `sendMessage` 消费 |
 | `hooks/useCamera.ts` | 单张拍照 hook，保持独立 |
-| `packages/project-neko-components/` | toolbar 已有 `screenEnabled` / `onToggleScreen` props |
+| `packages/project-neko-components/` | toolbar 已统一为 `cameraEnabled` / `onToggleCamera` props |
 | 后端所有文件 | 协议和限流完全兼容 |
 
 #### 4.2.5 实现顺序
@@ -385,7 +385,7 @@ React 渲染 CameraStreamOverlay + CameraView
         ↓
 onCameraReady 回调触发 → setCameraRef + start()
         ↓
-setInterval 启动 (每 1.5s) + 立即捕获第一帧
+定时启动 (每 1.5s) + 首帧延迟约 500ms
         ↓
 takePictureAsync() → ImageCompressionService.compress()
         ↓
diff --git a/hooks/useCameraStream.ts b/hooks/useCameraStream.ts
@@ -71,7 +71,7 @@ export function useCameraStream(
         setError(err.message);
         setStatus('error');
       },
-      frameInterval: 5000, // 5s
+      frameInterval: 1500, // 1.5s，与后端原生视觉节奏对齐
     });
 
     serviceRef.current = service;
diff --git a/packages/project-neko-components/src/Live2DRightToolbar/Live2DRightToolbar.native.tsx b/packages/project-neko-components/src/Live2DRightToolbar/Live2DRightToolbar.native.tsx
@@ -44,7 +44,7 @@ export function Live2DRightToolbar({
   top = 24,
   isMobile,
   micEnabled,
-  screenEnabled,
+  cameraEnabled,
   goodbyeMode,
   openPanel,
   onOpenPanelChange,
@@ -53,7 +53,7 @@ export function Live2DRightToolbar({
   agent,
   onAgentChange,
   onToggleMic,
-  onToggleScreen,
+  onToggleCamera,
   onGoodbye,
   onReturn,
   onSettingsMenuClick,
@@ -68,19 +68,19 @@ export function Live2DRightToolbar({
   // 使用共享的按钮配置（RN 使用本地 require() 资源）
   const buttons = useToolbarButtons<number>({
     micEnabled,
-    screenEnabled,
+    cameraEnabled,
     openPanel,
     goodbyeMode,
     isMobile,
     onToggleMic,
-    onToggleScreen,
+    onToggleCamera,
     onGoodbye,
     togglePanel,
     t,
     // RN: 使用本地打包的图标资源（确保这些文件存在于 assets/icons/）
     icons: {
       mic: require('../../../../assets/icons/mic_icon_off.png'),
-      screen: require('../../../../assets/icons/screen_icon_off.png'),
+      camera: require('../../../../assets/icons/screen_icon_off.png'),
       agent: require('../../../../assets/icons/Agent_off.png'),
       settings: require('../../../../assets/icons/set_off.png'),
       goodbye: require('../../../../assets/icons/rest_off.png'),
diff --git a/packages/project-neko-components/src/Live2DRightToolbar/Live2DRightToolbar.tsx b/packages/project-neko-components/src/Live2DRightToolbar/Live2DRightToolbar.tsx
@@ -25,7 +25,7 @@ export function Live2DRightToolbar({
   top,
   isMobile,
   micEnabled,
-  screenEnabled,
+  cameraEnabled,
   goodbyeMode,
   openPanel,
   onOpenPanelChange,
@@ -34,7 +34,7 @@ export function Live2DRightToolbar({
   agent,
   onAgentChange,
   onToggleMic,
-  onToggleScreen,
+  onToggleCamera,
   onGoodbye,
   onReturn,
   onSettingsMenuClick,
@@ -51,12 +51,12 @@ export function Live2DRightToolbar({
   // 使用共享的按钮配置
   const buttons = useToolbarButtons<string>({
     micEnabled,
-    screenEnabled,
+    cameraEnabled,
     openPanel,
     goodbyeMode,
     isMobile,
     onToggleMic,
-    onToggleScreen,
+    onToggleCamera,
     onGoodbye,
     togglePanel,
     t,
diff --git a/packages/project-neko-components/src/Live2DRightToolbar/hooks.ts b/packages/project-neko-components/src/Live2DRightToolbar/hooks.ts
@@ -87,32 +87,32 @@ export function usePanelToggle(
  */
 export function useToolbarButtons<TIcon = ToolbarIcon>({
   micEnabled,
-  screenEnabled,
+  cameraEnabled,
   openPanel,
   goodbyeMode,
   isMobile,
   onToggleMic,
-  onToggleScreen,
+  onToggleCamera,
   onGoodbye,
   togglePanel,
   t,
   iconBasePath = '/static/icons', // Web 使用
   icons, // RN 使用 require() 资源
 }: {
   micEnabled: boolean;
-  screenEnabled: boolean;
+  cameraEnabled: boolean;
   openPanel: Live2DRightToolbarPanel;
   goodbyeMode: boolean;
   isMobile?: boolean;
   onToggleMic: (next: boolean) => void;
-  onToggleScreen: (next: boolean) => void;
+  onToggleCamera: (next: boolean) => void;
   onGoodbye: () => void;
   togglePanel: (panel: Exclude<Live2DRightToolbarPanel, null>) => void;
   t?: TFunction;
   iconBasePath?: string;
   icons?: {
     mic: TIcon;
-    screen: TIcon;
+    camera: TIcon;
     agent: TIcon;
     settings: TIcon;
     goodbye: TIcon;
@@ -130,12 +130,12 @@ export function useToolbarButtons<TIcon = ToolbarIcon>({
           icon: (icons?.mic ?? `${iconBasePath}/mic_icon_off.png`) as TIcon,
         },
         {
-          id: 'screen' as const,
+          id: 'camera' as const,
           title: tOrDefault(t, 'buttons.cameraShare', '摄像头'),
           hidden: false,
-          active: screenEnabled,
-          onClick: () => onToggleScreen(!screenEnabled),
-          icon: (icons?.screen ?? `${iconBasePath}/screen_icon_off.png`) as TIcon,
+          active: cameraEnabled,
+          onClick: () => onToggleCamera(!cameraEnabled),
+          icon: (icons?.camera ?? `${iconBasePath}/screen_icon_off.png`) as TIcon,
         },
         {
           id: 'agent' as const,
@@ -165,7 +165,7 @@ export function useToolbarButtons<TIcon = ToolbarIcon>({
           hasPanel: false,
         },
       ].filter((b) => !b.hidden),
-    [goodbyeMode, isMobile, micEnabled, onGoodbye, onToggleMic, onToggleScreen, openPanel, screenEnabled, t, togglePanel, iconBasePath, icons]
+    [cameraEnabled, goodbyeMode, isMobile, micEnabled, onGoodbye, onToggleCamera, onToggleMic, openPanel, t, togglePanel, iconBasePath, icons]
   );
 }
 
diff --git a/packages/project-neko-components/src/Live2DRightToolbar/types.ts b/packages/project-neko-components/src/Live2DRightToolbar/types.ts
diff --git a/services/CameraStreamService.ts b/services/CameraStreamService.ts
