fix(ui): preserve playback after audio unlock

Author: NubsCarson

packages/ui/src/components/pages/chat-view-hooks.test.tsx

@@ -0,0 +1,131 @@
+// @vitest-environment jsdom
+
+import { act, cleanup, renderHook } from "@testing-library/react";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { useVoiceChat } from "../../hooks/useVoiceChat";
+import type { VoiceChatState } from "../../voice/voice-chat-types";
+import { useChatVoiceController } from "./chat-view-hooks";
+
+vi.mock("../../api/client", () => ({
+  client: {
+    getConfig: vi.fn(async () => ({})),
+    updateConfig: vi.fn(async () => ({})),
+  },
+}));
+
+vi.mock("../../hooks/useContinuousChat", () => ({
+  DEFAULT_VOICE_CONTINUOUS_MODE: "off",
+  useContinuousChat: vi.fn(() => ({
+    enabled: false,
+    setEnabled: vi.fn(),
+    mode: "off",
+    setMode: vi.fn(),
+  })),
+}));
+
+vi.mock("../../hooks/useDefaultProviderPresets", () => ({
+  useDefaultProviderPresets: vi.fn(() => ({
+    defaults: { asr: "local-inference", tts: "local-inference" },
+  })),
+}));
+
+vi.mock("../../hooks/useDocumentVisibility", () => ({
+  useDocumentVisibility: vi.fn(() => "visible"),
+}));
+
+vi.mock("../../hooks/useTimeout", () => ({
+  useTimeout: vi.fn(() => ({ setTimeout: globalThis.setTimeout })),
+}));
+
+vi.mock("../../hooks/useVoiceChat", () => ({
+  useVoiceChat: vi.fn(),
+}));
+
+vi.mock("../companion/injected", () => ({
+  useCompanionSceneStatus: vi.fn(() => ({ avatarReady: true })),
+}));
+
+const useVoiceChatMock = vi.mocked(useVoiceChat);
+
+function makeVoiceState(
+  overrides: Partial<VoiceChatState> = {},
+): VoiceChatState {
+  return {
+    assistantTtsQuality: "enhanced",
+    captureMode: "idle",
+    interimTranscript: "",
+    isListening: false,
+    isSpeaking: false,
+    mouthOpen: 0,
+    queueAssistantSpeech: vi.fn(),
+    speak: vi.fn(),
+    startListening: vi.fn(async () => {}),
+    stopListening: vi.fn(async () => {}),
+    stopSpeaking: vi.fn(),
+    supported: true,
+    toggleListening: vi.fn(),
+    usingAudioAnalysis: false,
+    voiceUnlockedGeneration: 0,
+    ...overrides,
+  };
+}
+
+const baseOptions = {
+  activeConversationId: "conversation-1",
+  agentVoiceMuted: false,
+  chatFirstTokenReceived: false,
+  chatInput: "",
+  chatSending: false,
+  conversationMessages: [],
+  elizaCloudConnected: false,
+  elizaCloudHasPersistedKey: false,
+  elizaCloudVoiceProxyAvailable: false,
+  handleChatEdit: vi.fn(async () => true),
+  handleChatSend: vi.fn(async () => {}),
+  isComposerLocked: false,
+  isGameModal: false,
+  setState: vi.fn(),
+  uiLanguage: "en",
+};
+
+describe("useChatVoiceController voice playback unlock", () => {
+  let voiceState: VoiceChatState;
+
+  beforeEach(() => {
+    voiceState = makeVoiceState();
+    useVoiceChatMock.mockImplementation(() => voiceState);
+  });
+
+  afterEach(() => {
+    cleanup();
+    vi.clearAllMocks();
+  });
+
+  it("does not cancel speech queued by the same user gesture that unlocks audio", () => {
+    const { rerender } = renderHook(() => useChatVoiceController(baseOptions));
+    const stopSpeaking = vi.mocked(voiceState.stopSpeaking);
+
+    voiceState = makeVoiceState({
+      stopSpeaking,
+      voiceUnlockedGeneration: 1,
+    });
+
+    act(() => {
+      rerender();
+    });
+
+    expect(stopSpeaking).not.toHaveBeenCalled();
+  });
+
+  it("passes message telemetry through manual Play message speech", () => {
+    const { result } = renderHook(() => useChatVoiceController(baseOptions));
+
+    act(() => {
+      result.current.handleSpeakMessage("message-1", "hello from Milady");
+    });
+
+    expect(voiceState.speak).toHaveBeenCalledWith("hello from Milady", {
+      telemetry: { messageId: "message-1" },
+    });
+  });
+});

packages/ui/src/components/pages/chat-view-hooks.tsx

@@ -414,6 +414,13 @@ export function useChatVoiceController(options: {
 
   const handleVoicePlaybackStart = useCallback(
     (event: VoicePlaybackStartEvent) => {
+      if (event.messageId) {
+        rememberCompanionSpeech(
+          activeConversationId,
+          event.messageId,
+          event.text,
+        );
+      }
       ttsDebug("chat:playback-start", {
         provider: event.provider,
         segment: event.segment,
@@ -465,7 +472,7 @@ export function useChatVoiceController(options: {
             : (prev?.assistantStreamToVoiceStartMs ?? null),
       }));
     },
-    [],
+    [activeConversationId],
   );
 
   const cloudVoiceAvailable = useMemo(() => {
@@ -511,8 +518,9 @@ export function useChatVoiceController(options: {
     voiceUnlockedGeneration,
   } = voice;
 
-  // After the user gesture unlocks audio, clear progressive TTS dedupe state so
-  // auto-speak can queue the greeting again (ElevenLabs was likely skipped once).
+  // After the user gesture unlocks audio, clear only the progressive TTS dedupe
+  // state so auto-speak can retry. Do not stop speaking here: this effect runs
+  // from the same click that may have just queued Play Greeting / Play Message.
   const prevVoiceUnlockGenRef = useRef<number | null>(null);
   useLayoutEffect(() => {
     if (prevVoiceUnlockGenRef.current === null) {
@@ -521,8 +529,8 @@ export function useChatVoiceController(options: {
     }
     if (prevVoiceUnlockGenRef.current === voiceUnlockedGeneration) return;
     prevVoiceUnlockGenRef.current = voiceUnlockedGeneration;
-    stopSpeaking();
-  }, [voiceUnlockedGeneration, stopSpeaking]);
+    companionBootstrapAutoSpeakRef.current = null;
+  }, [voiceUnlockedGeneration]);
 
   const beginVoiceCapture = useCallback(
     (mode: Exclude<VoiceCaptureMode, "idle"> = "compose") => {
@@ -557,10 +565,9 @@ export function useChatVoiceController(options: {
     (messageId: string, text: string) => {
       if (!text.trim()) return;
       suppressedAssistantSpeechRef.current = { messageId, text };
-      rememberCompanionSpeech(activeConversationId, messageId, text);
-      speak(text);
+      speak(text, { telemetry: { messageId } });
     },
-    [activeConversationId, speak],
+    [speak],
   );
 
   const handleEditMessage = useCallback(
@@ -743,7 +750,6 @@ export function useChatVoiceController(options: {
       replace: replacePlayback,
       telemetry,
     });
-    rememberCompanionSpeech(activeConversationId, messageId, text);
     suppressedAssistantSpeechRef.current = null;
     companionBootstrapAutoSpeakRef.current = {
       tick,

packages/ui/src/hooks/useVoiceChat.ts

@@ -1831,7 +1831,13 @@ export function useVoiceChat(options: VoiceChatOptions): VoiceChatState {
   // ── Public speak APIs ─────────────────────────────────────────────
 
   const speak = useCallback(
-    (text: string, speakOptions?: { append?: boolean }) => {
+    (
+      text: string,
+      speakOptions?: {
+        append?: boolean;
+        telemetry?: VoiceAssistantSpeechTelemetry;
+      },
+    ) => {
       if (assistantTtsDebounceRef.current != null) {
         clearTimeout(assistantTtsDebounceRef.current);
         assistantTtsDebounceRef.current = null;
@@ -1841,6 +1847,7 @@ export function useVoiceChat(options: VoiceChatOptions): VoiceChatState {
         text,
         append: Boolean(speakOptions?.append),
         segment: "full",
+        telemetry: speakOptions?.telemetry,
       });
     },
     [enqueueSpeech],
@@ -1923,7 +1930,7 @@ export function useVoiceChat(options: VoiceChatOptions): VoiceChatState {
           latestSpeakable: "",
           finalQueued: false,
           replacePlaybackOnFirstClip: queueOptions?.replace !== false,
-          telemetry: queueOptions?.telemetry,
+          telemetry: { messageId, ...queueOptions?.telemetry },
         };
       } else if (queueOptions?.telemetry) {
         current.telemetry = {

packages/ui/src/voice/voice-chat-types.ts

@@ -238,7 +238,10 @@ export interface VoiceChatState {
   /** End voice capture and optionally submit the transcript */
   stopListening: (options?: { submit?: boolean }) => Promise<void>;
   /** Speak text aloud with mouth animation */
-  speak: (text: string, options?: { append?: boolean }) => void;
+  speak: (
+    text: string,
+    options?: { append?: boolean; telemetry?: VoiceAssistantSpeechTelemetry },
+  ) => void;
   /** Progressively speak an assistant message while it streams */
   queueAssistantSpeech: (
     messageId: string,