Updated VAD settings and VAD behavior with mute/unmute in frontend

Author: goergenj

python/voice-live-voicerag-assistant/app/backend/web_handler.py

@@ -27,6 +27,7 @@
     ServerEventResponseFunctionCallArgumentsDone,
     AudioInputTranscriptionOptions,
     AzureSemanticVad,
+    AzureSemanticDetection,
     MessageItem,
     ResponseCreateParams,
 )
@@ -237,6 +238,35 @@ async def _setup_session(self, connection):
         try:
             # Create session configuration
             try:
+                # Determine if end_of_utterance_detection is supported
+                # It's only supported for cascaded pipelines, not gpt-realtime models
+                realtime_models = ["gpt-realtime", "gpt-realtime-mini", "phi4-mm-realtime"]
+                supports_eou_detection = self.model not in realtime_models
+                
+                # Build turn detection configuration
+                if supports_eou_detection:
+                    turn_detection = AzureSemanticVad(
+                        threshold=0.3,
+                        prefix_padding_ms=300,
+                        speech_duration_ms=80,
+                        silence_duration_ms=500,
+                        remove_filler_words=True,
+                        interrupt_response=True,
+                        end_of_utterance_detection=AzureSemanticDetection(
+                            threshold_level="default",
+                            timeout_ms=1000
+                        )
+                    )
+                else:
+                    turn_detection = AzureSemanticVad(
+                        threshold=0.3,
+                        prefix_padding_ms=300,
+                        speech_duration_ms=80,
+                        silence_duration_ms=500,
+                        remove_filler_words=True,
+                        interrupt_response=True,
+                    )
+                
                 session_config = RequestSession(
                     modalities=[Modality.TEXT, Modality.AUDIO],
                     instructions=self.instructions,
@@ -246,11 +276,7 @@ async def _setup_session(self, connection):
                     input_audio_transcription=AudioInputTranscriptionOptions(
                         model=self.transcribe_model
                     ),
-                    turn_detection=AzureSemanticVad(
-                        threshold=0.5,
-                        prefix_padding_ms=300,
-                        silence_duration_ms=200,
-                    ),
+                    turn_detection=turn_detection,
                     tools=self.tools,
                     tool_choice=ToolChoiceLiteral.AUTO,
                     temperature=0.6,

python/voice-live-voicerag-assistant/app/frontend/src/App.tsx

@@ -37,7 +37,8 @@ function App() {
         startSession,
         stopSession,
         sendAudio,
-        interruptAssistant
+        interruptAssistant,
+        setHonorVadInterruption
     } = useVoiceAssistant({
         onWebSocketOpen: () => {
             console.log("WebSocket connection opened to voice assistant");
@@ -64,7 +65,8 @@ function App() {
         },
         onSpeechStarted: () => {
             console.log("User started speaking");
-            stopAudioPlayer();
+            // Note: Removed automatic stopAudioPlayer() to allow pure mute/unmute
+            // Use the Interrupt button to manually stop assistant playback
         },
         onSpeechStopped: () => {
             console.log("User stopped speaking");
@@ -174,9 +176,14 @@ function App() {
 
     const onToggleListening = async () => {
         if (!isRecording && isSessionActive) {
+            // Starting to listen (unmuting) - enable VAD-triggered barge-in
+            setHonorVadInterruption(true);
             await startAudioRecording();
             setIsRecording(true);
         } else if (isRecording) {
+            // Stopping listening (muting) - disable VAD-triggered barge-in first
+            // This prevents the mute action from triggering a stop_playback
+            setHonorVadInterruption(false);
             await stopAudioRecording();
             setIsRecording(false);
         }

python/voice-live-voicerag-assistant/app/frontend/src/hooks/useVoiceAssistant.tsx

@@ -139,7 +139,12 @@ export default function useVoiceAssistant({
 
   const clientIdRef = useRef(clientId || `client-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`);
 
-    const getWebSocketUrl = () => {
+  // Track whether to honor VAD-triggered stop_playback messages
+  // When muting (stopping listening), we ignore stop_playback because muting shouldn't interrupt playback
+  // When unmuting (starting listening), we honor stop_playback so VAD can trigger barge-in
+  const honorVadInterruptionRef = useRef(false);
+
+  const getWebSocketUrl = () => {
     if (serverUrl) return serverUrl;
 
     // Auto-detect based on current page URL
@@ -315,9 +320,15 @@ export default function useVoiceAssistant({
         break;
 
       case 'stop_playback':
-        console.log('🛑 Stopping audio playback due to user interruption');
-        audioPlayer.stop();
-        onAudioPlaybackStop?.();
+        // Only honor VAD-triggered stop_playback when actively listening (unmuted)
+        // This prevents muting the mic from interrupting playback
+        if (honorVadInterruptionRef.current) {
+          console.log('🛑 Stopping audio playback due to user interruption (VAD barge-in)');
+          audioPlayer.stop();
+          onAudioPlaybackStop?.();
+        } else {
+          console.log('ℹ️ Ignoring stop_playback - microphone is muted');
+        }
         break;
 
       case 'user_speech_ended':
@@ -458,6 +469,13 @@ export default function useVoiceAssistant({
   const isConnecting = readyState === 0; // WebSocket.CONNECTING
   const isDisconnected = readyState === 3; // WebSocket.CLOSED
 
+  // Control whether VAD-triggered stop_playback should be honored
+  // Call with true when starting to listen (unmuting), false when stopping (muting)
+  const setHonorVadInterruption = useCallback((honor: boolean) => {
+    honorVadInterruptionRef.current = honor;
+    console.log(`VAD interruption ${honor ? 'enabled' : 'disabled'}`);
+  }, []);
+
   return {
     // Connection state
     isConnected,
@@ -471,6 +489,7 @@ export default function useVoiceAssistant({
     sendAudio,
     sendAudioChunk, // For real-time audio streaming
     interruptAssistant,
+    setHonorVadInterruption, // Control VAD barge-in behavior
 
     // Audio player controls
     audioPlayer, // Expose audio player for direct control