Add video audio source support to Voice Triggers

Users can now transcribe audio directly from the sample video (or any video source) instead of only microphone. Added audio source dropdown (Video Audio / Microphone) that defaults to Video Audio when sample video is playing. Uses Web Audio API MediaElementSource to capture audio from the video element. Auto-switches to microphone when user selects live camera.

Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-opencode)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>

Author: streamgeeks

15-voice-triggers/app.js

@@ -31,6 +31,10 @@ class VoiceControl {
         this.transcriptHistory = [];
         this.maxHistory = 50;
 
+        // Audio source: 'video' or 'microphone'
+        this.audioSource = options.audioSource || 'video';
+        this.videoElement = options.videoElement || null;
+        
         this._loadRules();
     }
 
@@ -107,6 +111,8 @@ class VoiceControl {
 
     async startAudioCapture() {
         try {
+            // Mute video when using microphone to avoid echo
+            if (this.videoElement) this.videoElement.muted = true;
             this.mediaStream = await navigator.mediaDevices.getUserMedia({
                 audio: {
                     channelCount: 1,
@@ -170,6 +176,61 @@ class VoiceControl {
         this.audioChunks = [];
     }
 
+    async startVideoAudioCapture(videoElement) {
+        try {
+            // Unmute video so MediaElementSource receives audio
+            videoElement.muted = false;
+            videoElement.volume = 0.3;
+            
+            this.audioContext = new (window.AudioContext || window.webkitAudioContext)({
+                sampleRate: this.sampleRate
+            });
+            
+            // MediaElementSource can only be created once per element
+            if (!videoElement._mediaElementSource) {
+                videoElement._mediaElementSource = this.audioContext.createMediaElementSource(videoElement);
+            }
+            const source = videoElement._mediaElementSource;
+            
+            // Must connect to destination so user can still hear the video
+            source.connect(this.audioContext.destination);
+            
+            this.analyser = this.audioContext.createAnalyser();
+            this.analyser.fftSize = 256;
+            source.connect(this.analyser);
+            
+            this.processor = this.audioContext.createScriptProcessor(4096, 1, 1);
+            source.connect(this.processor);
+            this.processor.connect(this.audioContext.destination);
+            
+            this.audioChunks = [];
+            
+            this.processor.onaudioprocess = (event) => {
+                if (!this.isRunning) return;
+                const inputData = event.inputBuffer.getChannelData(0);
+                this.audioChunks.push(new Float32Array(inputData));
+            };
+            
+            this._monitorAudioLevel();
+            
+            console.log('[VoiceControl] Video audio capture started');
+            return true;
+        } catch (error) {
+            console.error('[VoiceControl] Video audio capture failed:', error);
+            this.onStatusUpdate('Video audio capture failed: ' + error.message);
+            return false;
+        }
+    }
+
+    setAudioSource(source, videoElement) {
+        this.audioSource = source;
+        if (videoElement) this.videoElement = videoElement;
+        if (this.isRunning) {
+            this.stop();
+            this.start();
+        }
+    }
+
     _monitorAudioLevel() {
         if (!this.analyser || !this.isRunning) return;
 
@@ -340,9 +401,13 @@ class VoiceControl {
             if (!loaded) return false;
         }
 
-        const capturing = await this.startAudioCapture();
+        let capturing;
+        if (this.audioSource === 'video' && this.videoElement) {
+            capturing = await this.startVideoAudioCapture(this.videoElement);
+        } else {
+            capturing = await this.startAudioCapture();
+        }
         if (!capturing) return false;
-        
         this.isRunning = true;
 
         this.processingInterval = setInterval(() => {
@@ -415,6 +480,8 @@ class VoiceTriggersApp {
         this.voiceControl = new VoiceControl({
             chunkDuration: parseInt(document.getElementById('chunkDuration').value) * 1000,
             cooldown: parseInt(document.getElementById('cooldown').value) * 1000,
+            audioSource: 'video',
+            videoElement: document.getElementById('video'),
 
             onTranscript: (text, entry) => this.handleTranscript(text, entry),
             onRuleTriggered: (info) => this.handleRuleTrigger(info),
@@ -438,6 +505,16 @@ class VoiceTriggersApp {
                     cameraSelect.disabled = source === 'sample';
                     if (refreshBtn) refreshBtn.disabled = source === 'sample';
                     this.reasoningConsole.logInfo(`Switched to ${source === 'camera' ? 'live camera' : 'sample video'}`);
+                    
+                    // Auto-switch audio source based on video source
+                    const audioSourceSelect = document.getElementById('audioSourceSelect');
+                    if (source === 'sample') {
+                        if (audioSourceSelect) audioSourceSelect.value = 'video';
+                        this.voiceControl.setAudioSource('video', videoElement);
+                    } else {
+                        if (audioSourceSelect) audioSourceSelect.value = 'microphone';
+                        this.voiceControl.setAudioSource('microphone', videoElement);
+                    }
                 }
             });
             VideoSourceAdapter.switchToSample().catch(() => {
@@ -449,6 +526,23 @@ class VoiceTriggersApp {
     initEventListeners() {
         document.getElementById('startBtn').addEventListener('click', () => this.toggleVoice());
 
+        // Audio source toggle
+        const audioSourceSelect = document.getElementById('audioSourceSelect');
+        if (audioSourceSelect) {
+            audioSourceSelect.addEventListener('change', (e) => {
+                const source = e.target.value;
+                const video = document.getElementById('video');
+                if (source === 'video') {
+                    video.muted = false;
+                    video.volume = 0.3;
+                } else {
+                    video.muted = true;
+                }
+                this.voiceControl.setAudioSource(source, video);
+                this.reasoningConsole.logInfo(`Audio source: ${source === 'video' ? 'Video Audio' : 'Microphone'}`);
+            });
+        }
+        
         document.getElementById('chunkDuration').addEventListener('input', (e) => {
             const value = e.target.value;
             document.getElementById('chunkDurationValue').textContent = `${value}s`;

15-voice-triggers/index.html

@@ -698,7 +698,14 @@ <h4>No API Key Required for Speech Recognition</h4>
 
             <!-- Audio Visualizer -->
             <div class="audio-section">
-                <h3>&#x1F3A4; Microphone Input</h3>
+                <h3>🎤 Audio Input</h3>
+                <div style="display: flex; align-items: center; gap: 8px; margin-bottom: 8px;">
+                    <label for="audioSourceSelect" style="font-size: 0.85rem; font-weight: 600; color: var(--text-secondary, #aaa);">Source:</label>
+                    <select id="audioSourceSelect" style="padding: 4px 8px; border-radius: 4px; border: 1px solid var(--border, #444); background: var(--bg-secondary, #2a2a2a); color: var(--text-primary, #fff); font-size: 0.85rem;">
+                        <option value="video" selected>📹 Video Audio</option>
+                        <option value="microphone">🎤 Microphone</option>
+                    </select>
+                </div>
                 <div class="audio-visualizer" id="audioVisualizer">
                     <!-- Bars will be generated by JS -->
                 </div>