Fixing issues with audio capture for windows and linux

Author: srprasanna

apps/desktop/src-tauri/migrations/002_add_audio_file_path.sql

@@ -0,0 +1,7 @@
+-- Add audio file path to meetings table
+-- This allows tracking of recorded audio files for each meeting
+
+ALTER TABLE meetings ADD COLUMN audio_file_path TEXT;
+
+-- Index for searching meetings with audio files
+CREATE INDEX idx_meetings_audio_file ON meetings(audio_file_path) WHERE audio_file_path IS NOT NULL;

apps/desktop/src-tauri/src/adapters/audio/linux.rs

@@ -11,20 +11,29 @@ use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
 /// Linux PulseAudio capture implementation
+///
+/// Captures system audio output using PulseAudio monitor sources.
+/// Uses @DEFAULT_MONITOR@ to capture what's playing through the speakers.
+///
+/// Audio format: 44100 Hz, 2 channels (stereo), 16-bit signed little-endian
 pub struct PulseAudioCapture {
     is_capturing: Arc<Mutex<bool>>,
     audio_buffer: Arc<Mutex<Vec<f32>>>,
+    /// Audio format - placeholder until capture starts, then set to 44.1kHz stereo 16-bit
     format: AudioFormat,
     capture_handle: Option<tokio::task::JoinHandle<()>>,
 }
 
 impl PulseAudioCapture {
     /// Creates a new PulseAudio capture instance
+    ///
+    /// The format field is initialized to a default placeholder.
+    /// Actual format (44.1kHz stereo 16-bit) is set when `start_capture()` is called.
     pub fn new() -> Self {
         Self {
             is_capturing: Arc::new(Mutex::new(false)),
             audio_buffer: Arc::new(Mutex::new(Vec::new())),
-            format: AudioFormat::default(),
+            format: AudioFormat::default(), // Placeholder, updated during start_capture()
             capture_handle: None,
         }
     }
@@ -49,20 +58,29 @@ impl AudioCapturePort for PulseAudioCapture {
         Ok(vec!["Default Monitor Source".to_string()])
     }
 
-    async fn start_capture(&mut self, _device_name: Option<String>) -> Result<()> {
-        let mut is_capturing = self.is_capturing.lock().unwrap();
-        if *is_capturing {
-            return Err(AppError::AudioCapture(
-                "Capture already in progress".to_string(),
-            ));
-        }
+    async fn start_capture(&mut self, device_name: Option<String>) -> Result<()> {
+        {
+            let mut is_capturing = self.is_capturing.lock().unwrap();
+            if *is_capturing {
+                return Err(AppError::AudioCapture(
+                    "Capture already in progress".to_string(),
+                ));
+            }
 
-        *is_capturing = true;
-        drop(is_capturing);
+            *is_capturing = true;
+        } // Drop is_capturing guard here
 
         let is_capturing_clone = Arc::clone(&self.is_capturing);
         let audio_buffer_clone = Arc::clone(&self.audio_buffer);
 
+        // Determine which device to use for capture
+        // Default to system monitor source if not specified
+        let device = device_name.unwrap_or_else(|| "@DEFAULT_MONITOR@".to_string());
+
+        // Store format info to be updated after detection
+        let format_info = Arc::new(Mutex::new(AudioFormat::default()));
+        let format_info_clone = Arc::clone(&format_info);
+
         // Spawn background task for audio capture
         let handle = tokio::task::spawn_blocking(move || {
             // Set up PulseAudio sample specification
@@ -72,13 +90,20 @@ impl AudioCapturePort for PulseAudioCapture {
                 rate: 44100,                 // 44.1 kHz
             };
 
+            // Store the format
+            *format_info_clone.lock().unwrap() = AudioFormat {
+                sample_rate: spec.rate,
+                channels: spec.channels,
+                bits_per_sample: 16, // S16LE is 16-bit
+            };
+
             // Create a simple recording connection
-            // Using None for device name uses the default monitor source
+            // Use monitor source to capture system audio output
             let simple = match Simple::new(
                 None,                          // Use default server
                 "Meet-Scribe",                // Application name
                 Direction::Record,            // Recording
-                None,                         // Use default monitor device
+                Some(&device),                // Monitor source for system audio
                 "Audio Capture",              // Stream description
                 &spec,                        // Sample spec
                 None,                         // Use default channel map
@@ -93,6 +118,7 @@ impl AudioCapturePort for PulseAudioCapture {
             };
 
             log::info!("PulseAudio capture initialized successfully");
+            log::info!("Device: {}", device);
             log::info!("Format: {} Hz, {} channels, 16-bit", spec.rate, spec.channels);
 
             // Buffer for reading samples (1024 frames at a time)
@@ -137,7 +163,16 @@ impl AudioCapturePort for PulseAudioCapture {
         });
 
         self.capture_handle = Some(handle);
-        log::info!("Audio capture started");
+
+        // Wait for format initialization to complete
+        // Format is set to 44100 Hz, stereo, 16-bit in the background thread
+        tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
+
+        // Update our format from the initialized format
+        self.format = format_info.lock().unwrap().clone();
+
+        log::info!("Audio capture started with format: {} Hz, {} channels, {} bits",
+            self.format.sample_rate, self.format.channels, self.format.bits_per_sample);
         Ok(())
     }
 
@@ -197,9 +232,11 @@ mod tests {
     fn test_default_format() {
         let capture = PulseAudioCapture::new();
         let format = capture.get_format();
-        assert_eq!(format.sample_rate, 16000);
-        assert_eq!(format.channels, 1);
-        assert_eq!(format.bits_per_sample, 16);
+        // Before capture starts, format is the default placeholder
+        // Actual format is set during start_capture() to: 44100 Hz, stereo, 16-bit
+        assert_eq!(format.sample_rate, 16000); // Placeholder before capture
+        assert_eq!(format.channels, 1);         // Placeholder before capture
+        assert_eq!(format.bits_per_sample, 16); // Placeholder before capture
     }
 
     #[tokio::test]

apps/desktop/src-tauri/src/adapters/audio/windows.rs

@@ -18,20 +18,30 @@ use windows::Win32::System::Com::{
 };
 
 /// Windows WASAPI audio capture implementation
+///
+/// Captures system audio output using WASAPI loopback mode.
+/// The audio format (sample rate, channels, bit depth) is auto-detected
+/// from the system's default audio device during `start_capture()`.
+///
+/// Typical Windows audio format: 48000 Hz, 2 channels, 32-bit float
 pub struct WasapiAudioCapture {
     is_capturing: Arc<Mutex<bool>>,
     audio_buffer: Arc<Mutex<Vec<f32>>>,
+    /// Audio format - placeholder until capture starts, then auto-detected
     format: AudioFormat,
     capture_handle: Option<tokio::task::JoinHandle<()>>,
 }
 
 impl WasapiAudioCapture {
     /// Creates a new WASAPI audio capture instance
+    ///
+    /// The format field is initialized to a default placeholder.
+    /// Actual format is detected when `start_capture()` is called.
     pub fn new() -> Self {
         Self {
             is_capturing: Arc::new(Mutex::new(false)),
             audio_buffer: Arc::new(Mutex::new(Vec::new())),
-            format: AudioFormat::default(),
+            format: AudioFormat::default(), // Placeholder, updated during start_capture()
             capture_handle: None,
         }
     }
@@ -61,9 +71,15 @@ impl WasapiAudioCapture {
     }
 
     /// Initialize the audio client with the desired format
-    fn initialize_audio_client(audio_client: &IAudioClient) -> Result<(WAVEFORMATEX, u16, u16)> {
+    ///
+    /// Queries the WASAPI device for its mix format and initializes the audio client
+    /// for loopback capture. Returns the detected format parameters which are used
+    /// to update the WasapiAudioCapture.format field.
+    ///
+    /// Returns: (WAVEFORMATEX, sample_rate, bits_per_sample)
+    fn initialize_audio_client(audio_client: &IAudioClient) -> Result<(WAVEFORMATEX, u32, u16)> {
         unsafe {
-            // Get the device's mix format
+            // Get the device's mix format (auto-detected from system)
             let mix_format_ptr = audio_client
                 .GetMixFormat()
                 .map_err(|e| AppError::AudioCapture(format!("Failed to get mix format: {}", e)))?;
@@ -73,8 +89,8 @@ impl WasapiAudioCapture {
             }
 
             let mix_format = *mix_format_ptr;
-            let sample_rate = mix_format.nSamplesPerSec as u16;
-            let bits_per_sample = mix_format.wBitsPerSample;
+            let sample_rate = mix_format.nSamplesPerSec;  // Actual system sample rate
+            let bits_per_sample = mix_format.wBitsPerSample;  // Actual bit depth
 
             // Initialize the audio client for loopback capture
             let buffer_duration = 10_000_000; // 1 second in 100-nanosecond units
@@ -236,19 +252,24 @@ impl AudioCapturePort for WasapiAudioCapture {
     }
 
     async fn start_capture(&mut self, _device_name: Option<String>) -> Result<()> {
-        let mut is_capturing = self.is_capturing.lock().unwrap();
-        if *is_capturing {
-            return Err(AppError::AudioCapture(
-                "Capture already in progress".to_string(),
-            ));
-        }
+        {
+            let mut is_capturing = self.is_capturing.lock().unwrap();
+            if *is_capturing {
+                return Err(AppError::AudioCapture(
+                    "Capture already in progress".to_string(),
+                ));
+            }
 
-        *is_capturing = true;
-        drop(is_capturing);
+            *is_capturing = true;
+        } // Drop is_capturing guard here
 
         let is_capturing_clone = Arc::clone(&self.is_capturing);
         let audio_buffer_clone = Arc::clone(&self.audio_buffer);
 
+        // Store format info to be updated after detection
+        let format_info = Arc::new(Mutex::new(AudioFormat::default()));
+        let format_info_clone = Arc::clone(&format_info);
+
         // Spawn background task for audio capture
         let handle = tokio::task::spawn_blocking(move || {
             // Initialize COM for this thread
@@ -282,7 +303,8 @@ impl AudioCapturePort for WasapiAudioCapture {
                 }
             };
 
-            // Initialize the audio client
+            // Initialize the audio client and get the actual device format
+            // This is where the format is detected from the WASAPI device
             let (format, sample_rate, bits_per_sample) = match Self::initialize_audio_client(&audio_client) {
                 Ok(f) => f,
                 Err(e) => {
@@ -293,6 +315,15 @@ impl AudioCapturePort for WasapiAudioCapture {
                 }
             };
 
+            // IMPORTANT: Update format with actual detected values from the device
+            // This replaces the default placeholder values with the real audio format
+            let channels = format.nChannels;
+            *format_info_clone.lock().unwrap() = AudioFormat {
+                sample_rate,        // e.g., 48000 Hz (detected from device)
+                channels,           // e.g., 2 (stereo, detected from device)
+                bits_per_sample,    // e.g., 32 bits (float, detected from device)
+            };
+
             // Get the capture client
             let capture_client: IAudioCaptureClient = match unsafe {
                 audio_client.GetService::<IAudioCaptureClient>()
@@ -307,7 +338,7 @@ impl AudioCapturePort for WasapiAudioCapture {
             };
 
             log::info!("WASAPI audio capture initialized successfully");
-            log::info!("Format: {} Hz, {} bits", sample_rate, bits_per_sample);
+            log::info!("Format: {} Hz, {} channels, {} bits", sample_rate, channels, bits_per_sample);
 
             // Run the capture loop
             Self::capture_loop(
@@ -324,7 +355,17 @@ impl AudioCapturePort for WasapiAudioCapture {
         });
 
         self.capture_handle = Some(handle);
-        log::info!("Audio capture started");
+
+        // Wait for format detection to complete
+        // The background thread detects the system's audio format and stores it in format_info
+        // Typical Windows audio: 48000 Hz, stereo, 32-bit float
+        tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
+
+        // Update our format from the auto-detected format
+        self.format = format_info.lock().unwrap().clone();
+
+        log::info!("Audio capture started with format: {} Hz, {} channels, {} bits",
+            self.format.sample_rate, self.format.channels, self.format.bits_per_sample);
         Ok(())
     }
 
@@ -384,9 +425,12 @@ mod tests {
     fn test_default_format() {
         let capture = WasapiAudioCapture::new();
         let format = capture.get_format();
-        assert_eq!(format.sample_rate, 16000);
-        assert_eq!(format.channels, 1);
-        assert_eq!(format.bits_per_sample, 16);
+        // Before capture starts, format is the default placeholder
+        // Actual format is detected during start_capture() and varies by system
+        // Typical Windows audio: 48000 Hz, 2 channels, 32 bits (float)
+        assert_eq!(format.sample_rate, 16000); // Placeholder before capture
+        assert_eq!(format.channels, 1);         // Placeholder before capture
+        assert_eq!(format.bits_per_sample, 16); // Placeholder before capture
     }
 
     #[tokio::test]

apps/desktop/src-tauri/src/adapters/storage/sqlite.rs

@@ -33,9 +33,10 @@ impl SqliteStorage {
     pub fn run_migrations(&self) -> Result<()> {
         use rusqlite_migration::{Migrations, M};
 
-        let migrations = Migrations::new(vec![M::up(include_str!(
-            "../../../migrations/001_initial.sql"
-        ))]);
+        let migrations = Migrations::new(vec![
+            M::up(include_str!("../../../migrations/001_initial.sql")),
+            M::up(include_str!("../../../migrations/002_add_audio_file_path.sql")),
+        ]);
 
         let mut conn = self.conn.lock().unwrap();
         migrations.to_latest(&mut conn).map_err(|e| {
@@ -51,14 +52,15 @@ impl StoragePort for SqliteStorage {
     async fn create_meeting(&self, meeting: &Meeting) -> Result<i64> {
         let conn = self.conn.lock().unwrap();
         conn.execute(
-            "INSERT INTO meetings (platform, title, start_time, end_time, participant_count, created_at)
-             VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
+            "INSERT INTO meetings (platform, title, start_time, end_time, participant_count, audio_file_path, created_at)
+             VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
             params![
                 meeting.platform.to_string(),
                 meeting.title,
                 meeting.start_time,
                 meeting.end_time,
                 meeting.participant_count,
+                meeting.audio_file_path,
                 meeting.created_at,
             ],
         )?;
@@ -68,7 +70,7 @@ impl StoragePort for SqliteStorage {
     async fn get_meeting(&self, id: i64) -> Result<Option<Meeting>> {
         let conn = self.conn.lock().unwrap();
         let mut stmt = conn.prepare(
-            "SELECT id, platform, title, start_time, end_time, participant_count, created_at
+            "SELECT id, platform, title, start_time, end_time, participant_count, audio_file_path, created_at
              FROM meetings WHERE id = ?1",
         )?;
 
@@ -90,7 +92,8 @@ impl StoragePort for SqliteStorage {
                 start_time: row.get(3)?,
                 end_time: row.get(4)?,
                 participant_count: row.get(5)?,
-                created_at: row.get(6)?,
+                audio_file_path: row.get(6)?,
+                created_at: row.get(7)?,
             }))
         } else {
             Ok(None)
@@ -100,7 +103,7 @@ impl StoragePort for SqliteStorage {
     async fn list_meetings(&self, limit: Option<i32>, offset: Option<i32>) -> Result<Vec<Meeting>> {
         let conn = self.conn.lock().unwrap();
         let query = format!(
-            "SELECT id, platform, title, start_time, end_time, participant_count, created_at
+            "SELECT id, platform, title, start_time, end_time, participant_count, audio_file_path, created_at
              FROM meetings ORDER BY start_time DESC LIMIT ?1 OFFSET ?2"
         );
 
@@ -121,7 +124,8 @@ impl StoragePort for SqliteStorage {
                 start_time: row.get(3)?,
                 end_time: row.get(4)?,
                 participant_count: row.get(5)?,
-                created_at: row.get(6)?,
+                audio_file_path: row.get(6)?,
+                created_at: row.get(7)?,
             })
         })?;
 
@@ -137,13 +141,14 @@ impl StoragePort for SqliteStorage {
         let conn = self.conn.lock().unwrap();
         conn.execute(
             "UPDATE meetings SET platform = ?1, title = ?2, start_time = ?3, end_time = ?4,
-             participant_count = ?5 WHERE id = ?6",
+             participant_count = ?5, audio_file_path = ?6 WHERE id = ?7",
             params![
                 meeting.platform.to_string(),
                 meeting.title,
                 meeting.start_time,
                 meeting.end_time,
                 meeting.participant_count,
+                meeting.audio_file_path,
                 meeting.id,
             ],
         )?;