feat(audio): unify MicrophoneMuteMode, fix AudioManager.shared save/restore

- AudioPipelineConfiguration: MicrophoneMuteMode is no longer a LiveKit
  re-export -- it's the SDK's own 4-case enum (.inputMixer/.restart/
  .voiceProcessing/.software(speechThreshold:)), folding in what used to
  be 4 separate fields (microphoneMuteMode/useSoftwareMute/
  mutedSpeechThreshold + the onSpeechActivity/onMutedSpeech callback
  split). recordingAlwaysPrepared opt-out is removed -- engine pre-warm
  is now unconditional. LiveKit's SpeechActivityEvent no longer leaks
  into this public config surface.
- SoftwareMuteProcessor: onMutedSpeech(MutedSpeechEvent) (throttled,
  periodic, level-based) becomes onSpeakingWhileMutedChange(Bool) (fires
  once on the started/ended edge of a hangover-latched speech segment,
  including on unmute mid-segment). mutedSpeechThrottleInSeconds is gone.
- ConversationAudioManager: configure(with:) now snapshots and restores
  AudioManager.shared's capturePostProcessingDelegate/
  isVoiceProcessingBypassed/isVoiceProcessingAGCEnabled around this
  instance's lifetime (guarded by an identity check on restore) instead
  of unconditionally clearing them on cleanup -- fixes clobbering another
  component that took over the process-wide slot. setRecordingAlwaysPreparedMode
  is now awaited inside configure() (called before WebRTCConnectionManager.connect)
  instead of fired in the background at init -- this adds a new async hop
  to the front of the startup critical path; worth a real-device timing
  check given this SDK's startup-latency history, but not blocking this PR.

Author: renal128

Sources/ElevenLabs/Internal/Conversation/ConversationAudioManager.swift

@@ -4,147 +4,142 @@ import Foundation
 import LiveKit
 #endif
 
-/// Manages audio device configuration and speech activity handling for conversations.
-/// Encapsulates all AudioManager interactions to keep Conversation class focused on conversation logic.
+/// Owns this conversation's `AudioManager.shared` configuration (mute mode, voice
+/// processing, capture pre-warm) and speech-activity handling, keeping
+/// `Conversation` focused on conversation logic.
 @MainActor
 final class ConversationAudioManager {
-    private(set) var audioDevices: [AudioDevice] = []
-    private(set) var selectedAudioDeviceID: String = ""
     private(set) var softwareMuteProcessor: SoftwareMuteProcessor?
 
     private let audioManager = AudioManager.shared
     private var previousSpeechActivityHandler: AudioManager.OnSpeechActivity?
     private var audioSpeechHandlerInstalled = false
     private let logger: any Logging
 
-    /// Callback when audio devices list changes
-    var onDevicesChanged: (([AudioDevice]) -> Void)?
-
-    /// Callback when selected device changes
-    var onSelectedDeviceChanged: ((String) -> Void)?
+    // Snapshots of process-global `AudioManager.shared` state this instance
+    // overwrote in `configure`, so `cleanup`/`deinit` can restore it instead of
+    // leaking settings across sessions or clobbering the host app's values. A
+    // `nil` entry means this instance never changed that setting.
+    //
+    // `previousCaptureDelegate` is only meaningful while `softwareMuteProcessor`
+    // is non-nil.
+    private var previousCaptureDelegate: AudioCustomProcessingDelegate?
+    private var previousVoiceProcessingBypassed: Bool?
+    private var previousVoiceProcessingAGCEnabled: Bool?
 
     init(logger: any Logging) {
         self.logger = logger
-        audioDevices = audioManager.inputDevices
-        selectedAudioDeviceID = audioManager.inputDevice.deviceId
-        setupInitialConfiguration()
     }
 
     deinit {
-        // Reset callbacks directly since we can't call MainActor methods from deinit
-        audioManager.onDeviceUpdate = nil
+        // Best-effort restore if torn down without a clean `cleanup()`. We can't
+        // call MainActor methods here, but these `AudioManager.shared` accessors
+        // are safe off the main actor.
         if audioSpeechHandlerInstalled {
             audioManager.onMutedSpeechActivity = previousSpeechActivityHandler
         }
+        // Only revert the capture delegate if ours is still the installed one; a
+        // process-wide last-write-wins slot means something else may have taken
+        // over after us, and we must not stomp that.
+        if let processor = softwareMuteProcessor,
+           audioManager.capturePostProcessingDelegate.map({ $0 as AnyObject }) === processor
+        {
+            audioManager.capturePostProcessingDelegate = previousCaptureDelegate
+        }
+        if let bypass = previousVoiceProcessingBypassed {
+            audioManager.isVoiceProcessingBypassed = bypass
+        }
+        if let agc = previousVoiceProcessingAGCEnabled {
+            audioManager.isVoiceProcessingAGCEnabled = agc
+        }
     }
 
     // MARK: - Configuration
 
     /// Apply audio pipeline configuration from conversation options.
-    func configure(with options: ConversationOptions) async {
-        let config = options.audioConfiguration
+    ///
+    /// This is the single configuration entry point: it establishes the baseline
+    /// device state (mute mode + engine pre-warm) and applies any caller overrides.
+    func configure(
+        with config: ConversationConfig,
+        onSpeakingWhileMutedChange: @escaping @Sendable (Bool) -> Void
+    ) async {
+        let audioConfig = config.audioConfiguration
+        let muteMode = audioConfig?.microphoneMuteMode ?? .inputMixer
 
-        if let mode = config?.microphoneMuteMode {
-            do {
-                try audioManager.set(microphoneMuteMode: mode)
-            } catch {
-                logger.warning("Failed to set microphone mute mode", context: ["error": "\(error)"])
-            }
+        do {
+            try audioManager.set(microphoneMuteMode: muteMode.toLiveKit())
+        } catch {
+            logger.warning("Failed to set microphone mute mode", context: ["error": "\(error)"])
         }
 
-        if let bypass = config?.voiceProcessingBypassed {
+        // Snapshot before overwriting so `cleanup` can restore the prior value
+        // rather than leaking our override into later sessions / other consumers.
+        if let bypass = audioConfig?.voiceProcessingBypassed {
+            previousVoiceProcessingBypassed = audioManager.isVoiceProcessingBypassed
             audioManager.isVoiceProcessingBypassed = bypass
         }
 
-        if let agc = config?.voiceProcessingAGCEnabled {
+        if let agc = audioConfig?.voiceProcessingAGCEnabled {
+            previousVoiceProcessingAGCEnabled = audioManager.isVoiceProcessingAGCEnabled
             audioManager.isVoiceProcessingAGCEnabled = agc
         }
 
-        if let prepared = config?.recordingAlwaysPrepared {
-            do {
-                try await audioManager.setRecordingAlwaysPreparedMode(prepared)
-            } catch {
-                logger.warning("Failed to set recording always prepared mode", context: ["error": "\(error)"])
-            }
+        // Pre-warm the capture engine via "recording always prepared" mode so the
+        // first `setMicrophone(enabled:)` reuses a warm engine. Without it the
+        // engine cold-starts at publish time and the first VPIO init fails with
+        // audio-engine error -4010 (reproducible on the simulator after a fresh
+        // permission grant).
+        //
+        // Intentionally NOT reverted in `cleanup`: keeping it prepared lets
+        // back-to-back conversations reuse the warm engine, and is a benign global.
+        do {
+            try await audioManager.setRecordingAlwaysPreparedMode(true)
+        } catch {
+            logger.warning("Failed to set recording always prepared mode", context: ["error": "\(error)"])
         }
 
-        configureSpeechHandler(options: options)
-        configureSoftwareMuteProcessor(options: options)
+        configureSpeechHandler(onSpeakingWhileMutedChange: onSpeakingWhileMutedChange)
+        configureSoftwareMuteProcessor(muteMode: muteMode, onSpeakingWhileMutedChange: onSpeakingWhileMutedChange)
     }
 
     /// Cleanup audio state when conversation ends.
     func cleanup() {
         cleanupSpeechHandler()
         cleanupSoftwareMuteProcessor()
+        restoreVoiceProcessingState()
     }
 
     // MARK: - Private
 
-    private func setupInitialConfiguration() {
-        // Set initial microphone mute mode
-        do {
-            try audioManager.set(microphoneMuteMode: .inputMixer)
-        } catch {
-            logger.warning("Failed to set initial microphone mute mode", context: ["error": "\(error)"])
-        }
-
-        // Set recording always prepared mode asynchronously
-        Task { [weak self] in
-            guard let self else { return }
-            do {
-                try await audioManager.setRecordingAlwaysPreparedMode(true)
-            } catch {
-                logger.warning("Failed to set recording always prepared mode", context: ["error": "\(error)"])
-            }
-        }
-
-        // Setup device change observer
-        audioManager.onDeviceUpdate = { [weak self] _ in
-            Task { @MainActor in
-                guard let self else { return }
-                self.audioDevices = self.audioManager.inputDevices
-                self.selectedAudioDeviceID = self.audioManager.defaultInputDevice.deviceId
-                self.onDevicesChanged?(self.audioDevices)
-                self.onSelectedDeviceChanged?(self.selectedAudioDeviceID)
-            }
+    private func configureSpeechHandler(onSpeakingWhileMutedChange: @escaping @Sendable (Bool) -> Void) {
+        if !audioSpeechHandlerInstalled {
+            previousSpeechActivityHandler = audioManager.onMutedSpeechActivity
+            audioSpeechHandlerInstalled = true
         }
-    }
-
-    private func configureSpeechHandler(options: ConversationOptions) {
-        let config = options.audioConfiguration
-        let needsSpeechHandler = (config?.onSpeechActivity != nil) || (options.onSpeechActivity != nil)
-
-        if needsSpeechHandler {
-            if !audioSpeechHandlerInstalled {
-                previousSpeechActivityHandler = audioManager.onMutedSpeechActivity
-                audioSpeechHandlerInstalled = true
-            }
-            audioManager.onMutedSpeechActivity = { _, event in
-                // Handlers are @Sendable, they manage their own synchronization
-                if let handler = config?.onSpeechActivity {
-                    handler(event)
-                }
-                if let handler = options.onSpeechActivity {
-                    handler(event)
-                }
-            }
-        } else if audioSpeechHandlerInstalled {
-            cleanupSpeechHandler()
+        audioManager.onMutedSpeechActivity = { _, event in
+            // Handlers are @Sendable, they manage their own synchronization.
+            onSpeakingWhileMutedChange(event == .started)
         }
     }
 
-    private func configureSoftwareMuteProcessor(options: ConversationOptions) {
-        guard options.audioConfiguration?.useSoftwareMute == true else {
+    private func configureSoftwareMuteProcessor(
+        muteMode: MicrophoneMuteMode,
+        onSpeakingWhileMutedChange: @escaping @Sendable (Bool) -> Void
+    ) {
+        guard case let .software(speechThreshold) = muteMode else {
             return
         }
 
-        let audioConfig = options.audioConfiguration
-        softwareMuteProcessor = SoftwareMuteProcessor(
-            onMutedSpeech: audioConfig?.onMutedSpeech,
-            mutedSpeechThresholdInDb: audioConfig?.mutedSpeechThreshold ?? -35,
-            mutedSpeechThrottleInSeconds: 3.0
+        let processor = SoftwareMuteProcessor(
+            onSpeakingWhileMutedChange: onSpeakingWhileMutedChange,
+            mutedSpeechThresholdInDb: speechThreshold
         )
-        AudioManager.shared.capturePostProcessingDelegate = softwareMuteProcessor
+        softwareMuteProcessor = processor
+        // Snapshot any pre-existing delegate so `cleanup` restores it rather than
+        // nilling out a delegate this instance never owned.
+        previousCaptureDelegate = audioManager.capturePostProcessingDelegate
+        audioManager.capturePostProcessingDelegate = processor
     }
 
     private func cleanupSpeechHandler() {
@@ -156,7 +151,44 @@ final class ConversationAudioManager {
     }
 
     private func cleanupSoftwareMuteProcessor() {
-        AudioManager.shared.capturePostProcessingDelegate = nil
+        guard let processor = softwareMuteProcessor else { return }
+        // Restore the delegate we displaced — but only if ours is still the one
+        // installed. The slot is process-wide last-write-wins, so if another
+        // component set its own delegate after us, leave that in place.
+        if audioManager.capturePostProcessingDelegate.map({ $0 as AnyObject }) === processor {
+            audioManager.capturePostProcessingDelegate = previousCaptureDelegate
+        }
+        previousCaptureDelegate = nil
         softwareMuteProcessor = nil
     }
+
+    /// Restore the VPIO flags this instance overrode in `configure`. The capture
+    /// pre-warm (`setRecordingAlwaysPreparedMode(true)`) is intentionally left
+    /// enabled process-wide (see `configure`), so it is not restored here.
+    private func restoreVoiceProcessingState() {
+        if let bypass = previousVoiceProcessingBypassed {
+            audioManager.isVoiceProcessingBypassed = bypass
+            previousVoiceProcessingBypassed = nil
+        }
+        if let agc = previousVoiceProcessingAGCEnabled {
+            audioManager.isVoiceProcessingAGCEnabled = agc
+            previousVoiceProcessingAGCEnabled = nil
+        }
+    }
 }
+
+// MARK: - LiveKit mapping
+
+private extension MicrophoneMuteMode {
+    /// Maps to LiveKit's hardware mute mode. Software mute has no LiveKit
+    /// equivalent — the track is kept open and muting happens in
+    /// `SoftwareMuteProcessor`, so the engine runs with `.inputMixer` underneath.
+    func toLiveKit() -> LiveKit.MicrophoneMuteMode {
+        switch self {
+        case .voiceProcessing: .voiceProcessing
+        case .restart: .restart
+        case .inputMixer, .software: .inputMixer
+        }
+    }
+}
+

Sources/ElevenLabs/Internal/Utilities/SoftwareMuteProcessor.swift

@@ -19,35 +19,49 @@ final class SoftwareMuteProcessor: NSObject, @unchecked Sendable, AudioCustomPro
 
     private var lock = os_unfair_lock_s()
     private var isMuted: Bool = false
-    private var lastNotificationTime: Date = .distantPast
 
     private var consecutiveAboveCount: Int = 0
     private var consecutiveBelowCount: Int = 0
     private var hangoverLatched: Bool = false
 
-    private let onMutedSpeech: (@Sendable (MutedSpeechEvent) -> Void)?
+    private let onSpeakingWhileMutedChange: (@Sendable (Bool) -> Void)?
     private let mutedSpeechThresholdInDb: Float
-    private let mutedSpeechThrottleInSeconds: TimeInterval
 
     init(
-        onMutedSpeech: (@Sendable (MutedSpeechEvent) -> Void)?,
-        mutedSpeechThresholdInDb: Float = -35,
-        mutedSpeechThrottleInSeconds: TimeInterval = 3.0
+        onSpeakingWhileMutedChange: (@Sendable (Bool) -> Void)?,
+        mutedSpeechThresholdInDb: Float = -35
     ) {
-        self.onMutedSpeech = onMutedSpeech
+        self.onSpeakingWhileMutedChange = onSpeakingWhileMutedChange
         self.mutedSpeechThresholdInDb = mutedSpeechThresholdInDb
-        self.mutedSpeechThrottleInSeconds = mutedSpeechThrottleInSeconds
+    }
+
+    /// The current software-gate mute state. The source of truth for
+    /// `.software` mute mode, where the capture track stays open and the
+    /// hardware mic flag would misleadingly read as unmuted.
+    var muted: Bool {
+        os_unfair_lock_lock(&lock)
+        defer { os_unfair_lock_unlock(&lock) }
+        return isMuted
     }
 
     func setMuted(_ muted: Bool) {
+        var fireEnded = false
         os_unfair_lock_lock(&lock)
         if isMuted != muted {
+            // Unmuting while speech was latched ends the muted-speech segment.
+            if !muted, hangoverLatched {
+                fireEnded = true
+            }
             consecutiveAboveCount = 0
             consecutiveBelowCount = 0
             hangoverLatched = false
         }
         isMuted = muted
         os_unfair_lock_unlock(&lock)
+
+        if fireEnded {
+            DispatchQueue.main.async { self.onSpeakingWhileMutedChange?(false) }
+        }
     }
 
     func audioProcessingProcess(audioBuffer: LKAudioBuffer) {
@@ -73,37 +87,35 @@ final class SoftwareMuteProcessor: NSObject, @unchecked Sendable, AudioCustomPro
 
         let levelActive = db > mutedSpeechThresholdInDb
 
-        var shouldFire = false
-        var fireLevel: Float = 0
+        var fireStarted = false
+        var fireEnded = false
         os_unfair_lock_lock(&lock)
         if levelActive {
             consecutiveBelowCount = 0
             consecutiveAboveCount += 1
-            if consecutiveAboveCount >= Hangover.buffersAboveToConfirm {
+            if consecutiveAboveCount >= Hangover.buffersAboveToConfirm, !hangoverLatched {
                 hangoverLatched = true
+                fireStarted = true
             }
         } else {
             consecutiveAboveCount = 0
             consecutiveBelowCount += 1
-            if consecutiveBelowCount >= Hangover.buffersBelowToClear {
+            if consecutiveBelowCount >= Hangover.buffersBelowToClear, hangoverLatched {
                 hangoverLatched = false
                 consecutiveBelowCount = 0
+                fireEnded = true
             }
         }
+        os_unfair_lock_unlock(&lock)
 
-        if hangoverLatched, levelActive {
-            let now = Date()
-            if now.timeIntervalSince(lastNotificationTime) > mutedSpeechThrottleInSeconds {
-                lastNotificationTime = now
-                shouldFire = true
-                fireLevel = db
+        if fireStarted {
+            DispatchQueue.main.async {
+                self.onSpeakingWhileMutedChange?(true)
             }
         }
-        os_unfair_lock_unlock(&lock)
-
-        if shouldFire {
+        if fireEnded {
             DispatchQueue.main.async {
-                self.onMutedSpeech?(.init(audioLevel: fireLevel))
+                self.onSpeakingWhileMutedChange?(false)
             }
         }