feat: SDK hardening, improved startup chain, callbacks, expose audio config & startup metrics

Author: louisjoecodes

Sources/ElevenLabs/Auth/TokenService.swift

@@ -204,3 +204,5 @@ enum TokenError: LocalizedError, Sendable {
         }
     }
 }
+
+extension TokenService: TokenServicing {}

Sources/ElevenLabs/Conversation.swift

@@ -1,13 +1,45 @@
+import Foundation
 import LiveKit
 
+protocol TokenServicing: Sendable {
+    func fetchConnectionDetails(configuration: ElevenLabsConfiguration) async throws -> TokenService.ConnectionDetails
+}
+
+@MainActor
+protocol ConnectionManaging: AnyObject {
+    var onAgentReady: (() -> Void)? { get set }
+    var onAgentDisconnected: (() -> Void)? { get set }
+    var room: Room? { get }
+    var shouldObserveRoomConnection: Bool { get }
+    var errorHandler: (Error?) -> Void { get set }
+
+    func connect(
+        details: TokenService.ConnectionDetails,
+        enableMic: Bool,
+        graceTimeout: TimeInterval,
+    ) async throws
+
+    func disconnect() async
+    func dataEventsStream() -> AsyncStream<Data>
+    func waitForAgentReady(timeout: TimeInterval) async -> AgentReadyWaitResult
+    func publish(data: Data, options: DataPublishOptions) async throws
+}
+
+@MainActor
+protocol ConversationDependencyProvider: AnyObject {
+    var tokenService: any TokenServicing { get }
+    var connectionManager: any ConnectionManaging { get }
+    var errorHandler: (Error?) -> Void { get }
+}
+
 /// A minimalistic dependency injection container.
 /// It allows sharing common dependencies e.g. `Room` between view models and services.
 /// - Note: For production apps, consider using a more flexible approach offered by e.g.:
 ///   - [Factory](https://github.com/hmlongco/Factory)
 ///   - [swift-dependencies](https://github.com/pointfreeco/swift-dependencies)
 ///   - [Needle](https://github.com/uber/needle)
 @MainActor
-final class Dependencies {
+final class Dependencies: ConversationDependencyProvider {
     static let shared = Dependencies()
 
     private init() {}
@@ -18,7 +50,7 @@ final class Dependencies {
 
     // MARK: Services
 
-    lazy var tokenService: TokenService = {
+    lazy var tokenService: any TokenServicing = {
         let globalConfig = ElevenLabs.Global.shared.configuration
         let tokenServiceConfig = TokenService.Configuration(
             apiEndpoint: globalConfig.apiEndpoint?.absoluteString,
@@ -27,7 +59,7 @@ final class Dependencies {
         return TokenService(configuration: tokenServiceConfig)
     }()
 
-    lazy var connectionManager = ConnectionManager()
+    lazy var connectionManager: any ConnectionManaging = ConnectionManager()
 
     private lazy var localMessageSender = LocalMessageSender(room: room)
 

Sources/ElevenLabs/DI/Dependencies.swift

@@ -194,6 +194,8 @@ public enum ElevenLabs {
     // Protocol event types are already public from their respective files
     // Re-export AgentState from LiveKit for SDK users
     public typealias AgentState = LiveKit.AgentState
+    public typealias SpeechActivityEvent = LiveKit.SpeechActivityEvent
+    public typealias MicrophoneMuteMode = LiveKit.MicrophoneMuteMode
 
     // Re-export audio track types for advanced audio handling
     public typealias LocalAudioTrack = LiveKit.LocalAudioTrack

Sources/ElevenLabs/ElevenLabs.swift

@@ -0,0 +1,39 @@
+import Foundation
+import LiveKit
+
+/// Configures microphone pipeline and voice activity reporting exposed by the SDK.
+public struct AudioPipelineConfiguration: Sendable {
+    /// Override the microphone mute strategy. Defaults to `.inputMixer` to match previous SDK behaviour.
+    public var microphoneMuteMode: MicrophoneMuteMode?
+
+    /// Keep the recording engine warm to avoid first-spoken-word latency. Defaults to `true`.
+    public var recordingAlwaysPrepared: Bool?
+
+    /// Bypass WebRTC voice processing (AEC/NS/VAD). Leave `nil` to preserve system defaults.
+    public var voiceProcessingBypassed: Bool?
+
+    /// Toggle Auto Gain Control. Leave `nil` to preserve system defaults.
+    public var voiceProcessingAGCEnabled: Bool?
+
+    /// Observe LiveKit speech activity events while the microphone is muted.
+    public var onSpeechActivity: (@Sendable (SpeechActivityEvent) -> Void)?
+
+    public init(
+        microphoneMuteMode: MicrophoneMuteMode? = .inputMixer,
+        recordingAlwaysPrepared: Bool? = true,
+        voiceProcessingBypassed: Bool? = nil,
+        voiceProcessingAGCEnabled: Bool? = nil,
+        onSpeechActivity: (@Sendable (SpeechActivityEvent) -> Void)? = nil,
+    ) {
+        self.microphoneMuteMode = microphoneMuteMode
+        self.recordingAlwaysPrepared = recordingAlwaysPrepared
+        self.voiceProcessingBypassed = voiceProcessingBypassed
+        self.voiceProcessingAGCEnabled = voiceProcessingAGCEnabled
+        self.onSpeechActivity = onSpeechActivity
+    }
+
+    public static let `default` = AudioPipelineConfiguration()
+}
+
+extension MicrophoneMuteMode: @retroactive @unchecked Sendable {}
+extension SpeechActivityEvent: @retroactive @unchecked Sendable {}

Sources/ElevenLabs/Models/AudioPipelineConfiguration.swift

@@ -1,4 +1,5 @@
 import Foundation
+import LiveKit
 
 /// Main configuration for a conversation session
 public struct ConversationConfig: Sendable {
@@ -15,6 +16,51 @@ public struct ConversationConfig: Sendable {
     /// Called when the agent disconnects or the conversation ends
     public var onDisconnect: (@Sendable () -> Void)?
 
+    /// Called whenever the startup state transitions
+    public var onStartupStateChange: (@Sendable (ConversationStartupState) -> Void)?
+
+    /// Controls timings and retry behavior for the initialization handshake
+    public var startupConfiguration: ConversationStartupConfiguration
+
+    /// Controls microphone pipeline behaviour and VAD callbacks.
+    public var audioConfiguration: AudioPipelineConfiguration?
+
+    /// Called when a startup-related error occurs
+    public var onError: (@Sendable (ConversationError) -> Void)?
+
+    /// Called when LiveKit detects speech activity while muted.
+    public var onSpeechActivity: (@Sendable (SpeechActivityEvent) -> Void)?
+
+    /// Called for each agent response with the associated event identifier.
+    public var onAgentResponse: (@Sendable (_ text: String, _ eventId: Int) -> Void)?
+
+    /// Called when an agent response correction is received.
+    public var onAgentResponseCorrection: (@Sendable (_ original: String, _ corrected: String, _ eventId: Int) -> Void)?
+
+    /// Called for each user transcript event.
+    public var onUserTranscript: (@Sendable (_ text: String, _ eventId: Int) -> Void)?
+
+    /// Called when conversation metadata arrives.
+    public var onConversationMetadata: (@Sendable (ConversationMetadataEvent) -> Void)?
+
+    /// Called when the agent emits a tool response event.
+    public var onAgentToolResponse: (@Sendable (AgentToolResponseEvent) -> Void)?
+
+    /// Called when the agent detects an interruption.
+    public var onInterruption: (@Sendable (_ eventId: Int) -> Void)?
+
+    /// Called whenever a VAD score is emitted.
+    public var onVadScore: (@Sendable (_ score: Double) -> Void)?
+
+    /// Called when audio alignment metadata is emitted.
+    public var onAudioAlignment: (@Sendable (AudioAlignment) -> Void)?
+
+    /// Called when feedback availability changes.
+    public var onCanSendFeedbackChange: (@Sendable (Bool) -> Void)?
+
+    /// Called when a client tool call is received without a registered handler.
+    public var onUnhandledClientToolCall: (@Sendable (ClientToolCallEvent) -> Void)?
+
     public init(
         agentOverrides: AgentOverrides? = nil,
         ttsOverrides: TTSOverrides? = nil,
@@ -24,6 +70,21 @@ public struct ConversationConfig: Sendable {
         userId: String? = nil,
         onAgentReady: (@Sendable () -> Void)? = nil,
         onDisconnect: (@Sendable () -> Void)? = nil,
+        onStartupStateChange: (@Sendable (ConversationStartupState) -> Void)? = nil,
+        startupConfiguration: ConversationStartupConfiguration = .default,
+        audioConfiguration: AudioPipelineConfiguration? = nil,
+        onError: (@Sendable (ConversationError) -> Void)? = nil,
+        onSpeechActivity: (@Sendable (SpeechActivityEvent) -> Void)? = nil,
+        onAgentResponse: (@Sendable (_ text: String, _ eventId: Int) -> Void)? = nil,
+        onAgentResponseCorrection: (@Sendable (_ original: String, _ corrected: String, _ eventId: Int) -> Void)? = nil,
+        onUserTranscript: (@Sendable (_ text: String, _ eventId: Int) -> Void)? = nil,
+        onConversationMetadata: (@Sendable (ConversationMetadataEvent) -> Void)? = nil,
+        onAgentToolResponse: (@Sendable (AgentToolResponseEvent) -> Void)? = nil,
+        onInterruption: (@Sendable (_ eventId: Int) -> Void)? = nil,
+        onVadScore: (@Sendable (_ score: Double) -> Void)? = nil,
+        onAudioAlignment: (@Sendable (AudioAlignment) -> Void)? = nil,
+        onCanSendFeedbackChange: (@Sendable (Bool) -> Void)? = nil,
+        onUnhandledClientToolCall: (@Sendable (ClientToolCallEvent) -> Void)? = nil,
     ) {
         self.agentOverrides = agentOverrides
         self.ttsOverrides = ttsOverrides
@@ -33,6 +94,21 @@ public struct ConversationConfig: Sendable {
         self.userId = userId
         self.onAgentReady = onAgentReady
         self.onDisconnect = onDisconnect
+        self.onStartupStateChange = onStartupStateChange
+        self.startupConfiguration = startupConfiguration
+        self.audioConfiguration = audioConfiguration
+        self.onError = onError
+        self.onSpeechActivity = onSpeechActivity
+        self.onAgentResponse = onAgentResponse
+        self.onAgentResponseCorrection = onAgentResponseCorrection
+        self.onUserTranscript = onUserTranscript
+        self.onConversationMetadata = onConversationMetadata
+        self.onAgentToolResponse = onAgentToolResponse
+        self.onInterruption = onInterruption
+        self.onVadScore = onVadScore
+        self.onAudioAlignment = onAudioAlignment
+        self.onCanSendFeedbackChange = onCanSendFeedbackChange
+        self.onUnhandledClientToolCall = onUnhandledClientToolCall
     }
 }
 
@@ -101,6 +177,21 @@ extension ConversationConfig {
             userId: userId,
             onAgentReady: onAgentReady,
             onDisconnect: onDisconnect,
+            onStartupStateChange: onStartupStateChange,
+            startupConfiguration: startupConfiguration,
+            audioConfiguration: audioConfiguration,
+            onError: onError,
+            onSpeechActivity: onSpeechActivity,
+            onAgentResponse: onAgentResponse,
+            onAgentResponseCorrection: onAgentResponseCorrection,
+            onUserTranscript: onUserTranscript,
+            onConversationMetadata: onConversationMetadata,
+            onAgentToolResponse: onAgentToolResponse,
+            onInterruption: onInterruption,
+            onVadScore: onVadScore,
+            onAudioAlignment: onAudioAlignment,
+            onCanSendFeedbackChange: onCanSendFeedbackChange,
+            onUnhandledClientToolCall: onUnhandledClientToolCall,
         )
     }
 }