diff --git a/.gitignore b/.gitignore index b5cd6bb5..583f9f24 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ xcuserdata/ xcshareddata/ Package.resolved *.bak +DerivedData/ +*.xcuserstate diff --git a/Documentation/Usage.md b/Documentation/Usage.md index 51a30dc8..6ee1f80b 100644 --- a/Documentation/Usage.md +++ b/Documentation/Usage.md @@ -64,7 +64,8 @@ conversation.$agentState ### Connection State -Handle transitions between idle, connecting, connected, and ended states. +Handle transitions between idle, connecting, connected, ended, and error states. +Startup stages are nested under `.connecting`. ```swift conversation.$state @@ -72,10 +73,11 @@ conversation.$state switch state { case .idle: break - case .connecting: - break + case .connecting(let stage): + print("Connecting: \(stage)") case .connected(let callInfo): print("Connected to agent: \(callInfo.agentId)") + print("Conversation ID: \(callInfo.conversationId)") case .ended(let reason): print("Conversation ended: \(reason)") case .error(let error): @@ -190,21 +192,29 @@ struct AudioControlView: View { } ``` -### Raw Audio Tracks +### Raw Audio Observers -Access the underlying LiveKit audio tracks for advanced visualization (e.g., audio visualizers or level meters). +Observe decoded PCM from the agent output or local microphone without taking a LiveKit dependency in app code. Prefer this over reaching for transport tracks. ```swift -// Use these with LiveKit view components or custom processors -if let inputTrack = conversation.inputTrack as? LocalAudioTrack { - // Access local microphone track +final class SpectrumObserver: ConversationAudioObserver, @unchecked Sendable { + func didReceive(_ buffer: AVAudioPCMBuffer) { + // Time-critical path: copy what you need, then hop off this callback. + let copy = buffer.copy() + Task { @MainActor in + updateVisualizer(copy) + } + } } -if let agentTrack = conversation.agentAudioTrack as? RemoteAudioTrack { - // Access agent's audio track -} +let client = ConversationClient() +let observer = SpectrumObserver() +client.addAgentAudioObserver(observer) +client.addMicAudioObserver(observer) ``` +`didReceive(_:)` runs on the audio callback path (not the main actor). The buffer is borrowed and read-only. + --- ## Tool Calls @@ -277,13 +287,17 @@ class ConversationViewModel: ObservableObject { result = "Unknown tool: \(toolCall.toolName)" } - try await conversation.sendToolResult(for: toolCall.toolCallId, result: result) + try await conversation.sendToolResult( + .init(toolCallId: toolCall.toolCallId, result: result) + ) } catch { print("Tool execution failed: \(error)") try? await conversation.sendToolResult( - for: toolCall.toolCallId, - result: "Error: \(error.localizedDescription)", - isError: true + .init( + toolCallId: toolCall.toolCallId, + result: "Error: \(error.localizedDescription)", + isError: true + ) ) } } @@ -484,10 +498,6 @@ let callbacks = ConversationCallbacks( onAudioAlignment: { alignment in // Real-time word highlighting timing. }, - onCanSendFeedbackChange: { canSend in - // Enable/disable your 'Thumbs Up/Down' buttons in the UI - self.showFeedbackUI = canSend - }, onError: { error in print("A non-fatal or startup error occurred: \(error)") } @@ -519,20 +529,17 @@ let config = ConversationConfig(audioConfiguration: audioConfig) --- -## Startup Performance Tuning {#startup-tuning} +## Startup Timeouts {#startup-tuning} -Control the connection handshake and retry behavior. +Control how long startup waits for the agent and conversation metadata. ```swift let startupConfig = ConversationStartupConfiguration( - // Time to wait for agent to be 'ready' after room connection + // Time to wait for the agent after the room connects agentReadyTimeout: 10.0, - - // Backoff strategy for protocol initialization - initRetryDelays: [0, 1.0, 2.0, 5.0], - - // Whether to fail early if agent takes too long - failIfAgentNotReady: true + + // Time to wait for the server to initialize the conversation + initiationMetadataTimeout: 5.0 ) let config = ConversationConfig(startupConfiguration: startupConfig) @@ -542,35 +549,29 @@ let config = ConversationConfig(startupConfiguration: startupConfig) ## Feedback & Context {#feedback-context} -Handle feedback (like/dislike) and contextual updates to the agent. +Feedback can be sent whenever the conversation is connected. ```swift -// 1) Setup: react to feedback availability -var canSendFeedback = false - let callbacks = ConversationCallbacks( onAgentResponse: { text, eventId in print("Agent:", text, "(event:", eventId, ")") - }, - onCanSendFeedbackChange: { can in - canSendFeedback = can - // e.g., refresh your UI } ) -let conversation = try await ElevenLabs.startConversation(agentId: "agent_123", callbacks: callbacks) +let client = ConversationClient(callbacks: callbacks) +_ = try await client.startConversation(agentId: "agent_123") -// 2) Sending feedback from your UI -func thumbsUp(latestEventId: Int) { - Task { try? await conversation.sendFeedback(.like, eventId: latestEventId) } +func thumbsUp(eventId: Int) async throws { + guard client.state.isConnected else { return } + try await client.sendFeedback(.like, eventId: eventId) } -func thumbsDown(latestEventId: Int) { - Task { try? await conversation.sendFeedback(.dislike, eventId: latestEventId) } +func thumbsDown(eventId: Int) async throws { + guard client.state.isConnected else { return } + try await client.sendFeedback(.dislike, eventId: eventId) } -// 3) Contextual updates (e.g., user preferences) -Task { try? await conversation.updateContext("user_prefers_detailed_answers=true") } +try await client.updateContext("user_prefers_detailed_answers=true") ``` --- @@ -609,7 +610,7 @@ class ReconnectionManager: ObservableObject { self.showReconnectButton = false self.isReconnecting = false self.reconnectAttempts = 0 - default: + case .idle, .connecting, .error: break } } @@ -715,9 +716,7 @@ The SDK uses `os.Logger` for high-performance logging. You can filter logs in Xc Adjust the verbosity of the SDK: ```swift -ElevenLabs.configure( - ElevenLabs.Configuration(logLevel: .debug) // .trace for full event logs -) +let client = ConversationClient(logLevel: .debug) // .trace for full event logs ``` --- diff --git a/Examples/DemoApp/DemoApp.xcodeproj/project.pbxproj b/Examples/DemoApp/DemoApp.xcodeproj/project.pbxproj new file mode 100644 index 00000000..cd2e6dcf --- /dev/null +++ b/Examples/DemoApp/DemoApp.xcodeproj/project.pbxproj @@ -0,0 +1,281 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 77; + objects = { + +/* Begin PBXBuildFile section */ + E1000101000000000000A001 /* ElevenLabs in Frameworks */ = {isa = PBXBuildFile; productRef = E1000201000000000000A001 /* ElevenLabs */; }; + E1000102000000000000A001 /* ElevenLabsWidget in Frameworks */ = {isa = PBXBuildFile; productRef = E1000202000000000000A001 /* ElevenLabsWidget */; }; +/* End PBXBuildFile section */ + +/* Begin PBXFileReference section */ + E1000301000000000000A001 /* DemoApp.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = DemoApp.app; sourceTree = BUILT_PRODUCTS_DIR; }; +/* End PBXFileReference section */ + +/* Begin PBXFileSystemSynchronizedRootGroup section */ + E1000401000000000000A001 /* DemoApp */ = { + isa = PBXFileSystemSynchronizedRootGroup; + path = DemoApp; + sourceTree = ""; + }; +/* End PBXFileSystemSynchronizedRootGroup section */ + +/* Begin PBXFrameworksBuildPhase section */ + E1000501000000000000A001 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + E1000101000000000000A001 /* ElevenLabs in Frameworks */, + E1000102000000000000A001 /* ElevenLabsWidget in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + E1000601000000000000A001 = { + isa = PBXGroup; + children = ( + E1000401000000000000A001 /* DemoApp */, + E1000602000000000000A001 /* Products */, + ); + sourceTree = ""; + }; + E1000602000000000000A001 /* Products */ = { + isa = PBXGroup; + children = ( + E1000301000000000000A001 /* DemoApp.app */, + ); + name = Products; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + E1000701000000000000A001 /* DemoApp */ = { + isa = PBXNativeTarget; + buildConfigurationList = E1000901000000000000A001 /* Build configuration list for PBXNativeTarget "DemoApp" */; + buildPhases = ( + E1000801000000000000A001 /* Sources */, + E1000501000000000000A001 /* Frameworks */, + E1000802000000000000A001 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + fileSystemSynchronizedGroups = ( + E1000401000000000000A001 /* DemoApp */, + ); + name = DemoApp; + packageProductDependencies = ( + E1000201000000000000A001 /* ElevenLabs */, + E1000202000000000000A001 /* ElevenLabsWidget */, + ); + productName = DemoApp; + productReference = E1000301000000000000A001 /* DemoApp.app */; + productType = "com.apple.product-type.application"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + E1000001000000000000A001 /* Project object */ = { + isa = PBXProject; + attributes = { + BuildIndependentTargetsInParallel = 1; + LastSwiftUpdateCheck = 2640; + LastUpgradeCheck = 2640; + }; + buildConfigurationList = E1000902000000000000A001 /* Build configuration list for PBXProject "DemoApp" */; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = E1000601000000000000A001; + minimizedProjectReferenceProxies = 1; + packageReferences = ( + E1000A01000000000000A001 /* XCLocalSwiftPackageReference "../.." */, + ); + preferredProjectObjectVersion = 77; + productRefGroup = E1000602000000000000A001 /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + E1000701000000000000A001 /* DemoApp */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + E1000802000000000000A001 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + E1000801000000000000A001 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin XCBuildConfiguration section */ + E1000B01000000000000A001 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + ENABLE_USER_SCRIPT_SANDBOXING = YES; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + IPHONEOS_DEPLOYMENT_TARGET = 17.0; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_FAST_MATH = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = iphoneos; + SWIFT_ACTIVE_COMPILATION_CONDITIONS = "DEBUG $(inherited)"; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + }; + name = Debug; + }; + E1000B02000000000000A001 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_USER_SCRIPT_SANDBOXING = YES; + GCC_C_LANGUAGE_STANDARD = gnu17; + IPHONEOS_DEPLOYMENT_TARGET = 17.0; + MTL_ENABLE_DEBUG_INFO = NO; + MTL_FAST_MATH = YES; + SDKROOT = iphoneos; + SWIFT_COMPILATION_MODE = wholemodule; + VALIDATE_PRODUCT = YES; + }; + name = Release; + }; + E1000B03000000000000A001 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + ENABLE_PREVIEWS = YES; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_NSMicrophoneUsageDescription = "The demo uses the microphone for voice conversations."; + INFOPLIST_KEY_UIApplicationSceneManifest_Generation = YES; + INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; + INFOPLIST_KEY_UILaunchScreen_Generation = YES; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = io.elevenlabs.example.DemoApp; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; + E1000B04000000000000A001 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + ENABLE_PREVIEWS = YES; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_NSMicrophoneUsageDescription = "The demo uses the microphone for voice conversations."; + INFOPLIST_KEY_UIApplicationSceneManifest_Generation = YES; + INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; + INFOPLIST_KEY_UILaunchScreen_Generation = YES; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = io.elevenlabs.example.DemoApp; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + E1000901000000000000A001 /* Build configuration list for PBXNativeTarget "DemoApp" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + E1000B03000000000000A001 /* Debug */, + E1000B04000000000000A001 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + E1000902000000000000A001 /* Build configuration list for PBXProject "DemoApp" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + E1000B01000000000000A001 /* Debug */, + E1000B02000000000000A001 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + +/* Begin XCLocalSwiftPackageReference section */ + E1000A01000000000000A001 /* XCLocalSwiftPackageReference "../.." */ = { + isa = XCLocalSwiftPackageReference; + relativePath = "../.."; + }; +/* End XCLocalSwiftPackageReference section */ + +/* Begin XCSwiftPackageProductDependency section */ + E1000201000000000000A001 /* ElevenLabs */ = { + isa = XCSwiftPackageProductDependency; + productName = ElevenLabs; + }; + E1000202000000000000A001 /* ElevenLabsWidget */ = { + isa = XCSwiftPackageProductDependency; + productName = ElevenLabsWidget; + }; +/* End XCSwiftPackageProductDependency section */ + }; + rootObject = E1000001000000000000A001 /* Project object */; +} diff --git a/Examples/DemoApp/DemoApp/Assets.xcassets/AccentColor.colorset/Contents.json b/Examples/DemoApp/DemoApp/Assets.xcassets/AccentColor.colorset/Contents.json new file mode 100644 index 00000000..eb878970 --- /dev/null +++ b/Examples/DemoApp/DemoApp/Assets.xcassets/AccentColor.colorset/Contents.json @@ -0,0 +1,11 @@ +{ + "colors" : [ + { + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/Examples/DemoApp/DemoApp/Assets.xcassets/AppIcon.appiconset/Contents.json b/Examples/DemoApp/DemoApp/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 00000000..23058801 --- /dev/null +++ b/Examples/DemoApp/DemoApp/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1,35 @@ +{ + "images" : [ + { + "idiom" : "universal", + "platform" : "ios", + "size" : "1024x1024" + }, + { + "appearances" : [ + { + "appearance" : "luminosity", + "value" : "dark" + } + ], + "idiom" : "universal", + "platform" : "ios", + "size" : "1024x1024" + }, + { + "appearances" : [ + { + "appearance" : "luminosity", + "value" : "tinted" + } + ], + "idiom" : "universal", + "platform" : "ios", + "size" : "1024x1024" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/Examples/DemoApp/DemoApp/Assets.xcassets/Contents.json b/Examples/DemoApp/DemoApp/Assets.xcassets/Contents.json new file mode 100644 index 00000000..73c00596 --- /dev/null +++ b/Examples/DemoApp/DemoApp/Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/Examples/DemoApp/DemoApp/ContentView.swift b/Examples/DemoApp/DemoApp/ContentView.swift new file mode 100644 index 00000000..3eb6f1c2 --- /dev/null +++ b/Examples/DemoApp/DemoApp/ContentView.swift @@ -0,0 +1,95 @@ +import ElevenLabs +import ElevenLabsWidget +import SwiftUI + +/// Minimal host for `ChatWidget`. Paste a public agent ID below to try it. +struct ContentView: View { + /// Paste a public agent ID here, then rebuild. + private static let agentId = "" + + @StateObject private var chat = ChatWidgetController() + @State private var conversationMode: WidgetConversationMode = .voiceAndText + + private var hasAgentId: Bool { + !Self.agentId.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + } + + var body: some View { + ZStack { + VStack(alignment: .leading, spacing: 16) { + Text("ElevenLabs Agents") + .font(.title) + + Text( + """ + This is a demo app for ElevenLabs Agents. You can either \ + import the widget into your app, or import the core SDK \ + and implement your own UI. + """ + ) + .foregroundStyle(.secondary) + + if hasAgentId { + Text("Agent ID is set — open the chat button to start.") + .foregroundStyle(.secondary) + hostControls + } else { + Text( + """ + Set `ContentView.agentId` to a public agent ID, then \ + rebuild to try the widget. + """ + ) + .foregroundStyle(.orange) + } + + Spacer() + } + .padding() + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .topLeading) + + if hasAgentId { + ChatWidget( + authProvider: { .publicAgent(id: Self.agentId) }, + widgetConfig: ChatWidgetConfig(conversationMode: conversationMode), + controller: chat + ) + } + } + } + + private var hostControls: some View { + VStack(alignment: .leading, spacing: 8) { + Text(stateDescription) + .font(.caption) + .foregroundStyle(.secondary) + + Picker("Mode", selection: $conversationMode) { + Text("Voice + text").tag(WidgetConversationMode.voiceAndText) + Text("Voice only").tag(WidgetConversationMode.voiceOnly) + Text("Text only").tag(WidgetConversationMode.textOnly) + } + .pickerStyle(.segmented) + + HStack(spacing: 12) { + Button("Open chat") { chat.open() } + Button("Close chat") { chat.close() } + Button("End call") { + Task { await chat.endConversation() } + } + .disabled(!chat.state.isConnected) + } + .buttonStyle(.bordered) + } + } + + private var stateDescription: String { + switch chat.state { + case .idle: "Idle" + case .connecting: "Connecting…" + case .connected: "Connected" + case .ended: "Ended" + case .error: "Error" + } + } +} diff --git a/Examples/DemoApp/DemoApp/DemoAppApp.swift b/Examples/DemoApp/DemoApp/DemoAppApp.swift new file mode 100644 index 00000000..24add047 --- /dev/null +++ b/Examples/DemoApp/DemoApp/DemoAppApp.swift @@ -0,0 +1,10 @@ +import SwiftUI + +@main +struct DemoAppApp: App { + var body: some Scene { + WindowGroup { + ContentView() + } + } +} diff --git a/Package.swift b/Package.swift index 04022724..37fc063d 100644 --- a/Package.swift +++ b/Package.swift @@ -16,6 +16,10 @@ let package = Package( .library( name: "ElevenLabs", targets: ["ElevenLabs"] + ), + .library( + name: "ElevenLabsWidget", + targets: ["ElevenLabsWidget"] ) ], dependencies: [ @@ -37,6 +41,21 @@ let package = Package( .process("PrivacyInfo.xcprivacy") ] ), + .target( + name: "ElevenLabsWidget", + dependencies: [ + "ElevenLabs" + ], + resources: [ + .process("Resources/OrbShader.metal") + ] + ), + .testTarget( + name: "ElevenLabsWidgetTests", + dependencies: [ + "ElevenLabsWidget" + ] + ), .testTarget( name: "ElevenLabsTests", dependencies: [ diff --git a/Package@swift-6.0.swift b/Package@swift-6.0.swift index 876816fc..cabc3a04 100644 --- a/Package@swift-6.0.swift +++ b/Package@swift-6.0.swift @@ -16,6 +16,10 @@ let package = Package( .library( name: "ElevenLabs", targets: ["ElevenLabs"] + ), + .library( + name: "ElevenLabsWidget", + targets: ["ElevenLabsWidget"] ) ], dependencies: [ @@ -37,6 +41,21 @@ let package = Package( .process("PrivacyInfo.xcprivacy") ] ), + .target( + name: "ElevenLabsWidget", + dependencies: [ + "ElevenLabs" + ], + resources: [ + .process("Resources/OrbShader.metal") + ] + ), + .testTarget( + name: "ElevenLabsWidgetTests", + dependencies: [ + "ElevenLabsWidget" + ] + ), .testTarget( name: "ElevenLabsTests", dependencies: [ diff --git a/Package@swift-6.2.swift b/Package@swift-6.2.swift index 8bc17aa5..db87f2c8 100644 --- a/Package@swift-6.2.swift +++ b/Package@swift-6.2.swift @@ -16,6 +16,10 @@ let package = Package( .library( name: "ElevenLabs", targets: ["ElevenLabs"] + ), + .library( + name: "ElevenLabsWidget", + targets: ["ElevenLabsWidget"] ) ], dependencies: [ @@ -37,6 +41,21 @@ let package = Package( .process("PrivacyInfo.xcprivacy") ] ), + .target( + name: "ElevenLabsWidget", + dependencies: [ + "ElevenLabs" + ], + resources: [ + .process("Resources/OrbShader.metal") + ] + ), + .testTarget( + name: "ElevenLabsWidgetTests", + dependencies: [ + "ElevenLabsWidget" + ] + ), .testTarget( name: "ElevenLabsTests", dependencies: [ diff --git a/README.md b/README.md index 245c4790..c8c4c7cc 100644 --- a/README.md +++ b/README.md @@ -122,15 +122,16 @@ private func handleToolCall(_ toolCall: ClientToolCallEvent) async { // Send the tool result back to the agent try await conversation?.sendToolResult( - for: toolCall.toolCallId, - result: result + .init(toolCallId: toolCall.toolCallId, result: result) ) } catch { // Handle tool execution errors try? await conversation?.sendToolResult( - for: toolCall.toolCallId, - result: ["error": error.localizedDescription], - isError: true + .init( + toolCallId: toolCall.toolCallId, + result: ["error": error.localizedDescription], + isError: true + ) ) } } @@ -206,7 +207,9 @@ Task { let result = await myAppAction(params) // 3. Send result back to the agent - try? await conversation.sendToolResult(for: call.toolCallId, result: result) + try? await conversation.sendToolResult( + .init(toolCallId: call.toolCallId, result: result) + ) } } } @@ -219,13 +222,21 @@ Task { ## Configuration & Tuning -### Global Setup +### Logging -Configure logging and global behaviors at app launch: +Set verbosity per client: ```swift -ElevenLabs.configure( - ElevenLabs.Configuration(logLevel: .info) +let client = ConversationClient(logLevel: .info) +``` + +### Custom Endpoints + +Point the HTTP API base or either WebSocket endpoint at a proxy, regional host, or staging: + +```swift +let config = ConversationConfig( + endpoints: Endpoints(apiBase: URL(string: "https://my-proxy.example.com")!) ) ``` diff --git a/Scripts/generate-version.sh b/Scripts/generate-version.sh index 61382c1a..988a56bd 100755 --- a/Scripts/generate-version.sh +++ b/Scripts/generate-version.sh @@ -12,7 +12,7 @@ VERSION=$(git describe --tags --abbrev=0 2>/dev/null || echo "0.0.0") VERSION=${VERSION#v} # Path to the Version.swift file -VERSION_FILE="Sources/ElevenLabs/Internal/Version.swift" +VERSION_FILE="Sources/ElevenLabs/Public/Version.swift" # Generate the Version.swift file cat > "$VERSION_FILE" << EOF @@ -20,9 +20,7 @@ cat > "$VERSION_FILE" << EOF // Run Scripts/generate-version.sh to update import Foundation -enum SDKVersion { - static let version = "$VERSION" -} +public let version = "$VERSION" EOF echo "Generated Version.swift with version: $VERSION" diff --git a/Sources/ElevenLabs/Internal/Audio/AudioObserverRegistry.swift b/Sources/ElevenLabs/Internal/Audio/AudioObserverRegistry.swift new file mode 100644 index 00000000..5d9ba00e --- /dev/null +++ b/Sources/ElevenLabs/Internal/Audio/AudioObserverRegistry.swift @@ -0,0 +1,75 @@ +@preconcurrency import AVFoundation +import LiveKit + +/// Bridges a user-supplied ``ConversationAudioObserver`` to LiveKit's +/// `AudioRenderer`, keeping the LiveKit type fully internal. Holds the user +/// observer alive for as long as it is registered. +private final class LiveKitAudioRendererAdapter: AudioRenderer, @unchecked Sendable { + private let observer: any ConversationAudioObserver + + init(_ observer: any ConversationAudioObserver) { + self.observer = observer + } + + func render(pcmBuffer: AVAudioPCMBuffer) { + observer.didReceive(pcmBuffer) + } +} + +/// Tracks externally registered audio observers for a single stream (agent +/// output or mic input) and keeps them attached across track swaps. +/// +/// All mutating entry points are `@MainActor`; adapters forward buffers on the +/// audio callback path without touching this registry's state. +@MainActor +final class AudioObserverRegistry { + private var adapters: [ObjectIdentifier: LiveKitAudioRendererAdapter] = [:] + private weak var attachedTrack: (any AudioTrackProtocol)? + + /// Number of currently registered observers. + var registeredCount: Int { + adapters.count + } + + /// Register an observer, attaching it to the current track if one is present. + /// Re-adding the same observer instance is a no-op. + func add(_ observer: any ConversationAudioObserver) { + let key = ObjectIdentifier(observer) + guard adapters[key] == nil else { return } + let adapter = LiveKitAudioRendererAdapter(observer) + adapters[key] = adapter + attachedTrack?.add(audioRenderer: adapter) + } + + /// Unregister an observer, detaching it from the current track. + func remove(_ observer: any ConversationAudioObserver) { + let key = ObjectIdentifier(observer) + guard let adapter = adapters.removeValue(forKey: key) else { return } + attachedTrack?.remove(audioRenderer: adapter) + } + + /// Point the registry at a (possibly new or `nil`) track. Detaches every + /// adapter from the previous track and attaches them to the new one, so + /// registered observers survive track swaps. Idempotent for the same track. + func attach(to track: (any AudioTrackProtocol)?) { + guard track !== attachedTrack else { return } + if let attachedTrack { + for adapter in adapters.values { + attachedTrack.remove(audioRenderer: adapter) + } + } + attachedTrack = track + if let track { + for adapter in adapters.values { + track.add(audioRenderer: adapter) + } + } + } + + /// Detach from any track and drop all registrations. Used when a single-use + /// session ends; durable re-registration is owned by `ConversationClient`. + func reset() { + attach(to: nil) + adapters.removeAll() + } +} diff --git a/Sources/ElevenLabs/Internal/Utilities/SoftwareMuteProcessor.swift b/Sources/ElevenLabs/Internal/Audio/SoftwareMuteProcessor.swift similarity index 100% rename from Sources/ElevenLabs/Internal/Utilities/SoftwareMuteProcessor.swift rename to Sources/ElevenLabs/Internal/Audio/SoftwareMuteProcessor.swift diff --git a/Sources/ElevenLabs/Internal/Authorization/ConnectionConstants.swift b/Sources/ElevenLabs/Internal/Authorization/ConnectionConstants.swift deleted file mode 100644 index f15c5adb..00000000 --- a/Sources/ElevenLabs/Internal/Authorization/ConnectionConstants.swift +++ /dev/null @@ -1,9 +0,0 @@ -import Foundation - -enum ConnectionConstants { - /// LiveKit signaling endpoint used for voice conversations. - static let voiceConversationUrl = "wss://livekit.rtc.elevenlabs.io" - /// WebSocket endpoint used for text-only conversations. - static let textConversationUrl = "wss://api.elevenlabs.io/v1/convai/conversation" - static let tokenUrl = "https://api.elevenlabs.io/v1/convai/conversation/token" -} diff --git a/Sources/ElevenLabs/Public/Authorization/TokenService.swift b/Sources/ElevenLabs/Internal/Authorization/TokenService.swift similarity index 60% rename from Sources/ElevenLabs/Public/Authorization/TokenService.swift rename to Sources/ElevenLabs/Internal/Authorization/TokenService.swift index 106d0066..0df51a8c 100644 --- a/Sources/ElevenLabs/Public/Authorization/TokenService.swift +++ b/Sources/ElevenLabs/Internal/Authorization/TokenService.swift @@ -16,85 +16,55 @@ import Foundation /// Service for managing ElevenLabs authentication /// This is designed to be stateless and SDK-friendly -public struct TokenService: Sendable { - public struct ConnectionDetails: Codable, Sendable { - public let serverUrl: String - public let roomName: String - public let participantName: String - public let participantToken: String - } - - /// Optional configuration for advanced use cases - public struct Configuration: Sendable { - /// Custom API endpoint (for testing or enterprise deployments) - public let apiEndpoint: String? - /// Custom WebSocket URL (for testing or enterprise deployments) - public let websocketURL: String? - - public init(apiEndpoint: String? = nil, websocketURL: String? = nil) { - self.apiEndpoint = apiEndpoint - self.websocketURL = websocketURL - } - - public static let `default` = Configuration() - } - - private let configuration: Configuration +struct TokenService: Sendable { + private let endpoints: Endpoints private let urlSession: URLSession // Development-only API key for testing private agents // This should only be set in debug builds for local testing #if DEBUG - public let debugApiKey: String? + let debugApiKey: String? - public init( - configuration: Configuration = .default, + init( + endpoints: Endpoints = .production, urlSession: URLSession = .shared, debugApiKey: String? = nil ) { - self.configuration = configuration + self.endpoints = endpoints self.urlSession = urlSession self.debugApiKey = debugApiKey } #else - public init( - configuration: Configuration = .default, + init( + endpoints: Endpoints = .production, urlSession: URLSession = .shared ) { - self.configuration = configuration + self.endpoints = endpoints self.urlSession = urlSession } #endif - /// Fetch connection details for ElevenLabs conversation. + /// Resolve the token a voice conversation authenticates with. /// /// Translates internal `TokenError`s into public `ConversationError`s so /// callers only ever deal with one error type. - public func fetchConnectionDetails(configuration: ConversationCredentials) async throws -> ConnectionDetails { + func fetchToken(for credentials: ConversationCredentials) async throws -> String { do { - let token: String = switch configuration.authSource { + switch credentials.authSource { case let .publicAgentId(agentId): - try await fetchTokenFromAPI(agentId: agentId, environment: configuration.environment) + return try await fetchTokenFromAPI( + agentId: agentId, + environment: credentials.environment + ) case let .conversationToken(conversationToken): - conversationToken + return conversationToken case .signedWebSocketURL: throw ConversationError.authenticationFailed( "Signed WebSocket URLs are only supported for text-only conversations." ) case let .customTokenProvider(provider): - try await provider() + return try await provider() } - - let websocketURL = self.configuration.websocketURL ?? ConnectionConstants.voiceConversationUrl - - // ElevenLabs tokens contain room name and participant identity in the JWT - // LiveKit will extract these automatically, so we provide empty values - return ConnectionDetails( - serverUrl: websocketURL, - roomName: "", // LiveKit extracts from JWT - participantName: "", // LiveKit extracts from JWT - participantToken: token - ) } catch let error as ConversationError { throw error } catch let error as TokenError { @@ -104,17 +74,21 @@ public struct TokenService: Sendable { } } - private func fetchTokenFromAPI(agentId: String, environment: String? = nil) async throws -> String { - // Build URL with agent ID as query parameter - let apiUrl = configuration.apiEndpoint ?? ConnectionConstants.tokenUrl - - guard var components = URLComponents(string: apiUrl) else { + private func fetchTokenFromAPI( + agentId: String, + environment: String? = nil + ) async throws -> String { + guard var components = URLComponents( + url: endpoints.conversationToken, + resolvingAgainstBaseURL: false + ) else { throw TokenError.invalidURL } - var queryItems = [ + var queryItems = components.queryItems ?? [] + queryItems += [ URLQueryItem(name: "agent_id", value: agentId), URLQueryItem(name: "source", value: "swift_sdk"), - URLQueryItem(name: "version", value: SDKVersion.version) + URLQueryItem(name: "version", value: version) ] if let environment { queryItems.append(URLQueryItem(name: "environment", value: environment)) diff --git a/Sources/ElevenLabs/Internal/Authorization/TokenServicing.swift b/Sources/ElevenLabs/Internal/Authorization/TokenServicing.swift index b0a64778..5f6c21dd 100644 --- a/Sources/ElevenLabs/Internal/Authorization/TokenServicing.swift +++ b/Sources/ElevenLabs/Internal/Authorization/TokenServicing.swift @@ -1,10 +1,9 @@ import Foundation protocol TokenServicing: Sendable { - /// Fetch connection details for ElevenLabs conversation - /// - Parameter configuration: The configuration to use for fetching connection details - /// - Returns: The connection details for the ElevenLabs conversation - func fetchConnectionDetails(configuration: ConversationCredentials) async throws -> TokenService.ConnectionDetails + /// Resolve the token a voice conversation authenticates with. + /// - Parameter credentials: The credentials to authenticate with + func fetchToken(for credentials: ConversationCredentials) async throws -> String } extension TokenService: TokenServicing {} diff --git a/Sources/ElevenLabs/Internal/Conversation/AgentStateManager.swift b/Sources/ElevenLabs/Internal/Conversation/AgentStateManager.swift index 5b3595c0..14a1ddc5 100644 --- a/Sources/ElevenLabs/Internal/Conversation/AgentStateManager.swift +++ b/Sources/ElevenLabs/Internal/Conversation/AgentStateManager.swift @@ -15,8 +15,8 @@ private typealias TimerTask = Task @MainActor final class AgentStateManager { - private(set) var currentState: ElevenLabs.AgentState = .listening - var onStateChange: ((ElevenLabs.AgentState) -> Void)? + private(set) var currentState: AgentState = .listening + var onStateChange: ((AgentState) -> Void)? private let configuration: AgentStateConfiguration @@ -90,7 +90,7 @@ final class AgentStateManager { } } - private func transitionTo(_ newState: ElevenLabs.AgentState) { + private func transitionTo(_ newState: AgentState) { guard newState != currentState else { return } currentState = newState onStateChange?(newState) diff --git a/Sources/ElevenLabs/Internal/Conversation/Conversation+Events.swift b/Sources/ElevenLabs/Internal/Conversation/Conversation+Events.swift deleted file mode 100644 index fff2ad53..00000000 --- a/Sources/ElevenLabs/Internal/Conversation/Conversation+Events.swift +++ /dev/null @@ -1,139 +0,0 @@ -import Foundation - -@MainActor -extension Conversation { - // MARK: - Event Handling - - // swiftlint:disable:next cyclomatic_complexity function_body_length - func handleIncomingEvent(_ event: IncomingEvent) async { - switch event { - case let .userTranscript(e): - insertUserTranscript(content: e.transcript, eventId: e.eventId) - agentStateManager?.processSignal(.userTranscript) - callbacks.onUserTranscript?(e.transcript, e.eventId) - - case let .agentResponse(e): - upsertAgentMessage(content: e.response, eventId: e.eventId) - lastAgentEventId = e.eventId - agentStateManager?.processSignal(.agentResponse) - callbacks.onAgentResponse?(e.response, e.eventId) - if lastFeedbackSubmittedEventId.map({ e.eventId > $0 }) ?? true { - callbacks.onCanSendFeedbackChange?(true) - } - - case let .agentResponseCorrection(correction): - upsertAgentMessage(content: correction.correctedAgentResponse, eventId: correction.eventId) - callbacks.onAgentResponseCorrection?( - correction.originalAgentResponse, - correction.correctedAgentResponse, - correction.eventId - ) - - case let .agentResponseMetadata(metadata): - callbacks.onAgentResponseMetadata?( - metadata.metadataData, - metadata.eventId - ) - - case let .agentChatResponsePart(e): - let existing = messages.last(where: { $0.role == .agent && $0.eventId == e.eventId })?.content ?? "" - upsertAgentMessage(content: existing + e.text, eventId: e.eventId) - - case let .audio(audioEvent): - latestAudioEvent = audioEvent - latestAudioAlignment = audioEvent.alignment - if let alignment = audioEvent.alignment { - callbacks.onAudioAlignment?(alignment) - } - - case let .interruption(interruptionEvent): - speakingTimer?.cancel() - applyStateSignal(.interruption, fallback: .listening) - callbacks.onInterruption?(interruptionEvent.eventId) - callbacks.onCanSendFeedbackChange?(false) - - case let .conversationMetadata(metadata): - // Store the conversation metadata for public access - conversationMetadata = metadata - callbacks.onConversationMetadata?(metadata) - - case let .ping(p): - // Respond to ping with pong - let pong = OutgoingEvent.pong(PongEvent(eventId: p.eventId)) - try? await publish(pong) - - case let .clientToolCall(toolCall): - // Add to pending tool calls for the app to handle - callbacks.onUnhandledClientToolCall?(toolCall) - pendingToolCalls.append(toolCall) - - case let .vadScore(vad): - agentStateManager?.processSignal(.vadScore(vad.vadScore)) - callbacks.onVadScore?(vad.vadScore) - - case let .agentToolResponse(toolResponse): - applyStateSignal(.agentToolResponse, fallback: .listening) - - if toolResponse.toolName == "end_call" { - await endConversation() - } - callbacks.onAgentToolResponse?(toolResponse) - - case let .agentToolRequest(toolRequest): - applyStateSignal(.agentToolRequest, fallback: .thinking) - callbacks.onAgentToolRequest?(toolRequest) - - case .tentativeUserTranscript: - // Tentative user transcript (in-progress transcription) - break - - case let .mcpToolCall(toolCall): - // Update or append MCP tool call based on toolCallId - if let index = mcpToolCalls.firstIndex(where: { $0.toolCallId == toolCall.toolCallId }) { - mcpToolCalls[index] = toolCall - } else { - mcpToolCalls.append(toolCall) - } - - case let .mcpConnectionStatus(status): - // Update MCP connection status - mcpConnectionStatus = status - - case let .error(errorEvent): - logger.error("Received error event from server: code=\(errorEvent.code), message=\(errorEvent.message ?? "none")") - callbacks.onError?(.serverError(errorEvent)) - } - } - - /// Inserts the user transcript before the agent message with the same `eventId` - /// if one exists, since the agent's response may be received before the transcript. - private func insertUserTranscript(content: String, eventId: Int) { - let message = Message( - id: UUID().uuidString, - role: .user, - content: content, - timestamp: Date(), - eventId: eventId - ) - if let agentIdx = messages.firstIndex(where: { $0.role == .agent && $0.eventId == eventId }) { - messages.insert(message, at: agentIdx) - } else { - messages.append(message) - } - } - - private func upsertAgentMessage(content: String, eventId: Int) { - if let idx = messages.lastIndex(where: { $0.role == .agent && $0.eventId == eventId }) { - let existing = messages[idx] - messages[idx] = Message( - id: existing.id, - role: .agent, - content: content, - timestamp: existing.timestamp, - eventId: eventId - ) - } else { - appendMessage(role: .agent, content: content, eventId: eventId) - } - } -} diff --git a/Sources/ElevenLabs/Internal/Conversation/Conversation.swift b/Sources/ElevenLabs/Internal/Conversation/Conversation.swift new file mode 100644 index 00000000..5843d741 --- /dev/null +++ b/Sources/ElevenLabs/Internal/Conversation/Conversation.swift @@ -0,0 +1,606 @@ +import Combine +import Foundation +import LiveKit + +// swiftlint:disable file_length type_body_length + +/// A single-use conversation session, created by and owned by `ConversationClient`. +/// +/// Manages the lifecycle of one conversation: network layer +/// (`WebRTCConnectionManager`|`WebSocketConnectionManager`), protocol parser +/// (`EventParser`), and audio device setup. +@MainActor +final class Conversation: ObservableObject { + // MARK: - State + + @Published var state: ConversationState = .idle + @Published var messages: [Message] = [] + @Published var agentState: AgentState = .listening + + /// Stream of client tool calls that need to be executed by the app + @Published var pendingToolCalls: [ClientToolCallEvent] = [] + + /// Conversation metadata including conversation ID, received when the conversation is initialized + @Published var conversationMetadata: ConversationMetadataEvent? + + /// MCP tool calls from the agent + @Published var mcpToolCalls: [MCPToolCallEvent] = [] + + /// Current MCP connection status for all integrations + @Published var mcpConnectionStatus: MCPConnectionStatusEvent? + + /// Pending mute state to apply after connection completes. + /// Allows setting mute state during connection phase. + private var pendingMuteState: Bool? + + /// Audio device management + private var audioManager: ConversationAudioManager? + + /// Externally registered audio observers. Kept attached across track swaps. + let agentObserverRegistry = AudioObserverRegistry() + let micObserverRegistry = AudioObserverRegistry() + + /// Agent state manager for event-based state tracking + var agentStateManager: AgentStateManager? + + /// Forward a signal to the event-based state manager, or fall back to directly setting `agentState`. + func applyStateSignal(_ signal: AgentStateSignal, fallback: AgentState) { + if let manager = agentStateManager { + manager.processSignal(signal) + } else { + agentState = fallback + } + } + + func handleRemoteSpeakingUpdate(isSpeaking: Bool) { + if let manager = agentStateManager { + manager.processSignal(isSpeaking ? .agentStartedSpeaking : .agentStoppedSpeaking) + } else if isSpeaking { + speakingTimer?.cancel() + agentState = .speaking + } else { + scheduleBackToListening(delay: 1.0) + } + } + + /// Internal logger, accessible from nonisolated contexts. + nonisolated let logger: any Logging + + /// Context for logging (e.g. agentId) + private var activeContext: [String: String]? + + /// Internal LiveKit tracks used to attach ``ConversationAudioObserver``s. + var inputTrack: (any AudioTrackProtocol)? { + activeWebRTCConnectionManager?.inputTrack + } + + var agentAudioTrack: (any AudioTrackProtocol)? { + activeWebRTCConnectionManager?.agentAudioTrack + } + + // MARK: - Init + + init( + dependencyProvider: any ConversationDependencyProvider, + config: ConversationConfig = .init(), + callbacks: ConversationCallbacks = .init(), + initialMicMuted: Bool = false + ) { + self.dependencyProvider = dependencyProvider + self.config = config + self.callbacks = callbacks + pendingMuteState = initialMicMuted + logger = dependencyProvider.logger + setupAudioManager() + } + + private func setupAudioManager() { + guard !config.conversationOverrides.textOnly else { return } + audioManager = ConversationAudioManager(logger: logger) + } + + private func setupAgentStateManager() { + guard let configuration = config.agentStateConfiguration else { return } + let manager = AgentStateManager(configuration: configuration) + manager.onStateChange = { [weak self] state in + self?.agentState = state + self?.callbacks.onAgentStateChange?(state) + } + agentStateManager = manager + } + + // MARK: - API + + /// Start a conversation using authentication configuration. + func start(auth: ConversationCredentials) async throws -> ConversationStartResult { + guard state == .idle else { + throw ConversationError.alreadyStarted + } + + let result: ConversationStartResult = if config.conversationOverrides.textOnly { + try await startTextOnlyConversation(auth: auth) + } else { + try await startVoiceConversation(auth: auth) + } + + state = .connected(result.callInfo) + callbacks.onAgentReady?() + return result + } + + private func startVoiceConversation( + auth: ConversationCredentials + ) async throws -> ConversationStartResult { + let webRTCConnectionManager = dependencyProvider.webRTCConnectionManager + prepareConversationStart( + auth: auth, + connectionManager: webRTCConnectionManager + ) + + webRTCConnectionManager.onRemoteSpeakingChanged = { [weak self] isSpeaking in + Task { @MainActor in + self?.handleRemoteSpeakingUpdate(isSpeaking: isSpeaking) + } + } + webRTCConnectionManager.onTracksChanged = { [weak self] in + Task { @MainActor in + self?.refreshAudioObservers() + } + } + + await audioManager?.configure(with: config, callbacks: callbacks) + if let pendingMuteState { + audioManager?.softwareMuteProcessor?.setMuted(pendingMuteState) + } + + let result: ConversationStartResult + do { + result = try await webRTCConnectionManager.connect( + auth: auth, + config: config, + onStartupStateChange: { [weak self] stage in + self?.updateStartupStage(stage) + } + ) + } catch let error as ConversationError { + await handleStartupFailure(error, disconnecting: webRTCConnectionManager) + throw error + } catch is CancellationError { + await handleStartupCancellation(disconnecting: webRTCConnectionManager) + throw CancellationError() + } + + if let pendingMute = pendingMuteState { + pendingMuteState = nil + do { + if let softwareMuteProcessor = audioManager?.softwareMuteProcessor { + softwareMuteProcessor.setMuted(pendingMute) + } else { + try await webRTCConnectionManager.setMicrophoneMuted(pendingMute) + } + } catch { + logger.warning("Failed to apply pending mute state", context: ["error": "\(error)"]) + } + } + + refreshAudioObservers() + return result + } + + // MARK: - Audio observers + + /// Register an observer for the agent's decoded output audio. + func addAgentAudioObserver(_ observer: any ConversationAudioObserver) { + guard !isTearingDown else { return } + agentObserverRegistry.add(observer) + } + + /// Unregister a previously added agent audio observer. + func removeAgentAudioObserver(_ observer: any ConversationAudioObserver) { + agentObserverRegistry.remove(observer) + } + + /// Register an observer for the local microphone input audio. + func addMicAudioObserver(_ observer: any ConversationAudioObserver) { + guard !isTearingDown else { return } + micObserverRegistry.add(observer) + } + + /// Unregister a previously added mic audio observer. + func removeMicAudioObserver(_ observer: any ConversationAudioObserver) { + micObserverRegistry.remove(observer) + } + + /// Reconcile registered observers with the currently available tracks. + func refreshAudioObservers() { + guard !isTearingDown else { return } + agentObserverRegistry.attach(to: agentAudioTrack) + micObserverRegistry.attach(to: inputTrack) + } + + private func startTextOnlyConversation( + auth: ConversationCredentials + ) async throws -> ConversationStartResult { + let connectionManager = dependencyProvider.webSocketConnectionManager + prepareConversationStart( + auth: auth, + connectionManager: connectionManager + ) + + do { + return try await connectionManager.connect( + auth: auth, + config: config, + onStartupStateChange: { [weak self] stage in + self?.updateStartupStage(stage) + } + ) + } catch let error as ConversationError { + await handleStartupFailure(error, disconnecting: connectionManager) + throw error + } catch is CancellationError { + await handleStartupCancellation(disconnecting: connectionManager) + throw CancellationError() + } + } + + /// End and clean up. + /// Can be called during connection phase to cancel, or during connected conversation to end. + func endConversation(reason: EndReason = .userEnded) async { + if state == .idle { + state = .ended(reason: reason) + tearDownActiveSession() + return + } + + guard state.isConnected || state.isConnecting, + let connectionManager = activeConnectionManager + else { return } + state = .ended(reason: reason) + + tearDownActiveSession() + await connectionManager.disconnect() + + callbacks.onDisconnect?(reason) + } + + /// Send a text message to the agent. + func sendMessage(_ text: String) async throws { + guard state.isConnected else { + throw ConversationError.notConnected + } + let event = OutgoingEvent.userMessage(UserMessageEvent(text: text)) + try await publish(event) + appendMessage(role: .user, content: text) + } + + /// Mute or unmute the local microphone. + func setMicMuted(_ muted: Bool) async throws { + if let softwareMuteProcessor = audioManager?.softwareMuteProcessor { + softwareMuteProcessor.setMuted(muted) + if state.isConnecting { + pendingMuteState = muted + } + return + } + try await setHardwareMicMuted(muted) + } + + func setHardwareMicMuted(_ muted: Bool) async throws { + if state.isConnected { + guard let webRTCConnectionManager = activeWebRTCConnectionManager else { + throw ConversationError.notConnected + } + do { + try await webRTCConnectionManager.setMicrophoneMuted(muted) + pendingMuteState = nil + } catch WebRTCConnectionManagerError.roomUnavailable { + throw ConversationError.notConnected + } catch { + throw ConversationError.microphoneToggleFailed(error) + } + } else if state == .idle || state.isConnecting { + pendingMuteState = muted + } + } + + /// Interrupt the agent while speaking. + func interruptAgent() async throws { + guard state.isConnected else { throw ConversationError.notConnected } + let event = OutgoingEvent.userActivity + try await publish(event) + } + + /// Contextual update to agent (system prompt-ish). + func updateContext(_ context: String) async throws { + guard state.isConnected else { throw ConversationError.notConnected } + let event = OutgoingEvent.contextualUpdate(ContextualUpdateEvent(text: context)) + try await publish(event) + } + + /// Send feedback (like/dislike) for an event/message id. + func sendFeedback(_ score: FeedbackEvent.Score, eventId: Int) async throws { + guard state.isConnected else { + throw ConversationError.notConnected + } + + let event = OutgoingEvent.feedback(FeedbackEvent(score: score, eventId: eventId)) + try await publish(event) + } + + /// Approve or reject an MCP tool call request from the agent. + /// - Parameters: + /// - toolCallId: The tool call identifier from `MCPToolCallEvent`. + /// - isApproved: Pass `true` to approve, `false` to reject. + func sendMCPToolApproval(toolCallId: String, isApproved: Bool) async throws { + guard state.isConnected else { throw ConversationError.notConnected } + let approval = MCPToolApprovalResultEvent(toolCallId: toolCallId, isApproved: isApproved) + try await publish(.mcpToolApprovalResult(approval)) + } + + /// Send the result of a client tool call back to the agent. + func sendToolResult(_ result: ClientToolResultEvent) async throws { + guard state.isConnected else { throw ConversationError.notConnected } + try await publish(.clientToolResult(result)) + pendingToolCalls.removeAll { $0.toolCallId == result.toolCallId } + } + + /// Mark a tool call as completed without sending a result (for tools that don't expect responses). + func markToolCallCompleted(_ toolCallId: String) { + pendingToolCalls.removeAll { $0.toolCallId == toolCallId } + } + + // MARK: - Private + + private let dependencyProvider: any ConversationDependencyProvider + private var activeConnectionManager: (any ConnectionManaging)? + private var activeWebRTCConnectionManager: (any WebRTCConnectionManaging)? { + activeConnectionManager as? any WebRTCConnectionManaging + } + + let config: ConversationConfig + let callbacks: ConversationCallbacks + + var speakingTimer: Task? + private var isTearingDown = false + + private func updateStartupStage(_ stage: ConversationStartupState) { + guard state.isConnecting, state != .connecting(stage) else { return } + state = .connecting(stage) + } + + /// Common preparation shared by voice and text-only startup paths. + private func prepareConversationStart( + auth: ConversationCredentials, + connectionManager: any ConnectionManaging + ) { + state = .connecting(.preparing) + activeConnectionManager = connectionManager + + activeContext = ["agentId": auth.agentId] + let mode = config.conversationOverrides.textOnly ? "text-only" : "voice" + logger.info("Starting \(mode) conversation", context: activeContext) + + setupAgentStateManager() + + connectionManager.onEventReceived = { [weak self, weak connectionManager] event in + Task { @MainActor [weak self, weak connectionManager] in + guard let self, + let connectionManager, + activeConnectionManager === connectionManager, + state.isConnecting || state.isConnected + else { + return + } + + await handleIncomingEvent(event) + } + } + connectionManager.onDisconnected = { [weak self] in + guard let self else { return } + await endConversation(reason: .remoteDisconnected) + } + } + + private func handleStartupFailure( + _ error: ConversationError, + disconnecting connectionManager: any ConnectionManaging + ) async { + cleanupTransientResources() + await connectionManager.disconnect() + + // End/supersede may have already moved us out of connecting; don't + // overwrite `.ended` or fire shared `onError` for a discarded session. + guard state.isConnecting else { return } + state = .error(error) + callbacks.onError?(error) + } + + private func handleStartupCancellation(disconnecting connectionManager: any ConnectionManaging) async { + guard state.isConnecting else { return } + state = .ended(reason: .userEnded) + tearDownActiveSession() + await connectionManager.disconnect() + } + + /// Tear down operational state when an active session ends. + /// Preserves user-visible display state (messages, MCP activity, conversation + /// metadata) so `ConversationClient` can keep the completed transcript visible. + private func tearDownActiveSession() { + cleanupTransientResources() + + pendingToolCalls.removeAll() + } + + private func cleanupTransientResources() { + isTearingDown = true + speakingTimer?.cancel() + speakingTimer = nil + pendingMuteState = nil + agentState = .listening + + agentObserverRegistry.reset() + micObserverRegistry.reset() + + audioManager?.cleanup() + agentStateManager = nil + } + + private func scheduleBackToListening(delay: TimeInterval = 0.5) { + speakingTimer?.cancel() + speakingTimer = Task { + do { + try await Task.sleep(nanoseconds: UInt64(delay * 1_000_000_000)) + self.agentState = .listening + } catch { + // Task was cancelled, do nothing + } + } + } + + func publish(_ event: OutgoingEvent) async throws { + guard let connectionManager = activeConnectionManager else { + throw ConversationError.notConnected + } + + try await connectionManager.send(event: event) + } + + // MARK: - Event Handling + + // swiftlint:disable:next cyclomatic_complexity function_body_length + func handleIncomingEvent(_ event: IncomingEvent) async { + switch event { + case let .userTranscript(e): + insertUserTranscript(content: e.transcript, eventId: e.eventId) + agentStateManager?.processSignal(.userTranscript) + callbacks.onUserTranscript?(e.transcript, e.eventId) + + case let .agentResponse(e): + upsertAgentMessage(content: e.response, eventId: e.eventId) + agentStateManager?.processSignal(.agentResponse) + callbacks.onAgentResponse?(e.response, e.eventId) + + case let .agentResponseCorrection(correction): + upsertAgentMessage(content: correction.correctedAgentResponse, eventId: correction.eventId) + callbacks.onAgentResponseCorrection?( + correction.originalAgentResponse, + correction.correctedAgentResponse, + correction.eventId + ) + + case let .agentResponseMetadata(metadata): + callbacks.onAgentResponseMetadata?( + metadata.metadataData, + metadata.eventId + ) + + case let .agentChatResponsePart(e): + let existing = messages.last(where: { $0.role == .agent && $0.eventId == e.eventId })?.content ?? "" + upsertAgentMessage(content: existing + e.text, eventId: e.eventId) + + case let .audio(audioEvent): + if let alignment = audioEvent.alignment { + callbacks.onAudioAlignment?(alignment) + } + + case let .interruption(interruptionEvent): + speakingTimer?.cancel() + applyStateSignal(.interruption, fallback: .listening) + callbacks.onInterruption?(interruptionEvent.eventId) + + case let .conversationMetadata(metadata): + conversationMetadata = metadata + callbacks.onConversationMetadata?(metadata) + + case let .ping(p): + let pong = OutgoingEvent.pong(PongEvent(eventId: p.eventId)) + try? await publish(pong) + + case let .clientToolCall(toolCall): + callbacks.onClientToolCall?(toolCall) + pendingToolCalls.append(toolCall) + + case let .vadScore(vad): + agentStateManager?.processSignal(.vadScore(vad.vadScore)) + callbacks.onVadScore?(vad.vadScore) + + case let .agentToolResponse(toolResponse): + applyStateSignal(.agentToolResponse, fallback: .listening) + + if toolResponse.toolName == "end_call" { + await endConversation() + } + callbacks.onAgentToolResponse?(toolResponse) + + case let .agentToolRequest(toolRequest): + applyStateSignal(.agentToolRequest, fallback: .thinking) + callbacks.onAgentToolRequest?(toolRequest) + + case .tentativeUserTranscript: + break + + case let .mcpToolCall(toolCall): + if let index = mcpToolCalls.firstIndex(where: { $0.toolCallId == toolCall.toolCallId }) { + mcpToolCalls[index] = toolCall + } else { + mcpToolCalls.append(toolCall) + } + + case let .mcpConnectionStatus(status): + mcpConnectionStatus = status + + case let .error(errorEvent): + logger.error("Received error event from server: code=\(errorEvent.code), message=\(errorEvent.message ?? "none")") + callbacks.onError?(.serverError(errorEvent)) + } + } + + // MARK: - Message Helpers + + func appendMessage(role: Message.Role, content: String, eventId: Int? = nil) { + messages.append( + Message( + id: UUID().uuidString, + role: role, + content: content, + timestamp: Date(), + eventId: eventId + ) + ) + } + + /// Inserts the user transcript before the agent message with the same `eventId` + /// if one exists, since the agent's response may be received before the transcript. + private func insertUserTranscript(content: String, eventId: Int) { + let message = Message( + id: UUID().uuidString, + role: .user, + content: content, + timestamp: Date(), + eventId: eventId + ) + if let agentIdx = messages.firstIndex(where: { $0.role == .agent && $0.eventId == eventId }) { + messages.insert(message, at: agentIdx) + } else { + messages.append(message) + } + } + + private func upsertAgentMessage(content: String, eventId: Int) { + if let idx = messages.lastIndex(where: { $0.role == .agent && $0.eventId == eventId }) { + let existing = messages[idx] + messages[idx] = Message( + id: existing.id, + role: .agent, + content: content, + timestamp: existing.timestamp, + eventId: eventId + ) + } else { + appendMessage(role: .agent, content: content, eventId: eventId) + } + } +} + +// swiftlint:enable file_length type_body_length diff --git a/Sources/ElevenLabs/Internal/Conversation/ConversationAudioManager.swift b/Sources/ElevenLabs/Internal/Conversation/ConversationAudioManager.swift index 6177d924..8aac9e4a 100644 --- a/Sources/ElevenLabs/Internal/Conversation/ConversationAudioManager.swift +++ b/Sources/ElevenLabs/Internal/Conversation/ConversationAudioManager.swift @@ -8,8 +8,6 @@ import LiveKit /// Encapsulates all AudioManager interactions to keep Conversation class focused on conversation logic. @MainActor final class ConversationAudioManager { - private(set) var audioDevices: [AudioDevice] = [] - private(set) var selectedAudioDeviceID: String = "" private(set) var softwareMuteProcessor: SoftwareMuteProcessor? private let audioManager = AudioManager.shared @@ -17,22 +15,12 @@ final class ConversationAudioManager { private var audioSpeechHandlerInstalled = false private let logger: any Logging - /// Callback when audio devices list changes - var onDevicesChanged: (([AudioDevice]) -> Void)? - - /// Callback when selected device changes - var onSelectedDeviceChanged: ((String) -> Void)? - init(logger: any Logging) { self.logger = logger - audioDevices = audioManager.inputDevices - selectedAudioDeviceID = audioManager.inputDevice.deviceId setupInitialConfiguration() } deinit { - // Reset callbacks directly since we can't call MainActor methods from deinit - audioManager.onDeviceUpdate = nil if audioSpeechHandlerInstalled { audioManager.onMutedSpeechActivity = previousSpeechActivityHandler } @@ -96,17 +84,6 @@ final class ConversationAudioManager { logger.warning("Failed to set recording always prepared mode", context: ["error": "\(error)"]) } } - - // Setup device change observer - audioManager.onDeviceUpdate = { [weak self] _ in - Task { @MainActor in - guard let self else { return } - self.audioDevices = self.audioManager.inputDevices - self.selectedAudioDeviceID = self.audioManager.defaultInputDevice.deviceId - self.onDevicesChanged?(self.audioDevices) - self.onSelectedDeviceChanged?(self.selectedAudioDeviceID) - } - } } private func configureSpeechHandler(muteMode: MicrophoneMuteMode, callbacks: ConversationCallbacks) { diff --git a/Sources/ElevenLabs/Internal/Conversation/ConversationInitiationMetadataWaiter.swift b/Sources/ElevenLabs/Internal/Conversation/ConversationInitiationMetadataWaiter.swift new file mode 100644 index 00000000..1bf0b7d5 --- /dev/null +++ b/Sources/ElevenLabs/Internal/Conversation/ConversationInitiationMetadataWaiter.swift @@ -0,0 +1,64 @@ +import Foundation + +actor ConversationInitiationMetadataWaiter { + private let timeoutNanoseconds: UInt64 + private var outcome: Result? + private var continuation: CheckedContinuation? + private var timeoutTask: Task? + + init(timeout: TimeInterval) { + timeoutNanoseconds = UInt64(timeout * 1_000_000_000) + } + + func observe(_ metadata: ConversationMetadataEvent) { + complete(.success(metadata)) + } + + func wait() async throws -> ConversationMetadataEvent { + if let outcome { return try outcome.get() } + if Task.isCancelled { + let error = CancellationError() + complete(.failure(error)) + throw error + } + + return try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + if let outcome { + continuation.resume(with: outcome) + return + } + precondition(self.continuation == nil, "Only one metadata wait is allowed") + self.continuation = continuation + startTimeout() + } + } onCancel: { + Task { await self.cancel() } + } + } + + func cancel() { + complete(.failure(CancellationError())) + } + + private func startTimeout() { + guard timeoutTask == nil else { return } + timeoutTask = Task { [weak self, timeoutNanoseconds] in + do { + try await Task.sleep(nanoseconds: timeoutNanoseconds) + } catch { + return + } + await self?.complete(.failure(ConversationError.initiationMetadataTimeout)) + } + } + + private func complete(_ outcome: Result) { + guard self.outcome == nil else { return } + self.outcome = outcome + timeoutTask?.cancel() + timeoutTask = nil + continuation?.resume(with: outcome) + continuation = nil + } +} diff --git a/Sources/ElevenLabs/Internal/Conversation/StartupResult.swift b/Sources/ElevenLabs/Internal/Conversation/StartupResult.swift deleted file mode 100644 index 93e152ae..00000000 --- a/Sources/ElevenLabs/Internal/Conversation/StartupResult.swift +++ /dev/null @@ -1,28 +0,0 @@ -import Foundation - -struct StartupResult { - let agentId: String - let metrics: ConversationStartupMetrics -} - -struct StartupFailure: Error { - let reason: ConversationStartupFailure - let error: ConversationError - let metrics: ConversationStartupMetrics - - static func token(_ error: ConversationError, _ metrics: ConversationStartupMetrics) -> Self { - .init(reason: .token(error), error: error, metrics: metrics) - } - - static func room(_ error: ConversationError, _ metrics: ConversationStartupMetrics) -> Self { - .init(reason: .room(error), error: error, metrics: metrics) - } - - static func agentTimeout(_ metrics: ConversationStartupMetrics) -> Self { - .init(reason: .agentTimeout, error: .agentTimeout, metrics: metrics) - } - - static func conversationInit(_ error: ConversationError, _ metrics: ConversationStartupMetrics) -> Self { - .init(reason: .conversationInit(error), error: error, metrics: metrics) - } -} diff --git a/Sources/ElevenLabs/Internal/DI/Dependencies.swift b/Sources/ElevenLabs/Internal/DI/Dependencies.swift index 8c2e8cb1..0433f220 100644 --- a/Sources/ElevenLabs/Internal/DI/Dependencies.swift +++ b/Sources/ElevenLabs/Internal/DI/Dependencies.swift @@ -14,15 +14,14 @@ final class Dependencies: ConversationDependencyProvider { let webRTCConnectionManager: any WebRTCConnectionManaging let webSocketConnectionManager: any WebSocketConnectionManaging - init() { - let globalConfig = ElevenLabs.Global.shared.configuration - let tokenService = TokenService(configuration: TokenService.Configuration( - apiEndpoint: globalConfig.apiEndpoint?.absoluteString, - websocketURL: globalConfig.websocketUrl - )) - let logger = SDKLogger(logLevel: globalConfig.logLevel) + init(logLevel: LogLevel = .warning, endpoints: Endpoints = .production) { + let logger = SDKLogger(logLevel: logLevel) self.logger = logger - webRTCConnectionManager = WebRTCConnectionManager(logger: logger, tokenService: tokenService) - webSocketConnectionManager = WebSocketConnectionManager(logger: logger) + webRTCConnectionManager = WebRTCConnectionManager( + logger: logger, + tokenService: TokenService(endpoints: endpoints), + endpoints: endpoints + ) + webSocketConnectionManager = WebSocketConnectionManager(logger: logger, endpoints: endpoints) } } diff --git a/Sources/ElevenLabs/Internal/Conversation/ConnectionManaging.swift b/Sources/ElevenLabs/Internal/Networking/ConnectionManaging.swift similarity index 55% rename from Sources/ElevenLabs/Internal/Conversation/ConnectionManaging.swift rename to Sources/ElevenLabs/Internal/Networking/ConnectionManaging.swift index b13a1307..dc4e13f8 100644 --- a/Sources/ElevenLabs/Internal/Conversation/ConnectionManaging.swift +++ b/Sources/ElevenLabs/Internal/Networking/ConnectionManaging.swift @@ -10,34 +10,41 @@ protocol ConnectionManaging: AnyObject { var onDisconnected: (() async -> Void)? { get set } var errorHandler: ((Swift.Error?) -> Void)? { get set } + @MainActor + func connect( + auth: ConversationCredentials, + config: ConversationConfig, + onStartupStateChange: @escaping (ConversationStartupState) -> Void + ) async throws -> ConversationStartResult + func disconnect() async func send(data: Data) async throws } -protocol WebSocketConnectionManaging: ConnectionManaging { - func connect(auth: ConversationCredentials, config: ConversationConfig) async throws -> StartupResult -} +protocol WebSocketConnectionManaging: ConnectionManaging {} protocol WebRTCConnectionManaging: ConnectionManaging { var onRemoteSpeakingChanged: (@Sendable (Bool) -> Void)? { get set } - var inputTrack: LocalAudioTrack? { get } - var agentAudioTrack: RemoteAudioTrack? { get } + /// Fired when an audio track is published/subscribed/unpublished/unsubscribed. + var onTracksChanged: (@Sendable () -> Void)? { get set } + var inputTrack: (any AudioTrackProtocol)? { get } + var agentAudioTrack: (any AudioTrackProtocol)? { get } var isMicrophoneMuted: Bool { get } - @MainActor - func connect( - auth: ConversationCredentials, - config: ConversationConfig, - onStartupStateChange: @escaping (ConversationStartupState) -> Void - ) async throws -> StartupResult - func setMicrophoneMuted(_ muted: Bool) async throws } extension ConnectionManaging { - func handleIncomingData(_ data: Data, logger: any Logging) { + func handleIncomingData( + _ data: Data, + metadataWaiter: ConversationInitiationMetadataWaiter, + logger: any Logging + ) { do { if let event = try EventParser.parseIncomingEvent(from: data) { + if case let .conversationMetadata(metadata) = event { + Task { await metadataWaiter.observe(metadata) } + } onEventReceived?(event) } } catch let EventParseError.unknownEventType(type) { @@ -58,4 +65,22 @@ extension ConnectionManaging { throw ConversationError.notConnected } } + + @MainActor + func waitForInitiationMetadata( + config: ConversationConfig, + metrics: inout ConversationStartupMetrics, + startTime: Date, + metadataWaiter: ConversationInitiationMetadataWaiter, + onStartupStateChange: (ConversationStartupState) -> Void + ) async throws -> ConversationMetadataEvent { + let timeout = config.startupConfiguration.initiationMetadataTimeout + onStartupStateChange(.waitingForInitiationMetadata(timeout: timeout)) + + let waitStart = Date() + let metadata = try await metadataWaiter.wait() + metrics.initiationMetadata = Date().timeIntervalSince(waitStart) + metrics.total = Date().timeIntervalSince(startTime) + return metadata + } } diff --git a/Sources/ElevenLabs/Internal/Networking/WebRTCConnectionManager.swift b/Sources/ElevenLabs/Internal/Networking/WebRTCConnectionManager.swift index 91234d8c..8d92ad06 100644 --- a/Sources/ElevenLabs/Internal/Networking/WebRTCConnectionManager.swift +++ b/Sources/ElevenLabs/Internal/Networking/WebRTCConnectionManager.swift @@ -5,6 +5,7 @@ import LiveKit enum AgentReadyWaitResult: Equatable { case success(elapsed: TimeInterval) case timedOut(elapsed: TimeInterval) + case cancelled(elapsed: TimeInterval) } enum WebRTCConnectionManagerError: Error { @@ -18,13 +19,11 @@ enum WebRTCConnectionManagerError: Error { /// remote disconnect) plus an async readiness API. /// /// LiveKit observation is split across two `RoomDelegate` instances: -/// - `LiveKitRoomEventDelegate` — data, speaking, remote disconnect +/// - `LiveKitRoomEventDelegate` — data, speaking, remote disconnect, track changes /// - `LiveKitReadinessDelegate` — signals when the agent's audio track subscribes /// -/// Note: `Room`, `LocalAudioTrack`, and `RemoteAudioTrack` are intentionally -/// exposed on the public SDK surface (e.g. `Conversation.inputTrack`), so this -/// type does not fully hide LiveKit from callers. It does centralize the -/// dependency in one place. +/// LiveKit room/track types stay internal to this manager; consumers observe +/// audio via ``ConversationAudioObserver`` rather than LiveKit track APIs. final class WebRTCConnectionManager: WebRTCConnectionManaging { /// Fired when the remote agent leaves, the room disconnects, or all remote participants are gone. var onDisconnected: (() async -> Void)? @@ -35,18 +34,22 @@ final class WebRTCConnectionManager: WebRTCConnectionManaging { /// Fired when a remote participant starts or stops speaking. var onRemoteSpeakingChanged: (@Sendable (Bool) -> Void)? + /// Fired when an audio track is published/subscribed/unpublished/unsubscribed, + /// so callers can (re)attach audio observers to the current tracks. + var onTracksChanged: (@Sendable () -> Void)? + var errorHandler: ((Swift.Error?) -> Void)? // MARK: – Public state accessors private(set) var room: Room? - var inputTrack: LocalAudioTrack? { - room?.localParticipant.firstAudioPublication?.track as? LocalAudioTrack + var inputTrack: (any AudioTrackProtocol)? { + room?.localParticipant.firstAudioPublication?.track as? any AudioTrackProtocol } - var agentAudioTrack: RemoteAudioTrack? { - room?.remoteParticipants.values.first?.firstAudioPublication?.track as? RemoteAudioTrack + var agentAudioTrack: (any AudioTrackProtocol)? { + room?.remoteParticipants.values.first?.firstAudioPublication?.track as? any AudioTrackProtocol } var isMicrophoneMuted: Bool { @@ -58,37 +61,45 @@ final class WebRTCConnectionManager: WebRTCConnectionManaging { private var eventDelegate: LiveKitRoomEventDelegate? private var readinessDelegate: LiveKitReadinessDelegate? + private var initiationMetadataWaiter: ConversationInitiationMetadataWaiter? private static let reliableDataPublishOptions = DataPublishOptions(reliable: true) private let logger: any Logging private let tokenService: any TokenServicing + private let endpoints: Endpoints - init(logger: any Logging, tokenService: any TokenServicing) { + init(logger: any Logging, tokenService: any TokenServicing, endpoints: Endpoints) { self.logger = logger self.tokenService = tokenService + self.endpoints = endpoints } // MARK: – Public API /// Full WebRTC startup sequence: resolve token → request mic permission → - /// connect room → wait for agent → send conversation_init (sent once). + /// connect room → wait for agent → send init → wait for initiation metadata. @MainActor func connect( auth: ConversationCredentials, config: ConversationConfig, onStartupStateChange: @escaping (ConversationStartupState) -> Void - ) async throws -> StartupResult { + ) async throws -> ConversationStartResult { + await initiationMetadataWaiter?.cancel() + let waiter = ConversationInitiationMetadataWaiter( + timeout: config.startupConfiguration.initiationMetadataTimeout + ) + initiationMetadataWaiter = waiter let startTime = Date() var metrics = ConversationStartupMetrics() logger.info("Starting conversation startup sequence", context: ["agentId": auth.agentId]) - // 1. Resolve token / connection details. + // 1. Resolve the conversation token. onStartupStateChange(.resolvingToken) - let connectionDetails = try await runPhase( - timing: \.tokenFetch, metrics: &metrics, startTime: startTime, failure: StartupFailure.token + let token = try await runPhase( + timing: \.tokenFetch, metrics: &metrics, startTime: startTime ) { - try await tokenService.fetchConnectionDetails(configuration: auth) + try await tokenService.fetchToken(for: auth) } // 2. Request microphone permission (denial doesn't block startup). @@ -98,43 +109,59 @@ final class WebRTCConnectionManager: WebRTCConnectionManaging { onStartupStateChange(.connectingRoom) let throwOnMicFailure = !config.continueWithoutMicrophoneOnFailure try await runPhase( - timing: \.roomConnect, metrics: &metrics, startTime: startTime, failure: StartupFailure.room + timing: \.roomConnect, metrics: &metrics, startTime: startTime ) { try await connectToRoom( - details: connectionDetails, + token: token, enableMic: permissionGranted, throwOnMicrophoneFailure: throwOnMicFailure, - networkConfiguration: config.networkConfiguration + networkConfiguration: config.networkConfiguration, + metadataWaiter: waiter ) } // 4. Wait for the agent to be ready (fails outright if it doesn't join in time). let agentTimeout = config.startupConfiguration.agentReadyTimeout onStartupStateChange(.waitingForAgent(timeout: agentTimeout)) - guard case let .success(elapsed) = await waitForAgentReady(timeout: agentTimeout) else { + switch await waitForAgentReady(timeout: agentTimeout) { + case let .success(elapsed): + metrics.agentReady = elapsed + onStartupStateChange(.agentReady(elapsed: elapsed)) + case let .timedOut(elapsed): + metrics.agentReady = elapsed metrics.total = Date().timeIntervalSince(startTime) logger.warning("Agent not ready within \(String(format: "%.3f", agentTimeout))s") - throw StartupFailure.agentTimeout(metrics) + throw ConversationError.agentTimeout + case let .cancelled(elapsed): + metrics.agentReady = elapsed + metrics.total = Date().timeIntervalSince(startTime) + throw CancellationError() } - metrics.agentReady = elapsed - onStartupStateChange(.agentReady(ConversationAgentReadyReport(elapsed: elapsed))) // 5. Send conversation_initiation_client_data (sent once). - onStartupStateChange(.sendingConversationInit(attempt: 1)) + onStartupStateChange(.sendingConversationInit) try await runPhase( - timing: \.conversationInit, metrics: &metrics, startTime: startTime, - failure: StartupFailure.conversationInit + timing: \.conversationInit, metrics: &metrics, startTime: startTime ) { try await send(event: .conversationInit(ConversationInitEvent(config: config))) } - metrics.conversationInitAttempts = 1 - metrics.total = Date().timeIntervalSince(startTime) - return StartupResult(agentId: auth.agentId, metrics: metrics) + let metadata = try await waitForInitiationMetadata( + config: config, + metrics: &metrics, + startTime: startTime, + metadataWaiter: waiter, + onStartupStateChange: onStartupStateChange + ) + return ConversationStartResult( + callInfo: CallInfo(agentId: auth.agentId, conversationId: metadata.conversationId), + metrics: metrics + ) } /// Race the delegate's "first remote participant joined" signal against `timeout`. - /// A `.timedOut` result makes `connect` fail with `StartupFailure.agentTimeout`. + /// `.timedOut` → `ConversationError.agentTimeout`; `.cancelled` → `CancellationError` + /// (disconnect/release during the wait). private func waitForAgentReady(timeout: TimeInterval) async -> AgentReadyWaitResult { guard let delegate = readinessDelegate else { return .timedOut(elapsed: 0) @@ -145,6 +172,8 @@ final class WebRTCConnectionManager: WebRTCConnectionManaging { do { try await delegate.awaitRemoteParticipant() return .success(elapsed: Date().timeIntervalSince(start)) + } catch is CancellationError { + return .cancelled(elapsed: Date().timeIntervalSince(start)) } catch { return .timedOut(elapsed: Date().timeIntervalSince(start)) } @@ -190,15 +219,16 @@ final class WebRTCConnectionManager: WebRTCConnectionManaging { /// used by `connect`). /// /// - Parameters: - /// - details: Token-service credentials (URL + participant token). + /// - token: Conversation token to authenticate the room with. /// - enableMic: Whether to enable the local microphone immediately. /// - throwOnMicrophoneFailure: If true, throws error when microphone setup fails. /// If false, logs warning and continues. private func connectToRoom( - details: TokenService.ConnectionDetails, + token: String, enableMic: Bool, throwOnMicrophoneFailure: Bool, - networkConfiguration: WebRTCConfiguration + networkConfiguration: WebRTCConfiguration, + metadataWaiter: ConversationInitiationMetadataWaiter ) async throws { await readinessDelegate?.release() @@ -207,9 +237,14 @@ final class WebRTCConnectionManager: WebRTCConnectionManaging { let logger = logger let eventDelegate = LiveKitRoomEventDelegate( - onData: { [weak self] data in self?.handleIncomingData(data, logger: logger) }, + onData: { [weak self] data in + self?.handleIncomingData(data, metadataWaiter: metadataWaiter, logger: logger) + }, onRemoteSpeaking: { [weak self] isSpeaking in self?.onRemoteSpeakingChanged?(isSpeaking) }, - onRemoteDisconnect: { [weak self] in await self?.onDisconnected?() } + onRemoteDisconnect: { [weak self] in await self?.onDisconnected?() }, + onTracksChanged: { [weak self] in + Task { @MainActor in self?.onTracksChanged?() } + } ) self.eventDelegate = eventDelegate @@ -225,8 +260,8 @@ final class WebRTCConnectionManager: WebRTCConnectionManaging { let connectStart = Date() do { try await room.connect( - url: details.serverUrl, - token: details.participantToken, + url: endpoints.voiceWebSocket.absoluteString, + token: token, connectOptions: connectOptions ) logger.info("LiveKit room.connect completed", context: ["duration": "\(Date().timeIntervalSince(connectStart))"]) @@ -259,7 +294,10 @@ final class WebRTCConnectionManager: WebRTCConnectionManaging { onDisconnected = nil errorHandler = nil onRemoteSpeakingChanged = nil + onTracksChanged = nil + await initiationMetadataWaiter?.cancel() + initiationMetadataWaiter = nil await readinessDelegate?.release() readinessDelegate = nil @@ -271,14 +309,13 @@ final class WebRTCConnectionManager: WebRTCConnectionManaging { // MARK: – Private helpers /// Run one timed startup phase: record its duration into `metrics[keyPath:]`, - /// let `CancellationError` propagate unwrapped, and wrap any other error via - /// `failure` (stamping `total`). + /// let `CancellationError` propagate unwrapped, and wrap any other error as + /// `ConversationError` (stamping `total`). @MainActor private func runPhase( timing keyPath: WritableKeyPath, metrics: inout ConversationStartupMetrics, startTime: Date, - failure: (ConversationError, ConversationStartupMetrics) -> StartupFailure, _ body: () async throws -> T ) async throws -> T { let start = Date() @@ -293,7 +330,7 @@ final class WebRTCConnectionManager: WebRTCConnectionManaging { } catch { metrics[keyPath: keyPath] = Date().timeIntervalSince(start) metrics.total = Date().timeIntervalSince(startTime) - throw failure(error as? ConversationError ?? .connectionFailed(error), metrics) + throw error as? ConversationError ?? ConversationError.connectionFailed(error) } } @@ -330,21 +367,24 @@ private func isAgentParticipant(_ participant: Participant) -> Bool { // MARK: – Room event delegate -/// `RoomDelegate` that forwards data, remote speaking, and remote-disconnect events -/// (agent leaving or room disconnect) to manager-supplied closures. +/// `RoomDelegate` that forwards data, remote speaking, remote-disconnect, and +/// audio track availability events to manager-supplied closures. final class LiveKitRoomEventDelegate: RoomDelegate { private let onData: @Sendable (Data) -> Void private let onRemoteSpeaking: @Sendable (Bool) -> Void private let onRemoteDisconnect: @Sendable () async -> Void + private let onTracksChanged: @Sendable () -> Void init( onData: @escaping @Sendable (Data) -> Void, onRemoteSpeaking: @escaping @Sendable (Bool) -> Void, - onRemoteDisconnect: @escaping @Sendable () async -> Void + onRemoteDisconnect: @escaping @Sendable () async -> Void, + onTracksChanged: @escaping @Sendable () -> Void = {} ) { self.onData = onData self.onRemoteSpeaking = onRemoteSpeaking self.onRemoteDisconnect = onRemoteDisconnect + self.onTracksChanged = onTracksChanged } nonisolated func room( @@ -372,6 +412,27 @@ final class LiveKitRoomEventDelegate: RoomDelegate { await onRemoteDisconnect() } } + + /// Audio track availability changes — used to (re)attach audio observers. + nonisolated func room(_: Room, participant _: RemoteParticipant, didSubscribeTrack publication: RemoteTrackPublication) { + guard publication.kind == .audio else { return } + onTracksChanged() + } + + nonisolated func room(_: Room, participant _: RemoteParticipant, didUnsubscribeTrack publication: RemoteTrackPublication) { + guard publication.kind == .audio else { return } + onTracksChanged() + } + + nonisolated func room(_: Room, participant _: LocalParticipant, didPublishTrack publication: LocalTrackPublication) { + guard publication.kind == .audio else { return } + onTracksChanged() + } + + nonisolated func room(_: Room, participant _: LocalParticipant, didUnpublishTrack publication: LocalTrackPublication) { + guard publication.kind == .audio else { return } + onTracksChanged() + } } // MARK: – Readiness delegate diff --git a/Sources/ElevenLabs/Internal/Networking/WebSocketConnectionManager.swift b/Sources/ElevenLabs/Internal/Networking/WebSocketConnectionManager.swift index 7f96ffc4..2dfb6b7a 100644 --- a/Sources/ElevenLabs/Internal/Networking/WebSocketConnectionManager.swift +++ b/Sources/ElevenLabs/Internal/Networking/WebSocketConnectionManager.swift @@ -4,8 +4,8 @@ import Foundation /// /// Opens a single `URLSessionWebSocketTask` to the text conversation endpoint, /// sends `conversationInit` once the socket is open, then runs a receive loop -/// that parses incoming JSON into `IncomingEvent`s and forwards them via -/// `onEventReceived`. On runtime socket error, fires `onDisconnected` once. +/// that forwards incoming events and waits for initiation metadata. On runtime +/// socket error, fires `onDisconnected` once. /// /// Used instead of `WebRTCConnectionManager` because the WebRTC transport /// drops rooms with no audio — text-only needs a transport that stays open @@ -17,11 +17,14 @@ final class WebSocketConnectionManager: WebSocketConnectionManaging { private let urlSession: URLSession private let logger: any Logging + private let endpoints: Endpoints private var task: URLSessionWebSocketTask? private var receiveTask: Task? + private var initiationMetadataWaiter: ConversationInitiationMetadataWaiter? - init(logger: any Logging) { + init(logger: any Logging, endpoints: Endpoints = .production) { self.logger = logger + self.endpoints = endpoints urlSession = URLSession(configuration: .default) } @@ -29,17 +32,27 @@ final class WebSocketConnectionManager: WebSocketConnectionManaging { urlSession.invalidateAndCancel() } - func connect(auth: ConversationCredentials, config: ConversationConfig) async throws -> StartupResult { + @MainActor + func connect( + auth: ConversationCredentials, + config: ConversationConfig, + onStartupStateChange: @escaping (ConversationStartupState) -> Void + ) async throws -> ConversationStartResult { + await initiationMetadataWaiter?.cancel() + let waiter = ConversationInitiationMetadataWaiter( + timeout: config.startupConfiguration.initiationMetadataTimeout + ) + initiationMetadataWaiter = waiter let startTime = Date() var metrics = ConversationStartupMetrics() let url: URL do { - url = try Self.url(for: auth) + url = try Self.websocketUrl(for: auth, endpoints: endpoints) } catch { metrics.total = Date().timeIntervalSince(startTime) let convError = error as? ConversationError ?? .authenticationFailed(error.localizedDescription) - throw StartupFailure.token(convError, metrics) + throw convError } let task = urlSession.webSocketTask(with: url) @@ -48,6 +61,7 @@ final class WebSocketConnectionManager: WebSocketConnectionManaging { // The first send awaits the WebSocket handshake internally — // any connection failure surfaces here. + onStartupStateChange(.sendingConversationInit) do { let initEvent = ConversationInitEvent(config: config) try await send(data: EventSerializer.serializeOutgoingEvent(.conversationInit(initEvent))) @@ -57,19 +71,26 @@ final class WebSocketConnectionManager: WebSocketConnectionManaging { } catch { tearDownTask(task) metrics.total = Date().timeIntervalSince(startTime) - let convError = error as? ConversationError ?? .connectionFailed(error) - throw StartupFailure.conversationInit(convError, metrics) + throw error as? ConversationError ?? ConversationError.connectionFailed(error) } // Socket is up and the init message is sent. Start consuming responses. receiveTask = Task { [weak self, weak task] in guard let self, let task else { return } - await receiveLoop(task: task) + await receiveLoop(task: task, metadataWaiter: waiter) } - metrics.conversationInitAttempts = 1 - metrics.total = Date().timeIntervalSince(startTime) - return StartupResult(agentId: auth.agentId, metrics: metrics) + let metadata = try await waitForInitiationMetadata( + config: config, + metrics: &metrics, + startTime: startTime, + metadataWaiter: waiter, + onStartupStateChange: onStartupStateChange + ) + return ConversationStartResult( + callInfo: CallInfo(agentId: auth.agentId, conversationId: metadata.conversationId), + metrics: metrics + ) } func send(data: Data) async throws { @@ -84,6 +105,8 @@ final class WebSocketConnectionManager: WebSocketConnectionManaging { onDisconnected = nil errorHandler = nil + await initiationMetadataWaiter?.cancel() + initiationMetadataWaiter = nil receiveTask?.cancel() receiveTask = nil task?.cancel(with: .normalClosure, reason: nil) @@ -97,13 +120,20 @@ final class WebSocketConnectionManager: WebSocketConnectionManaging { } } - private func receiveLoop(task: URLSessionWebSocketTask) async { + private func receiveLoop( + task: URLSessionWebSocketTask, + metadataWaiter: ConversationInitiationMetadataWaiter + ) async { while !Task.isCancelled { do { let message = try await task.receive() switch message { case let .string(text): - handleIncomingData(Data(text.utf8), logger: logger) + handleIncomingData( + Data(text.utf8), + metadataWaiter: metadataWaiter, + logger: logger + ) case .data: logger.warning("Ignoring binary WebSocket message") @unknown default: @@ -119,12 +149,16 @@ final class WebSocketConnectionManager: WebSocketConnectionManaging { } } - static func url(for auth: ConversationCredentials) throws -> URL { + static func websocketUrl(for auth: ConversationCredentials, endpoints: Endpoints) throws -> URL { switch auth.authSource { case let .publicAgentId(agentId): - var components = URLComponents(string: ConnectionConstants.textConversationUrl) - components?.queryItems = [URLQueryItem(name: "agent_id", value: agentId)] - guard let url = components?.url else { + guard var components = URLComponents(url: endpoints.textWebSocket, resolvingAgainstBaseURL: false) else { + throw ConversationError.authenticationFailed("Invalid conversation URL") + } + var queryItems = components.queryItems ?? [] + queryItems.append(URLQueryItem(name: "agent_id", value: agentId)) + components.queryItems = queryItems + guard let url = components.url else { throw ConversationError.authenticationFailed("Invalid conversation URL") } return url diff --git a/Sources/ElevenLabs/Internal/Utilities/EventSerializer.swift b/Sources/ElevenLabs/Internal/Utilities/EventSerializer.swift index 74bd5c21..53c604d4 100644 --- a/Sources/ElevenLabs/Internal/Utilities/EventSerializer.swift +++ b/Sources/ElevenLabs/Internal/Utilities/EventSerializer.swift @@ -120,7 +120,7 @@ enum EventSerializer { // Add source_info (equivalent to client in React Native) var sourceInfo: [String: Any] = [:] sourceInfo["source"] = "swift_sdk" - sourceInfo["version"] = SDKVersion.version + sourceInfo["version"] = version json["source_info"] = sourceInfo // Add user_id if provided diff --git a/Sources/ElevenLabs/Internal/Utilities/SDKLogger.swift b/Sources/ElevenLabs/Internal/Utilities/SDKLogger.swift index 8d1b5bde..5cd9de6b 100644 --- a/Sources/ElevenLabs/Internal/Utilities/SDKLogger.swift +++ b/Sources/ElevenLabs/Internal/Utilities/SDKLogger.swift @@ -37,12 +37,12 @@ extension Logging { struct SDKLogger: Logging { private let subsystem: String private let category: String - private let logLevel: ElevenLabs.LogLevel + private let logLevel: LogLevel init( subsystem: String = "com.elevenlabs.sdk", category: String = "ElevenLabs", - logLevel: ElevenLabs.LogLevel = .info + logLevel: LogLevel = .info ) { self.subsystem = subsystem self.category = category diff --git a/Sources/ElevenLabs/Public/Conversation/AgentStateConfiguration.swift b/Sources/ElevenLabs/Public/Conversation/AgentStateConfiguration.swift deleted file mode 100644 index 9d51b61b..00000000 --- a/Sources/ElevenLabs/Public/Conversation/AgentStateConfiguration.swift +++ /dev/null @@ -1,24 +0,0 @@ -import Foundation - -/// Configuration for event-based agent state management using VAD and client events. -/// Pass `nil` to use the default LiveKit-based behaviour. -public struct AgentStateConfiguration: Sendable { - public var vadSpeakingThreshold: Double - public var minSpeechDuration: TimeInterval - public var minSilenceDuration: TimeInterval - public var speakingToListeningDelay: TimeInterval - - public init( - vadSpeakingThreshold: Double = 0.5, - minSpeechDuration: TimeInterval = 0.15, - minSilenceDuration: TimeInterval = 0.05, - speakingToListeningDelay: TimeInterval = 0.5 - ) { - self.vadSpeakingThreshold = vadSpeakingThreshold - self.minSpeechDuration = minSpeechDuration - self.minSilenceDuration = minSilenceDuration - self.speakingToListeningDelay = speakingToListeningDelay - } - - public static let `default` = AgentStateConfiguration() -} diff --git a/Sources/ElevenLabs/Public/Conversation/AudioPipelineConfiguration.swift b/Sources/ElevenLabs/Public/Conversation/AudioPipelineConfiguration.swift deleted file mode 100644 index be4c6120..00000000 --- a/Sources/ElevenLabs/Public/Conversation/AudioPipelineConfiguration.swift +++ /dev/null @@ -1,58 +0,0 @@ -import Foundation - -/// Configures microphone pipeline and voice activity reporting exposed by the SDK. -public struct AudioPipelineConfiguration: Sendable { - /// Override the microphone mute strategy. Defaults to `.inputMixer` to match previous SDK behaviour. - public var microphoneMuteMode: MicrophoneMuteMode? - - /// Keep the recording engine warm to avoid first-spoken-word latency. Defaults to `true`. - public var recordingAlwaysPrepared: Bool? - - /// Bypass WebRTC voice processing (AEC/NS/VAD). Leave `nil` to preserve system defaults. - public var voiceProcessingBypassed: Bool? - - /// Toggle Auto Gain Control. Leave `nil` to preserve system defaults. - public var voiceProcessingAGCEnabled: Bool? - - public init( - microphoneMuteMode: MicrophoneMuteMode? = .inputMixer, - recordingAlwaysPrepared: Bool? = true, - voiceProcessingBypassed: Bool? = nil, - voiceProcessingAGCEnabled: Bool? = nil - ) { - self.microphoneMuteMode = microphoneMuteMode - self.recordingAlwaysPrepared = recordingAlwaysPrepared - self.voiceProcessingBypassed = voiceProcessingBypassed - self.voiceProcessingAGCEnabled = voiceProcessingAGCEnabled - } - - public static let `default` = AudioPipelineConfiguration() -} - -/// Strategy used when muting the local microphone. Exactly one strategy is active -/// at a time. -public enum MicrophoneMuteMode: Sendable, Equatable { - /// Mutes instantly by silencing the input mixer. The mic stays open and the - /// audio session remains active. Recommended default. - /// - /// Use ``software(speechThreshold:notificationThrottle:)`` for silent muting - /// with speech detection. - case inputMixer - - /// Mutes by restarting the engine without mic input. Releases the mic, but - /// mute/unmute is slower and speech detection is unavailable. - case restart - - /// Mutes the voice-processing input. Fast, supports - /// ``ConversationCallbacks/onSpeechDetectedWhileMuted``, and keeps the audio - /// session active. - case voiceProcessing - - /// Mutes in software by zeroing captured audio before it leaves the device. - /// Supports ``ConversationCallbacks/onSpeechDetectedWhileMuted``. - /// - /// - Parameters: - /// - speechThreshold: dB threshold for muted-speech detection. - /// - notificationThrottle: Minimum interval between muted-speech callbacks. - case software(speechThreshold: Float = -35, notificationThrottle: TimeInterval = 3.0) -} diff --git a/Sources/ElevenLabs/Public/Conversation/Conversation.swift b/Sources/ElevenLabs/Public/Conversation/Conversation.swift deleted file mode 100644 index ca230714..00000000 --- a/Sources/ElevenLabs/Public/Conversation/Conversation.swift +++ /dev/null @@ -1,563 +0,0 @@ -import Combine -import Foundation -import LiveKit - -// swiftlint:disable file_length type_body_length - -/// The central entry point for the ElevenLabs Conversational AI SDK. -/// -/// **Role:** -/// - Manages the lifecycle of a single conversation session. -/// - Coordinates state between the network layer (`WebRTCConnectionManager`|`WebSocketConnectionManager`), protocol parser -/// (`EventParser`), and the UI (`ObservableObject`). -/// - Handles audio device management and permission checks. -/// -/// **Usage:** -/// Create an instance via `ElevenLabs.startConversation(...)`. Use the `@Published` properties -/// to bind your UI to conversation state. -@MainActor -public final class Conversation: ObservableObject { - // MARK: - Public State - - @Published public internal(set) var state: ConversationState = .idle - @Published public internal(set) var startupState: ConversationStartupState = .idle - @Published public internal(set) var startupMetrics: ConversationStartupMetrics? - @Published public internal(set) var messages: [Message] = [] - @Published public internal(set) var agentState: ElevenLabs.AgentState = .listening - @Published public internal(set) var isMicMuted: Bool = true - - /// Stream of client tool calls that need to be executed by the app - @Published public internal(set) var pendingToolCalls: [ClientToolCallEvent] = [] - - /// Conversation metadata including conversation ID, received when the conversation is initialized - @Published public internal(set) var conversationMetadata: ConversationMetadataEvent? - - /// MCP tool calls from the agent - @Published public internal(set) var mcpToolCalls: [MCPToolCallEvent] = [] - - /// Current MCP connection status for all integrations - @Published public internal(set) var mcpConnectionStatus: MCPConnectionStatusEvent? - - /// Latest audio alignment payload emitted by the agent. - @Published public internal(set) var latestAudioAlignment: AudioAlignment? - - /// Latest audio event emitted by the agent. - @Published public internal(set) var latestAudioEvent: AudioEvent? - - /// Device lists (optional to expose; keep `internal` if you don't want them public) - @Published public internal(set) var audioDevices: [AudioDevice] = [] - @Published public internal(set) var selectedAudioDeviceID: String = "" - - var lastAgentEventId: Int? - var lastFeedbackSubmittedEventId: Int? - - /// Pending mute state to apply after connection completes. - /// Allows setting mute state during connection phase. - private var pendingMuteState: Bool? - - /// Audio device management - private var audioManager: ConversationAudioManager? - - /// Agent state manager for event-based state tracking - var agentStateManager: AgentStateManager? - - /// Forward a signal to the event-based state manager, or fall back to directly setting `agentState`. - func applyStateSignal(_ signal: AgentStateSignal, fallback: ElevenLabs.AgentState) { - if let manager = agentStateManager { - manager.processSignal(signal) - } else { - agentState = fallback - } - } - - func handleRemoteSpeakingUpdate(isSpeaking: Bool) { - if let manager = agentStateManager { - manager.processSignal(isSpeaking ? .agentStartedSpeaking : .agentStoppedSpeaking) - } else if isSpeaking { - speakingTimer?.cancel() - agentState = .speaking - } else { - scheduleBackToListening(delay: 1.0) - } - } - - /// Internal logger, accessible from nonisolated contexts. - nonisolated let logger: any Logging - - /// Context for logging (e.g. agentId) - private var activeContext: [String: String]? - - /// Audio tracks for advanced use cases - public var inputTrack: LocalAudioTrack? { - activeWebRTCConnectionManager?.inputTrack - } - - public var agentAudioTrack: RemoteAudioTrack? { - activeWebRTCConnectionManager?.agentAudioTrack - } - - // MARK: - Init - - init( - dependencyProvider: any ConversationDependencyProvider, - config: ConversationConfig = .init(), - callbacks: ConversationCallbacks = .init() - ) { - self.dependencyProvider = dependencyProvider - self.config = config - self.callbacks = callbacks - logger = dependencyProvider.logger - setupAudioManager() - } - - private func setupAudioManager() { - guard !config.conversationOverrides.textOnly else { return } - let manager = ConversationAudioManager(logger: logger) - manager.onDevicesChanged = { [weak self] devices in - self?.audioDevices = devices - } - manager.onSelectedDeviceChanged = { [weak self] deviceId in - self?.selectedAudioDeviceID = deviceId - } - audioManager = manager - // Sync initial values - audioDevices = manager.audioDevices - selectedAudioDeviceID = manager.selectedAudioDeviceID - } - - private func setupAgentStateManager() { - guard let configuration = config.agentStateConfiguration else { return } - let manager = AgentStateManager(configuration: configuration) - manager.onStateChange = { [weak self] state in - self?.agentState = state - self?.callbacks.onAgentStateChange?(state) - } - agentStateManager = manager - } - - // MARK: - Public API - - /// Start a conversation with an agent using agent ID. - /// - /// Each call to this method creates a fresh Room object, ensuring clean state - /// and preventing any interference from previous conversations. - public func startConversation( - with agentId: String, - config: ConversationConfig = .init() - ) async throws { - let authConfig = ConversationCredentials.publicAgent(id: agentId, environment: config.environment) - try await startConversation(auth: authConfig, config: config) - } - - /// Start a conversation using authentication configuration. - /// - /// Each call to this method creates a fresh Room object, ensuring clean state - /// and preventing any interference from previous conversations. - public func startConversation( - auth: ConversationCredentials, - config: ConversationConfig = .init() - ) async throws { - guard state == .idle || state.isEnded else { - throw ConversationError.alreadyActive - } - - let result: StartupResult = if config.conversationOverrides.textOnly { - try await startTextOnlyConversation(auth: auth, config: config, provider: dependencyProvider) - } else { - try await startVoiceConversation(auth: auth, config: config, provider: dependencyProvider) - } - - state = .connected(.init(agentId: result.agentId)) - startupMetrics = result.metrics - updateStartupState(.connected(CallInfo(agentId: result.agentId), result.metrics)) - callbacks.onAgentReady?() - } - - private func startVoiceConversation( - auth: ConversationCredentials, - config: ConversationConfig, - provider: any ConversationDependencyProvider - ) async throws -> StartupResult { - let webRTCConnectionManager = provider.webRTCConnectionManager - await prepareConversationStart( - auth: auth, config: config, - connectionManager: webRTCConnectionManager - ) - - webRTCConnectionManager.onRemoteSpeakingChanged = { [weak self] isSpeaking in - Task { @MainActor in - self?.handleRemoteSpeakingUpdate(isSpeaking: isSpeaking) - } - } - - if audioManager == nil { - setupAudioManager() - } - await audioManager?.configure(with: config, callbacks: callbacks) - - let result: StartupResult - do { - result = try await webRTCConnectionManager.connect( - auth: auth, - config: config, - onStartupStateChange: { [weak self] newState in - self?.updateStartupState(newState) - } - ) - } catch let failure as StartupFailure { - await handleStartupFailure(failure, disconnecting: webRTCConnectionManager) - throw failure.error - } catch is CancellationError { - await handleStartupCancellation(disconnecting: webRTCConnectionManager) - throw CancellationError() - } - - if let pendingMute = pendingMuteState { - pendingMuteState = nil - do { - try await webRTCConnectionManager.setMicrophoneMuted(pendingMute) - isMicMuted = pendingMute - } catch { - logger.warning("Failed to apply pending mute state", context: ["error": "\(error)"]) - } - } - - isMicMuted = webRTCConnectionManager.isMicrophoneMuted - return result - } - - private func startTextOnlyConversation( - auth: ConversationCredentials, - config: ConversationConfig, - provider: ConversationDependencyProvider - ) async throws -> StartupResult { - let connectionManager = provider.webSocketConnectionManager - await prepareConversationStart( - auth: auth, config: config, - connectionManager: connectionManager - ) - - updateStartupState(.connectingRoom) - - do { - return try await connectionManager.connect(auth: auth, config: config) - } catch let failure as StartupFailure { - await handleStartupFailure(failure, disconnecting: connectionManager) - throw failure.error - } catch is CancellationError { - await handleStartupCancellation(disconnecting: connectionManager) - throw CancellationError() - } - } - - /// End and clean up. - /// Can be called during connection phase to cancel, or during connected conversation to end. - public func endConversation() async { - await endConversation(disconnectReason: .user, endReason: .userEnded) - } - - private func endConversation(disconnectReason: DisconnectionReason = .user, endReason: EndReason = .userEnded) async { - // Allow ending during both connected and connecting states - guard state.isConnected || state == .connecting else { return } - guard let connectionManager = activeConnectionManager else { - // No connection manager yet, just reset state - if state == .connecting { - state = .idle - tearDownActiveSession() - } - return - } - state = .ended(reason: endReason) - - // Disconnect synchronously to ensure clean state - await connectionManager.disconnect() - - tearDownActiveSession() - - // Call user's onDisconnect callback if provided - callbacks.onDisconnect?(disconnectReason) - callbacks.onCanSendFeedbackChange?(false) - } - - /// Send a text message to the agent. - public func sendMessage(_ text: String) async throws { - guard state.isConnected else { - throw ConversationError.notConnected - } - let event = OutgoingEvent.userMessage(UserMessageEvent(text: text)) - try await publish(event) - appendMessage(role: .user, content: text) - } - - /// Toggle the local microphone mute state. - public func toggleMicMute() async throws { - try await setMicMuted(!isMicMuted) - } - - /// Mute or unmute the local microphone. - public func setMicMuted(_ muted: Bool) async throws { - if let softwareMuteProcessor = audioManager?.softwareMuteProcessor { - softwareMuteProcessor.setMuted(muted) - isMicMuted = muted - return - } - try await setHardwareMicMuted(muted) - } - - func setHardwareMicMuted(_ muted: Bool) async throws { - if state.isConnected { - guard let webRTCConnectionManager = activeWebRTCConnectionManager else { - throw ConversationError.notConnected - } - do { - try await webRTCConnectionManager.setMicrophoneMuted(muted) - isMicMuted = muted - pendingMuteState = nil - } catch WebRTCConnectionManagerError.roomUnavailable { - throw ConversationError.notConnected - } catch { - throw ConversationError.microphoneToggleFailed(error) - } - } else if state == .connecting { - // Buffer the mute state to apply after connection completes - pendingMuteState = muted - isMicMuted = muted - } else { - throw ConversationError.notConnected - } - } - - /// Interrupt the agent while speaking. - public func interruptAgent() async throws { - guard state.isConnected else { throw ConversationError.notConnected } - let event = OutgoingEvent.userActivity - try await publish(event) - } - - /// Contextual update to agent (system prompt-ish). - public func updateContext(_ context: String) async throws { - guard state.isConnected else { throw ConversationError.notConnected } - let event = OutgoingEvent.contextualUpdate(ContextualUpdateEvent(text: context)) - try await publish(event) - } - - /// Send feedback (like/dislike) for an event/message id. - public func sendFeedback(_ score: FeedbackEvent.Score, eventId: Int) async throws { - guard state.isConnected else { - throw ConversationError.notConnected - } - - let event = OutgoingEvent.feedback(FeedbackEvent(score: score, eventId: eventId)) - try await publish(event) - lastFeedbackSubmittedEventId = eventId - callbacks.onCanSendFeedbackChange?(false) - } - - /// Approve or reject an MCP tool call request from the agent. - /// - Parameters: - /// - toolCallId: The tool call identifier from `MCPToolCallEvent`. - /// - isApproved: Pass `true` to approve, `false` to reject. - public func sendMCPToolApproval(toolCallId: String, isApproved: Bool) async throws { - guard state.isConnected else { throw ConversationError.notConnected } - let approval = MCPToolApprovalResultEvent(toolCallId: toolCallId, isApproved: isApproved) - try await publish(.mcpToolApprovalResult(approval)) - } - - /// Send the result of a client tool call back to the agent. - /// - /// The `Encodable` result is JSON-encoded before sending. - public func sendToolResult( - for toolCallId: String, - result: some Encodable, - isError: Bool = false, - errorType: ClientToolErrorType? = nil - ) async throws { - let json = try String(decoding: JSONEncoder().encode(result), as: UTF8.self) - // `json` is statically a String, so this dispatches to the String overload - // below (a concrete match beats the generic) — not a recursive call. - try await sendToolResult(for: toolCallId, result: json, isError: isError, errorType: errorType) - } - - /// Send a client tool result that is already a string (sent verbatim). - public func sendToolResult( - for toolCallId: String, - result: String, - isError: Bool = false, - errorType: ClientToolErrorType? = nil - ) async throws { - guard state.isConnected else { throw ConversationError.notConnected } - let toolResult = ClientToolResultEvent( - toolCallId: toolCallId, result: result, isError: isError, errorType: errorType - ) - try await publish(.clientToolResult(toolResult)) - pendingToolCalls.removeAll { $0.toolCallId == toolResult.toolCallId } - } - - /// Mark a tool call as completed without sending a result (for tools that don't expect responses). - public func markToolCallCompleted(_ toolCallId: String) { - pendingToolCalls.removeAll { $0.toolCallId == toolCallId } - } - - // MARK: - Private - - private let dependencyProvider: any ConversationDependencyProvider - private var activeConnectionManager: (any ConnectionManaging)? - private var activeWebRTCConnectionManager: (any WebRTCConnectionManaging)? { - activeConnectionManager as? any WebRTCConnectionManaging - } - - var config: ConversationConfig - let callbacks: ConversationCallbacks - - var speakingTimer: Task? - - private func updateStartupState(_ newState: ConversationStartupState) { - startupState = newState - callbacks.onStartupStateChange?(newState) - } - - /// Common preparation shared by voice and text-only startup paths. - private func prepareConversationStart( - auth: ConversationCredentials, - config: ConversationConfig, - connectionManager: any ConnectionManaging - ) async { - let previousConnectionManager = activeConnectionManager - state = .connecting - - if let previousConnectionManager, previousConnectionManager !== connectionManager { - await previousConnectionManager.disconnect() - } - - activeConnectionManager = connectionManager - // Reset the target manager too; dependency providers may reuse manager instances across starts. - await connectionManager.disconnect() - cleanupPreviousConversation() - self.config = config - - activeContext = ["agentId": auth.agentId] - let mode = config.conversationOverrides.textOnly ? "text-only" : "voice" - logger.info("Starting \(mode) conversation", context: activeContext) - - callbacks.onCanSendFeedbackChange?(false) - setupAgentStateManager() - - connectionManager.onEventReceived = { [weak self, weak connectionManager] event in - Task { @MainActor [weak self, weak connectionManager] in - guard let self, - let connectionManager, - activeConnectionManager === connectionManager, - state == .connecting || state.isConnected - else { - return - } - - await handleIncomingEvent(event) - } - } - connectionManager.onDisconnected = { [weak self] in - guard let self else { return } - await endConversation(disconnectReason: .agent, endReason: .remoteDisconnected) - } - } - - private func handleStartupFailure( - _ failure: StartupFailure, - disconnecting connectionManager: any ConnectionManaging - ) async { - cleanupTransientResources() - await connectionManager.disconnect() - - startupMetrics = failure.metrics - state = .idle - updateStartupState(.failed(failure.reason, failure.metrics)) - callbacks.onError?(failure.error) - } - - private func handleStartupCancellation(disconnecting connectionManager: any ConnectionManaging) async { - cleanupTransientResources() - await connectionManager.disconnect() - startupMetrics = nil - state = .idle - updateStartupState(.idle) - } - - /// Clean up state from any previous conversation to ensure a fresh start. - /// Called when starting a new session; wipes both operational and display state. - private func cleanupPreviousConversation() { - tearDownActiveSession() - - messages.removeAll() - mcpToolCalls.removeAll() - mcpConnectionStatus = nil - conversationMetadata = nil - - startupState = .idle - startupMetrics = nil - - logger.debug("Previous conversation state cleaned up for fresh Room", context: activeContext) - } - - /// Tear down operational state when an active session ends. - /// Preserves user-visible display state (messages, MCP activity, conversation - /// metadata, startup metrics) so the transcript remains visible until a new - /// conversation is started. - private func tearDownActiveSession() { - cleanupTransientResources() - - pendingToolCalls.removeAll() - - lastAgentEventId = nil - lastFeedbackSubmittedEventId = nil - callbacks.onCanSendFeedbackChange?(false) - latestAudioEvent = nil - latestAudioAlignment = nil - } - - private func cleanupTransientResources() { - speakingTimer?.cancel() - speakingTimer = nil - pendingMuteState = nil - agentState = .listening - isMicMuted = true - - audioManager?.cleanup() - agentStateManager = nil - } - - private func scheduleBackToListening(delay: TimeInterval = 0.5) { - speakingTimer?.cancel() - speakingTimer = Task { - do { - try await Task.sleep(nanoseconds: UInt64(delay * 1_000_000_000)) - self.agentState = .listening - } catch { - // Task was cancelled, do nothing - } - } - } - - func publish(_ event: OutgoingEvent) async throws { - guard let connectionManager = activeConnectionManager else { - throw ConversationError.notConnected - } - - try await connectionManager.send(event: event) - } - - // MARK: - Message Helpers - - func appendMessage(role: Message.Role, content: String, eventId: Int? = nil) { - messages.append( - Message( - id: UUID().uuidString, - role: role, - content: content, - timestamp: Date(), - eventId: eventId - ) - ) - } -} - -// swiftlint:enable file_length type_body_length diff --git a/Sources/ElevenLabs/Public/Conversation/ConversationAgentReadyReport.swift b/Sources/ElevenLabs/Public/Conversation/ConversationAgentReadyReport.swift deleted file mode 100644 index ea439fcf..00000000 --- a/Sources/ElevenLabs/Public/Conversation/ConversationAgentReadyReport.swift +++ /dev/null @@ -1,9 +0,0 @@ -import Foundation - -public struct ConversationAgentReadyReport: Sendable, Equatable { - public let elapsed: TimeInterval - - public init(elapsed: TimeInterval) { - self.elapsed = elapsed - } -} diff --git a/Sources/ElevenLabs/Public/Conversation/ConversationAudioObserver.swift b/Sources/ElevenLabs/Public/Conversation/ConversationAudioObserver.swift new file mode 100644 index 00000000..15ad9f7c --- /dev/null +++ b/Sources/ElevenLabs/Public/Conversation/ConversationAudioObserver.swift @@ -0,0 +1,19 @@ +import AVFoundation + +/// Observes decoded PCM audio from a conversation stream. +/// +/// Attach an observer via ``ConversationClient/addAgentAudioObserver(_:)`` (agent +/// output) or ``ConversationClient/addMicAudioObserver(_:)`` (local microphone) +/// to receive raw audio for visualization, recording, or custom analysis. +/// +/// - Important: ``didReceive(_:)`` is called synchronously on a time-critical +/// audio callback path. The SDK does not dispatch it to the main actor. Keep +/// the work minimal and non-blocking; copy any samples you need during the +/// call, then hop to your own queue or actor before touching UI or doing +/// heavy processing. +/// - Note: The buffer is borrowed and must be treated as read-only. Mutating it +/// is unsupported and must not be relied on to affect capture or playback. +public protocol ConversationAudioObserver: AnyObject, Sendable { + /// Called with each decoded PCM buffer as it becomes available. + func didReceive(_ buffer: AVAudioPCMBuffer) +} diff --git a/Sources/ElevenLabs/Public/Conversation/ConversationCallbacks.swift b/Sources/ElevenLabs/Public/Conversation/ConversationCallbacks.swift index 7b65af3c..6984e0c0 100644 --- a/Sources/ElevenLabs/Public/Conversation/ConversationCallbacks.swift +++ b/Sources/ElevenLabs/Public/Conversation/ConversationCallbacks.swift @@ -6,10 +6,7 @@ public struct ConversationCallbacks: Sendable { public var onAgentReady: (@Sendable () -> Void)? /// Called when the agent disconnects or the conversation ends - public var onDisconnect: (@Sendable (DisconnectionReason) -> Void)? - - /// Called whenever the startup state transitions - public var onStartupStateChange: (@Sendable (ConversationStartupState) -> Void)? + public var onDisconnect: (@Sendable (EndReason) -> Void)? /// Called when a startup-related error occurs public var onError: (@Sendable (ConversationError) -> Void)? @@ -48,19 +45,15 @@ public struct ConversationCallbacks: Sendable { /// Called when audio alignment metadata is emitted. public var onAudioAlignment: (@Sendable (AudioAlignment) -> Void)? - /// Called when feedback availability changes. - public var onCanSendFeedbackChange: (@Sendable (Bool) -> Void)? - /// Called when a client tool call is received without a registered handler. - public var onUnhandledClientToolCall: (@Sendable (ClientToolCallEvent) -> Void)? + public var onClientToolCall: (@Sendable (ClientToolCallEvent) -> Void)? /// Called whenever the agent state changes (event-based mode only). - public var onAgentStateChange: (@Sendable (ElevenLabs.AgentState) -> Void)? + public var onAgentStateChange: (@Sendable (AgentState) -> Void)? public init( onAgentReady: (@Sendable () -> Void)? = nil, - onDisconnect: (@Sendable (DisconnectionReason) -> Void)? = nil, - onStartupStateChange: (@Sendable (ConversationStartupState) -> Void)? = nil, + onDisconnect: (@Sendable (EndReason) -> Void)? = nil, onError: (@Sendable (ConversationError) -> Void)? = nil, onSpeechDetectedWhileMuted: (@Sendable () -> Void)? = nil, onAgentResponse: (@Sendable (_ text: String, _ eventId: Int) -> Void)? = nil, @@ -73,13 +66,11 @@ public struct ConversationCallbacks: Sendable { onInterruption: (@Sendable (_ eventId: Int) -> Void)? = nil, onVadScore: (@Sendable (_ score: Double) -> Void)? = nil, onAudioAlignment: (@Sendable (AudioAlignment) -> Void)? = nil, - onCanSendFeedbackChange: (@Sendable (Bool) -> Void)? = nil, - onUnhandledClientToolCall: (@Sendable (ClientToolCallEvent) -> Void)? = nil, - onAgentStateChange: (@Sendable (ElevenLabs.AgentState) -> Void)? = nil + onClientToolCall: (@Sendable (ClientToolCallEvent) -> Void)? = nil, + onAgentStateChange: (@Sendable (AgentState) -> Void)? = nil ) { self.onAgentReady = onAgentReady self.onDisconnect = onDisconnect - self.onStartupStateChange = onStartupStateChange self.onError = onError self.onSpeechDetectedWhileMuted = onSpeechDetectedWhileMuted self.onAgentResponse = onAgentResponse @@ -92,8 +83,7 @@ public struct ConversationCallbacks: Sendable { self.onInterruption = onInterruption self.onVadScore = onVadScore self.onAudioAlignment = onAudioAlignment - self.onCanSendFeedbackChange = onCanSendFeedbackChange - self.onUnhandledClientToolCall = onUnhandledClientToolCall + self.onClientToolCall = onClientToolCall self.onAgentStateChange = onAgentStateChange } } diff --git a/Sources/ElevenLabs/Public/Conversation/ConversationConfig.swift b/Sources/ElevenLabs/Public/Conversation/ConversationConfig.swift index 9fa37f96..db59af5e 100644 --- a/Sources/ElevenLabs/Public/Conversation/ConversationConfig.swift +++ b/Sources/ElevenLabs/Public/Conversation/ConversationConfig.swift @@ -1,12 +1,5 @@ import Foundation -/// Reason for the conversation disconnection -public enum DisconnectionReason: Sendable { - case agent - case user - case error -} - /// Main configuration for a conversation session public struct ConversationConfig: Sendable { public var agentOverrides: AgentOverrides? @@ -35,6 +28,9 @@ public struct ConversationConfig: Sendable { /// instead of relying on LiveKit's isSpeaking detection. public var agentStateConfiguration: AgentStateConfiguration? + /// Network endpoints to connect to. Override for proxies, regional hosts, or staging. + public var endpoints: Endpoints + public init( agentOverrides: AgentOverrides? = nil, ttsOverrides: TTSOverrides? = nil, @@ -47,7 +43,8 @@ public struct ConversationConfig: Sendable { startupConfiguration: ConversationStartupConfiguration = .default, audioConfiguration: AudioPipelineConfiguration? = nil, networkConfiguration: WebRTCConfiguration = .default, - agentStateConfiguration: AgentStateConfiguration? = nil + agentStateConfiguration: AgentStateConfiguration? = nil, + endpoints: Endpoints = .production ) { self.agentOverrides = agentOverrides self.ttsOverrides = ttsOverrides @@ -61,6 +58,7 @@ public struct ConversationConfig: Sendable { self.audioConfiguration = audioConfiguration self.networkConfiguration = networkConfiguration self.agentStateConfiguration = agentStateConfiguration + self.endpoints = endpoints } } @@ -114,3 +112,152 @@ public struct ConversationOverrides: Sendable { self.clientEvents = clientEvents } } + +/// The network endpoints the SDK talks to. Defaults to ``production``. +/// +/// Override for proxies, regional hosts, or staging. Credentials follow these +/// endpoints — conversation tokens are sent to whichever `apiBase` is configured. +public struct Endpoints: Sendable, Equatable { + /// Base URL for the HTTP API. Request paths are appended to it. + public var apiBase: URL + /// WebSocket endpoint for voice conversations. + public var voiceWebSocket: URL + /// WebSocket endpoint for text-only conversations. + public var textWebSocket: URL + + public init( + apiBase: URL = Endpoints.production.apiBase, + voiceWebSocket: URL = Endpoints.production.voiceWebSocket, + textWebSocket: URL = Endpoints.production.textWebSocket + ) { + self.apiBase = apiBase + self.voiceWebSocket = voiceWebSocket + self.textWebSocket = textWebSocket + } + + public static let production = Endpoints( + apiBase: URL(string: "https://api.elevenlabs.io")!, + voiceWebSocket: URL(string: "wss://livekit.rtc.elevenlabs.io")!, + textWebSocket: URL(string: "wss://api.elevenlabs.io/v1/convai/conversation")! + ) + + var conversationToken: URL { + apiBase.appendingPathComponent("v1/convai/conversation/token") + } +} + +public struct ConversationStartupConfiguration: Sendable, Equatable { + public var agentReadyTimeout: TimeInterval + public var initiationMetadataTimeout: TimeInterval + + public init( + agentReadyTimeout: TimeInterval = 3.0, + initiationMetadataTimeout: TimeInterval = 5.0 + ) { + self.agentReadyTimeout = agentReadyTimeout + self.initiationMetadataTimeout = initiationMetadataTimeout + } + + public static let `default` = ConversationStartupConfiguration() +} + +/// Controls how the SDK establishes WebRTC connections. +/// +/// The default configuration gathers all ICE candidate types. Use ``Strategy/relayOnly`` +/// to restrict connections to TURN relays. +public struct WebRTCConfiguration: Sendable { + /// Describes how ICE transport candidates should be gathered. + public enum Strategy: Sendable, Equatable { + /// Gather all candidate types. + case automatic + /// Force TURN relay candidates only. + case relayOnly + } + + /// The strategy to use for ICE gathering. Defaults to ``Strategy/automatic``. + public var strategy: Strategy + + public init(strategy: Strategy = .automatic) { + self.strategy = strategy + } + + /// Default configuration using automatic ICE candidate gathering. + public static let `default` = WebRTCConfiguration() +} + +/// Configuration for event-based agent state management using VAD and client events. +/// Pass `nil` to use the default LiveKit-based behaviour. +public struct AgentStateConfiguration: Sendable { + public var vadSpeakingThreshold: Double + public var minSpeechDuration: TimeInterval + public var minSilenceDuration: TimeInterval + public var speakingToListeningDelay: TimeInterval + + public init( + vadSpeakingThreshold: Double = 0.5, + minSpeechDuration: TimeInterval = 0.15, + minSilenceDuration: TimeInterval = 0.05, + speakingToListeningDelay: TimeInterval = 0.5 + ) { + self.vadSpeakingThreshold = vadSpeakingThreshold + self.minSpeechDuration = minSpeechDuration + self.minSilenceDuration = minSilenceDuration + self.speakingToListeningDelay = speakingToListeningDelay + } + + public static let `default` = AgentStateConfiguration() +} + +/// Configures microphone pipeline and voice activity reporting exposed by the SDK. +public struct AudioPipelineConfiguration: Sendable { + /// Override the microphone mute strategy. Defaults to `.inputMixer` to match previous SDK behaviour. + public var microphoneMuteMode: MicrophoneMuteMode? + + /// Keep the recording engine warm to avoid first-spoken-word latency. Defaults to `true`. + public var recordingAlwaysPrepared: Bool? + + /// Bypass WebRTC voice processing (AEC/NS/VAD). Leave `nil` to preserve system defaults. + public var voiceProcessingBypassed: Bool? + + /// Toggle Auto Gain Control. Leave `nil` to preserve system defaults. + public var voiceProcessingAGCEnabled: Bool? + + public init( + microphoneMuteMode: MicrophoneMuteMode? = .inputMixer, + recordingAlwaysPrepared: Bool? = true, + voiceProcessingBypassed: Bool? = nil, + voiceProcessingAGCEnabled: Bool? = nil + ) { + self.microphoneMuteMode = microphoneMuteMode + self.recordingAlwaysPrepared = recordingAlwaysPrepared + self.voiceProcessingBypassed = voiceProcessingBypassed + self.voiceProcessingAGCEnabled = voiceProcessingAGCEnabled + } + + public static let `default` = AudioPipelineConfiguration() +} + +/// Strategy used when muting the local microphone. Exactly one strategy is active +/// at a time. +public enum MicrophoneMuteMode: Sendable, Equatable { + /// Mutes instantly by silencing the input mixer. The mic stays open and the + /// audio session remains active. Recommended default. + case inputMixer + + /// Mutes by restarting the engine without mic input. Releases the mic, but + /// mute/unmute is slower and speech detection is unavailable. + case restart + + /// Mutes the voice-processing input. Fast, supports + /// ``ConversationCallbacks/onSpeechDetectedWhileMuted``, and keeps the audio + /// session active. + case voiceProcessing + + /// Mutes in software by zeroing captured audio before it leaves the device. + /// Supports ``ConversationCallbacks/onSpeechDetectedWhileMuted``. + /// + /// - Parameters: + /// - speechThreshold: dB threshold for muted-speech detection. + /// - notificationThrottle: Minimum interval between muted-speech callbacks. + case software(speechThreshold: Float = -35, notificationThrottle: TimeInterval = 3.0) +} diff --git a/Sources/ElevenLabs/Public/Conversation/ConversationError.swift b/Sources/ElevenLabs/Public/Conversation/ConversationError.swift index 1646f515..4ff48d3f 100644 --- a/Sources/ElevenLabs/Public/Conversation/ConversationError.swift +++ b/Sources/ElevenLabs/Public/Conversation/ConversationError.swift @@ -2,10 +2,11 @@ import Foundation public enum ConversationError: LocalizedError, Sendable, Equatable { case notConnected - case alreadyActive + case alreadyStarted case connectionFailed(String) // Store error description instead of Error for Equatable case authenticationFailed(String) case agentTimeout + case initiationMetadataTimeout case microphoneToggleFailed(String) // Store error description instead of Error for Equatable case serverError(ErrorEvent) @@ -21,10 +22,11 @@ public enum ConversationError: LocalizedError, Sendable, Equatable { public var errorDescription: String? { switch self { case .notConnected: "Conversation is not connected." - case .alreadyActive: "Conversation is already active." + case .alreadyStarted: "Conversation has already been started." case let .connectionFailed(description): "Connection failed: \(description)" case let .authenticationFailed(msg): "Authentication failed: \(msg)" case .agentTimeout: "Agent did not join in time." + case .initiationMetadataTimeout: "Conversation metadata was not received in time." case let .microphoneToggleFailed(description): "Failed to toggle microphone: \(description)" case let .serverError(event): "Server error (\(event.code)): \(event.message ?? "unknown")" } diff --git a/Sources/ElevenLabs/Public/Conversation/Startup/ConversationStartupMetrics.swift b/Sources/ElevenLabs/Public/Conversation/ConversationStartup.swift similarity index 55% rename from Sources/ElevenLabs/Public/Conversation/Startup/ConversationStartupMetrics.swift rename to Sources/ElevenLabs/Public/Conversation/ConversationStartup.swift index 7d917d9e..489fd259 100644 --- a/Sources/ElevenLabs/Public/Conversation/Startup/ConversationStartupMetrics.swift +++ b/Sources/ElevenLabs/Public/Conversation/ConversationStartup.swift @@ -1,5 +1,17 @@ import Foundation +/// In-flight startup stage while `ConversationState` is `.connecting`. +public enum ConversationStartupState: Sendable, Equatable { + /// Session teardown / wiring before transport-specific stages begin. + case preparing + case resolvingToken + case connectingRoom + case waitingForAgent(timeout: TimeInterval) + case agentReady(elapsed: TimeInterval) + case sendingConversationInit + case waitingForInitiationMetadata(timeout: TimeInterval) +} + public struct ConversationStartupMetrics: Sendable, Equatable { public var total: TimeInterval? public var tokenFetch: TimeInterval? @@ -8,7 +20,7 @@ public struct ConversationStartupMetrics: Sendable, Equatable { public var agentReadyBuffer: TimeInterval? public var conversationInit: TimeInterval? - public var conversationInitAttempts: Int + public var initiationMetadata: TimeInterval? public init( total: TimeInterval? = nil, @@ -19,7 +31,7 @@ public struct ConversationStartupMetrics: Sendable, Equatable { agentReadyTimedOut _: Bool = false, agentReadyBuffer: TimeInterval? = nil, conversationInit: TimeInterval? = nil, - conversationInitAttempts: Int = 0 + initiationMetadata: TimeInterval? = nil ) { self.total = total self.tokenFetch = tokenFetch @@ -27,6 +39,11 @@ public struct ConversationStartupMetrics: Sendable, Equatable { self.agentReady = agentReady self.agentReadyBuffer = agentReadyBuffer self.conversationInit = conversationInit - self.conversationInitAttempts = conversationInitAttempts + self.initiationMetadata = initiationMetadata } } + +public struct ConversationStartResult: Equatable, Sendable { + public let callInfo: CallInfo + public let metrics: ConversationStartupMetrics +} diff --git a/Sources/ElevenLabs/Public/Conversation/ConversationState.swift b/Sources/ElevenLabs/Public/Conversation/ConversationState.swift index 68922063..da27667c 100644 --- a/Sources/ElevenLabs/Public/Conversation/ConversationState.swift +++ b/Sources/ElevenLabs/Public/Conversation/ConversationState.swift @@ -2,11 +2,16 @@ import Foundation public enum ConversationState: Equatable, Sendable { case idle - case connecting + case connecting(ConversationStartupState) case connected(CallInfo) case ended(reason: EndReason) case error(ConversationError) + public var isConnecting: Bool { + if case .connecting = self { return true } + return false + } + public var isConnected: Bool { if case .connected = self { return true } return false @@ -17,8 +22,23 @@ public enum ConversationState: Equatable, Sendable { return false } + var isError: Bool { + if case .error = self { return true } + return false + } + var connectedAgentId: String? { if case let .connected(info) = self { return info.agentId } return nil } } + +public struct CallInfo: Equatable, Sendable { + public let agentId: String + public let conversationId: String +} + +public enum EndReason: Equatable, Sendable { + case userEnded + case remoteDisconnected +} diff --git a/Sources/ElevenLabs/Public/ElevenLabs/Events/IncomingEvents.swift b/Sources/ElevenLabs/Public/Conversation/Events/IncomingEvents.swift similarity index 100% rename from Sources/ElevenLabs/Public/ElevenLabs/Events/IncomingEvents.swift rename to Sources/ElevenLabs/Public/Conversation/Events/IncomingEvents.swift diff --git a/Sources/ElevenLabs/Public/ElevenLabs/Events/OutgoingEvents.swift b/Sources/ElevenLabs/Public/Conversation/Events/OutgoingEvents.swift similarity index 86% rename from Sources/ElevenLabs/Public/ElevenLabs/Events/OutgoingEvents.swift rename to Sources/ElevenLabs/Public/Conversation/Events/OutgoingEvents.swift index def9de05..d4ee46ad 100644 --- a/Sources/ElevenLabs/Public/ElevenLabs/Events/OutgoingEvents.swift +++ b/Sources/ElevenLabs/Public/Conversation/Events/OutgoingEvents.swift @@ -87,6 +87,22 @@ public struct ClientToolResultEvent: Sendable { self.isError = isError || errorType != nil self.errorType = errorType } + + /// JSON-encodes `result` before creating the event. + public init( + toolCallId: String, + result: some Encodable, + isError: Bool = false, + errorType: ClientToolErrorType? = nil + ) throws { + let json = try String(decoding: JSONEncoder().encode(result), as: UTF8.self) + self.init( + toolCallId: toolCallId, + result: json, + isError: isError, + errorType: errorType + ) + } } /// Contextual update to the conversation diff --git a/Sources/ElevenLabs/Public/Conversation/Models/AgentState.swift b/Sources/ElevenLabs/Public/Conversation/Models/AgentState.swift new file mode 100644 index 00000000..35625c1a --- /dev/null +++ b/Sources/ElevenLabs/Public/Conversation/Models/AgentState.swift @@ -0,0 +1,11 @@ +import Foundation + +/// Agent state indicating what the agent is currently doing. +public enum AgentState: Sendable, Equatable { + /// Agent is listening to the user + case listening + /// Agent is speaking + case speaking + /// Agent is thinking (e.g. preparing a tool call) + case thinking +} diff --git a/Sources/ElevenLabs/Public/Conversation/Models/CallInfo.swift b/Sources/ElevenLabs/Public/Conversation/Models/CallInfo.swift deleted file mode 100644 index 3cdcdf97..00000000 --- a/Sources/ElevenLabs/Public/Conversation/Models/CallInfo.swift +++ /dev/null @@ -1,5 +0,0 @@ -import Foundation - -public struct CallInfo: Equatable, Sendable { - public let agentId: String -} diff --git a/Sources/ElevenLabs/Public/Conversation/Models/EndReason.swift b/Sources/ElevenLabs/Public/Conversation/Models/EndReason.swift deleted file mode 100644 index aca4d528..00000000 --- a/Sources/ElevenLabs/Public/Conversation/Models/EndReason.swift +++ /dev/null @@ -1,6 +0,0 @@ -import Foundation - -public enum EndReason: Equatable, Sendable { - case userEnded - case remoteDisconnected -} diff --git a/Sources/ElevenLabs/Public/Models/Language.swift b/Sources/ElevenLabs/Public/Conversation/Models/Language.swift similarity index 100% rename from Sources/ElevenLabs/Public/Models/Language.swift rename to Sources/ElevenLabs/Public/Conversation/Models/Language.swift diff --git a/Sources/ElevenLabs/Public/Conversation/Startup/ConversationStartupConfiguration.swift b/Sources/ElevenLabs/Public/Conversation/Startup/ConversationStartupConfiguration.swift deleted file mode 100644 index 42c27975..00000000 --- a/Sources/ElevenLabs/Public/Conversation/Startup/ConversationStartupConfiguration.swift +++ /dev/null @@ -1,13 +0,0 @@ -import Foundation - -public struct ConversationStartupConfiguration: Sendable, Equatable { - public var agentReadyTimeout: TimeInterval - - public init( - agentReadyTimeout: TimeInterval = 3.0 - ) { - self.agentReadyTimeout = agentReadyTimeout - } - - public static let `default` = ConversationStartupConfiguration() -} diff --git a/Sources/ElevenLabs/Public/Conversation/Startup/ConversationStartupFailure.swift b/Sources/ElevenLabs/Public/Conversation/Startup/ConversationStartupFailure.swift deleted file mode 100644 index 61df605b..00000000 --- a/Sources/ElevenLabs/Public/Conversation/Startup/ConversationStartupFailure.swift +++ /dev/null @@ -1,8 +0,0 @@ -import Foundation - -public enum ConversationStartupFailure: Sendable, Equatable { - case token(ConversationError) - case room(ConversationError) - case agentTimeout - case conversationInit(ConversationError) -} diff --git a/Sources/ElevenLabs/Public/Conversation/Startup/ConversationStartupState.swift b/Sources/ElevenLabs/Public/Conversation/Startup/ConversationStartupState.swift deleted file mode 100644 index 624b18b4..00000000 --- a/Sources/ElevenLabs/Public/Conversation/Startup/ConversationStartupState.swift +++ /dev/null @@ -1,12 +0,0 @@ -import Foundation - -public enum ConversationStartupState: Sendable, Equatable { - case idle - case resolvingToken - case connectingRoom - case waitingForAgent(timeout: TimeInterval) - case agentReady(ConversationAgentReadyReport) - case sendingConversationInit(attempt: Int) - case connected(CallInfo, ConversationStartupMetrics) - case failed(ConversationStartupFailure, ConversationStartupMetrics) -} diff --git a/Sources/ElevenLabs/Public/ConversationClient.swift b/Sources/ElevenLabs/Public/ConversationClient.swift new file mode 100644 index 00000000..f179e44b --- /dev/null +++ b/Sources/ElevenLabs/Public/ConversationClient.swift @@ -0,0 +1,248 @@ +import Combine +import Foundation + +/// The central entry point for the ElevenLabs Conversational AI SDK. +/// +/// Create one and hold it for the lifetime of your screen (e.g. a SwiftUI +/// `@StateObject`). It exposes conversation state as `@Published` properties +/// and controls as methods. Each `startConversation` call runs a fresh, +/// single-use session internally; the client itself is reusable — call +/// `startConversation` again to start another. +@MainActor +public final class ConversationClient: ObservableObject { + // MARK: - Public State + + @Published public private(set) var state: ConversationState = .idle + @Published public private(set) var messages: [Message] = [] + @Published public private(set) var agentState: AgentState = .listening + @Published public private(set) var isMicMuted: Bool = false + + /// Stream of client tool calls that need to be executed by the app + @Published public private(set) var pendingToolCalls: [ClientToolCallEvent] = [] + + /// Conversation metadata including conversation ID, received when the conversation is initialized + @Published public private(set) var conversationMetadata: ConversationMetadataEvent? + + /// MCP tool calls from the agent + @Published public private(set) var mcpToolCalls: [MCPToolCallEvent] = [] + + /// Current MCP connection status for all integrations + @Published public private(set) var mcpConnectionStatus: MCPConnectionStatusEvent? + + // MARK: - Init + + private let callbacks: ConversationCallbacks + private let dependencyProvider: (any ConversationDependencyProvider)? + private let logLevel: LogLevel + + /// The current single-use session. Internal plumbing — never exposed. + private var session: Conversation? + /// Subscriptions mirroring the active session's state; reset on each `startConversation`. + private var cancellables = Set() + + /// Durable observers re-attached to every session this client starts. + private var agentAudioObservers: [any ConversationAudioObserver] = [] + private var micAudioObservers: [any ConversationAudioObserver] = [] + + public init(callbacks: ConversationCallbacks = .init(), logLevel: LogLevel = .warning) { + self.callbacks = callbacks + self.logLevel = logLevel + dependencyProvider = nil + } + + /// Test-only initializer that injects a dependency provider. + init(callbacks: ConversationCallbacks = .init(), dependencyProvider: any ConversationDependencyProvider) { + self.callbacks = callbacks + logLevel = .warning + self.dependencyProvider = dependencyProvider + } + + // MARK: - Lifecycle + + /// Start a conversation with an ElevenLabs agent using a public agent ID - the most common use case. + public func startConversation( + agentId: String, + config: ConversationConfig = .init() + ) async throws -> ConversationStartResult { + let authConfig = ConversationCredentials.publicAgent(id: agentId, environment: config.environment) + return try await startConversation(auth: authConfig, config: config) + } + + /// Start a conversation using a conversation token from your backend - for private agents. + public func startConversation( + conversationToken: String, + config: ConversationConfig = .init() + ) async throws -> ConversationStartResult { + let authConfig = ConversationCredentials.conversationToken(conversationToken, environment: config.environment) + return try await startConversation(auth: authConfig, config: config) + } + + /// Start a conversation using a custom token provider - for advanced authentication scenarios. + public func startConversation( + tokenProvider: @escaping @Sendable () async throws -> String, + config: ConversationConfig = .init() + ) async throws -> ConversationStartResult { + let authConfig = ConversationCredentials.customTokenProvider(tokenProvider, environment: config.environment) + return try await startConversation(auth: authConfig, config: config) + } + + /// Start a text-only conversation using a signed WebSocket URL from your backend. + public func startConversation( + signedWebSocketURL: String, + config: ConversationConfig = .init(conversationOverrides: .init(textOnly: true)) + ) async throws -> ConversationStartResult { + let authConfig = try ConversationCredentials.signedWebSocketURL(signedWebSocketURL) + var updatedConfig = config + updatedConfig.conversationOverrides.textOnly = true + return try await startConversation(auth: authConfig, config: updatedConfig) + } + + /// Advanced: start a conversation with full authentication control. + /// + /// Any previously-started session still running is ended first, then a fresh + /// single-use session is created and connected. + public func startConversation( + auth: ConversationCredentials, + config: ConversationConfig = .init() + ) async throws -> ConversationStartResult { + let previousConversation = session + let conversation = Conversation( + dependencyProvider: dependencyProvider ?? Dependencies(logLevel: logLevel, endpoints: config.endpoints), + config: config, + callbacks: callbacks, + initialMicMuted: isMicMuted + ) + bind(conversation) + + await previousConversation?.endConversation() + return try await conversation.start(auth: auth) + } + + /// End the current conversation, if any. Mirrored state (messages, etc.) is kept + /// so the UI can still show the last session until `reset()` or a new start. + public func endConversation() async { + await session?.endConversation() + } + + /// End any live session and clear all mirrored state back to idle defaults. + /// Use this when dismissing a screen or starting over with a blank client. + public func reset() async { + await session?.endConversation() + cancellables.removeAll() + session = nil + + state = .idle + messages = [] + agentState = .listening + isMicMuted = false + pendingToolCalls = [] + conversationMetadata = nil + mcpToolCalls = [] + mcpConnectionStatus = nil + } + + /// Mirror the new session's `@Published` state onto this object. + private func bind(_ session: Conversation) { + cancellables.removeAll() + self.session = session + + session.$state.sink { [weak self] in self?.state = $0 }.store(in: &cancellables) + session.$messages.sink { [weak self] in self?.messages = $0 }.store(in: &cancellables) + session.$agentState.sink { [weak self] in self?.agentState = $0 }.store(in: &cancellables) + session.$pendingToolCalls.sink { [weak self] in self?.pendingToolCalls = $0 }.store(in: &cancellables) + session.$conversationMetadata.sink { [weak self] in self?.conversationMetadata = $0 }.store(in: &cancellables) + session.$mcpToolCalls.sink { [weak self] in self?.mcpToolCalls = $0 }.store(in: &cancellables) + session.$mcpConnectionStatus.sink { [weak self] in self?.mcpConnectionStatus = $0 }.store(in: &cancellables) + + // Re-attach durable observers to the new session. + agentAudioObservers.forEach(session.addAgentAudioObserver) + micAudioObservers.forEach(session.addMicAudioObserver) + } + + private func requireSession() throws -> Conversation { + guard let session else { throw ConversationError.notConnected } + return session + } + + // MARK: - Messaging + + /// Send a text message to the agent. + public func sendMessage(_ text: String) async throws { + try await requireSession().sendMessage(text) + } + + /// Interrupt the agent while it is speaking. + public func interruptAgent() async throws { + try await requireSession().interruptAgent() + } + + /// Send a silent contextual update to the agent (no user-visible message). + public func updateContext(_ context: String) async throws { + try await requireSession().updateContext(context) + } + + // MARK: - Microphone + + /// Set the microphone mute state for the current or next conversation. + public func setMicMuted(_ muted: Bool) async throws { + try await session?.setMicMuted(muted) + isMicMuted = muted + } + + // MARK: - Audio observers + + /// Register an observer for the agent's decoded output audio. + /// + /// The observer is durable: it is re-attached to every session this client + /// starts. ``ConversationAudioObserver/didReceive(_:)`` is called off the + /// main actor on a time-critical audio callback path. + public func addAgentAudioObserver(_ observer: any ConversationAudioObserver) { + if !agentAudioObservers.contains(where: { $0 === observer }) { + agentAudioObservers.append(observer) + } + session?.addAgentAudioObserver(observer) + } + + /// Unregister a previously added agent audio observer. + public func removeAgentAudioObserver(_ observer: any ConversationAudioObserver) { + agentAudioObservers.removeAll { $0 === observer } + session?.removeAgentAudioObserver(observer) + } + + /// Register a durable observer for the local microphone input. + public func addMicAudioObserver(_ observer: any ConversationAudioObserver) { + if !micAudioObservers.contains(where: { $0 === observer }) { + micAudioObservers.append(observer) + } + session?.addMicAudioObserver(observer) + } + + /// Unregister a previously added mic audio observer. + public func removeMicAudioObserver(_ observer: any ConversationAudioObserver) { + micAudioObservers.removeAll { $0 === observer } + session?.removeMicAudioObserver(observer) + } + + // MARK: - Tools & feedback + + /// Send in-conversation feedback (like/dislike) for an agent message. + public func sendFeedback(_ score: FeedbackEvent.Score, eventId: Int) async throws { + try await requireSession().sendFeedback(score, eventId: eventId) + } + + /// Approve or reject an MCP tool-call request from the agent. + public func sendMCPToolApproval(toolCallId: String, isApproved: Bool) async throws { + try await requireSession().sendMCPToolApproval(toolCallId: toolCallId, isApproved: isApproved) + } + + /// Send the result of a client tool call back to the agent. + public func sendToolResult(_ result: ClientToolResultEvent) async throws { + try await requireSession().sendToolResult(result) + } + + /// Mark a tool call as completed without sending a result. A best-effort + /// no-op when there is no live session. + public func markToolCallCompleted(_ toolCallId: String) { + session?.markToolCallCompleted(toolCallId) + } +} diff --git a/Sources/ElevenLabs/Public/ElevenLabs/ElevenLabs+AgentState.swift b/Sources/ElevenLabs/Public/ElevenLabs/ElevenLabs+AgentState.swift deleted file mode 100644 index 52ac3a5f..00000000 --- a/Sources/ElevenLabs/Public/ElevenLabs/ElevenLabs+AgentState.swift +++ /dev/null @@ -1,13 +0,0 @@ -import Foundation - -extension ElevenLabs { - /// Agent state indicating what the agent is currently doing. - public enum AgentState: Sendable, Equatable { - /// Agent is listening to the user - case listening - /// Agent is speaking - case speaking - /// Agent is thinking (e.g. preparing a tool call) - case thinking - } -} diff --git a/Sources/ElevenLabs/Public/ElevenLabs/ElevenLabs+Configuration.swift b/Sources/ElevenLabs/Public/ElevenLabs/ElevenLabs+Configuration.swift deleted file mode 100644 index 6deb3a36..00000000 --- a/Sources/ElevenLabs/Public/ElevenLabs/ElevenLabs+Configuration.swift +++ /dev/null @@ -1,40 +0,0 @@ -import Foundation - -extension ElevenLabs { - /// Global SDK configuration. - public struct Configuration: Sendable { - public let apiEndpoint: URL? - public let websocketUrl: String? - public let logLevel: LogLevel - public let debugMode: Bool - - public init( - apiEndpoint: URL? = nil, - websocketUrl: String? = nil, - logLevel: LogLevel = .warning, - debugMode: Bool = false - ) { - self.apiEndpoint = apiEndpoint - self.websocketUrl = websocketUrl - self.logLevel = logLevel - self.debugMode = debugMode - } - - public static let `default` = Configuration() - - /// Create a new configuration with updated values (builder pattern) - public func with( - apiEndpoint: URL? = nil, - websocketUrl: String? = nil, - logLevel: LogLevel? = nil, - debugMode: Bool? = nil - ) -> Configuration { - Configuration( - apiEndpoint: apiEndpoint ?? self.apiEndpoint, - websocketUrl: websocketUrl ?? self.websocketUrl, - logLevel: logLevel ?? self.logLevel, - debugMode: debugMode ?? self.debugMode - ) - } - } -} diff --git a/Sources/ElevenLabs/Public/ElevenLabs/ElevenLabs+LogLevel.swift b/Sources/ElevenLabs/Public/ElevenLabs/ElevenLabs+LogLevel.swift deleted file mode 100644 index ca17c226..00000000 --- a/Sources/ElevenLabs/Public/ElevenLabs/ElevenLabs+LogLevel.swift +++ /dev/null @@ -1,16 +0,0 @@ -import Foundation - -extension ElevenLabs { - /// Logging level for SDK internal diagnostics - public enum LogLevel: Int, Comparable, Sendable { - case error = 0 - case warning = 1 - case info = 2 - case debug = 3 - case trace = 4 - - public static func < (lhs: LogLevel, rhs: LogLevel) -> Bool { - lhs.rawValue < rhs.rawValue - } - } -} diff --git a/Sources/ElevenLabs/Public/ElevenLabs/ElevenLabs.swift b/Sources/ElevenLabs/Public/ElevenLabs/ElevenLabs.swift deleted file mode 100644 index df702bc8..00000000 --- a/Sources/ElevenLabs/Public/ElevenLabs/ElevenLabs.swift +++ /dev/null @@ -1,217 +0,0 @@ -import Foundation -import LiveKit - -// Main namespace & entry point for the ElevenLabs Conversational AI SDK. -// -// ```swift -// // Start a conversation directly - simple and clean -// let conversation = try await ElevenLabs.startConversation( -// agentId: "agent_123", -// config: .init(conversationOverrides: .init(textOnly: false)) -// ) -// -// // Send a message -// try await conversation.sendMessage("Hello!") -// -// // End the conversation -// await conversation.endConversation() -// ``` - -public enum ElevenLabs { - // MARK: - Version - - public static let version = "3.2.2" - - // MARK: - Configuration - - /// Global, optional SDK configuration. Provide once at app start. - /// If you never call `configure(_:)`, sensible defaults are used. - @MainActor - public static func configure(_ configuration: Configuration) { - Global.shared.configuration = configuration - } - - // MARK: - SDK interface - - /// Start a conversation with an ElevenLabs agent using a public agent ID - the most common use case. - /// - /// This method handles all the complexity of connection setup, authentication, - /// and protocol initialization. Simply provide a public agent ID and optional configuration. - /// - /// - Parameters: - /// - agentId: The public ElevenLabs agent ID to connect to - /// - config: Optional conversation configuration (voice/text mode, overrides, etc.) - /// - callbacks: Optional event hooks (onAgentReady, onDisconnect, etc.), set once for this conversation - /// - Returns: An active `Conversation` instance ready for interaction - /// - Throws: `ConversationError` if connection fails, agent not found, or configuration invalid - /// - /// ```swift - /// // Voice conversation (default) - simplest usage - /// let conversation = try await ElevenLabs.startConversation(agentId: "agent_123") - /// - /// // Text-only conversation - /// let textConversation = try await ElevenLabs.startConversation( - /// agentId: "agent_123", - /// config: .init(conversationOverrides: .init(textOnly: true)) - /// ) - /// - /// // Conversation with event handlers - /// let conversation = try await ElevenLabs.startConversation( - /// agentId: "agent_123", - /// callbacks: .init( - /// onAgentReady: { - /// print("Agent is ready!") - /// }, - /// onDisconnect: { reason in - /// print("Agent disconnected: \(reason)") - /// } - /// ) - /// ) - /// ``` - @MainActor - public static func startConversation( - agentId: String, - config: ConversationConfig = .init(), - callbacks: ConversationCallbacks = .init() - ) async throws -> Conversation { - let authConfig = ConversationCredentials.publicAgent(id: agentId, environment: config.environment) - return try await startConversation(auth: authConfig, config: config, callbacks: callbacks) - } - - /// Start a conversation using a conversation token from your backend - for private agents. - /// - /// Use this method when you have private agents that require authentication. - /// Your backend should generate conversation tokens using your ElevenLabs API key. - /// - /// Security: Never include your ElevenLabs API key in client apps! - /// - /// - Parameters: - /// - conversationToken: The conversation token from your backend - /// - config: Optional conversation configuration (voice/text mode, overrides, etc.) - /// - callbacks: Optional event hooks (onAgentReady, onDisconnect, etc.), set once for this conversation - /// - Returns: An active `Conversation` instance ready for interaction - /// - Throws: `ConversationError` if connection fails or token is invalid - /// - /// ```swift - /// // Get token from your backend - /// let token = try await fetchTokenFromMyBackend() - /// - /// // Start conversation with private agent - /// let conversation = try await ElevenLabs.startConversation( - /// conversationToken: token, - /// config: .init( - /// agentOverrides: .init(firstMessage: "Hello! How can I help you today?") - /// ) - /// ) - /// ``` - @MainActor - public static func startConversation( - conversationToken: String, - config: ConversationConfig = .init(), - callbacks: ConversationCallbacks = .init() - ) async throws -> Conversation { - let authConfig = ConversationCredentials.conversationToken(conversationToken, environment: config.environment) - return try await startConversation(auth: authConfig, config: config, callbacks: callbacks) - } - - /// Start a conversation using a custom token provider - for advanced authentication scenarios. - /// - /// Use this method when you need dynamic token generation or complex authentication flows. - /// - /// - Parameters: - /// - tokenProvider: An async closure that returns a conversation token - /// - config: Optional conversation configuration (voice/text mode, overrides, etc.) - /// - callbacks: Optional event hooks (onAgentReady, onDisconnect, etc.), set once for this conversation - /// - Returns: An active `Conversation` instance ready for interaction - /// - Throws: `ConversationError` if connection fails or token provider throws - /// - /// ```swift - /// // Dynamic token provider - /// let conversation = try await ElevenLabs.startConversation( - /// tokenProvider: { - /// // Your custom authentication logic - /// let userAuth = try await authenticateUser() - /// return try await fetchElevenLabsToken(for: userAuth) - /// }, - /// config: .init(conversationOverrides: .init(textOnly: false)) - /// ) - /// ``` - @MainActor - public static func startConversation( - tokenProvider: @escaping @Sendable () async throws -> String, - config: ConversationConfig = .init(), - callbacks: ConversationCallbacks = .init() - ) async throws -> Conversation { - let authConfig = ConversationCredentials.customTokenProvider(tokenProvider, environment: config.environment) - return try await startConversation(auth: authConfig, config: config, callbacks: callbacks) - } - - /// Start a text-only conversation using a signed WebSocket URL from your backend. - /// - /// Use this for private/non-public agents in text-only mode. Signed URLs should be generated - /// server-side using your ElevenLabs API key. - @MainActor - public static func startConversation( - signedWebSocketURL: String, - config: ConversationConfig = .init(conversationOverrides: .init(textOnly: true)), - callbacks: ConversationCallbacks = .init() - ) async throws -> Conversation { - let authConfig = try ConversationCredentials.signedWebSocketURL(signedWebSocketURL) - var updatedConfig = config - updatedConfig.conversationOverrides.textOnly = true - return try await startConversation(auth: authConfig, config: updatedConfig, callbacks: callbacks) - } - - /// Advanced: Start a conversation with full authentication control. - /// - /// This is the most flexible method that all other convenience methods use internally. - /// Most developers should use the simpler `startConversation(agentId:)` method instead. - /// - /// - Parameters: - /// - auth: The authentication configuration - /// - config: Optional conversation configuration - /// - Returns: An active `Conversation` instance ready for interaction - @MainActor - public static func startConversation( - auth: ConversationCredentials, - config: ConversationConfig = .init(), - callbacks: ConversationCallbacks = .init() - ) async throws -> Conversation { - let conversation = createConversation(config: config, callbacks: callbacks) - try await conversation.startConversation( - auth: auth, config: config - ) - return conversation - } - - // MARK: - Internal Factory Methods - - /// Creates a new Conversation instance with proper dependency injection. - @MainActor - private static func createConversation( - config: ConversationConfig = .init(), - callbacks: ConversationCallbacks = .init() - ) -> Conversation { - Conversation(dependencyProvider: Dependencies(), config: config, callbacks: callbacks) - } - - // MARK: - Re-exports - - // Re-export audio track types for advanced audio handling - public typealias LocalAudioTrack = LiveKit.LocalAudioTrack - public typealias RemoteAudioTrack = LiveKit.RemoteAudioTrack - public typealias AudioTrack = LiveKit.AudioTrack - - // Language enum is already public and accessible as ElevenLabs.Language - - // MARK: - Internal Global State - - /// Internal container for global (process-wide) configuration. - /// This mimics the old `Dependencies` singleton but keeps it internal. - @MainActor - final class Global { - static let shared = Global() - var configuration: Configuration = .default - private init() {} - } -} diff --git a/Sources/ElevenLabs/Public/LiveKit/WebRTCConfiguration.swift b/Sources/ElevenLabs/Public/LiveKit/WebRTCConfiguration.swift deleted file mode 100644 index 27434213..00000000 --- a/Sources/ElevenLabs/Public/LiveKit/WebRTCConfiguration.swift +++ /dev/null @@ -1,23 +0,0 @@ -/// Controls how the SDK establishes WebRTC connections. -/// -/// The default configuration gathers all ICE candidate types. Use ``Strategy/relayOnly`` -/// to restrict connections to TURN relays. -public struct WebRTCConfiguration: Sendable { - /// Describes how ICE transport candidates should be gathered. - public enum Strategy: Sendable, Equatable { - /// Gather all candidate types. - case automatic - /// Force TURN relay candidates only. - case relayOnly - } - - /// The strategy to use for ICE gathering. Defaults to ``Strategy/automatic``. - public var strategy: Strategy - - public init(strategy: Strategy = .automatic) { - self.strategy = strategy - } - - /// Default configuration using automatic ICE candidate gathering. - public static let `default` = WebRTCConfiguration() -} diff --git a/Sources/ElevenLabs/Public/LogLevel.swift b/Sources/ElevenLabs/Public/LogLevel.swift new file mode 100644 index 00000000..3065906b --- /dev/null +++ b/Sources/ElevenLabs/Public/LogLevel.swift @@ -0,0 +1,14 @@ +import Foundation + +/// Logging level for SDK internal diagnostics +public enum LogLevel: Int, Comparable, Sendable { + case error = 0 + case warning = 1 + case info = 2 + case debug = 3 + case trace = 4 + + public static func < (lhs: LogLevel, rhs: LogLevel) -> Bool { + lhs.rawValue < rhs.rawValue + } +} diff --git a/Sources/ElevenLabs/Internal/Version.swift b/Sources/ElevenLabs/Public/Version.swift similarity index 67% rename from Sources/ElevenLabs/Internal/Version.swift rename to Sources/ElevenLabs/Public/Version.swift index 60bb96ae..af513997 100644 --- a/Sources/ElevenLabs/Internal/Version.swift +++ b/Sources/ElevenLabs/Public/Version.swift @@ -2,6 +2,4 @@ // Run Scripts/generate-version.sh to update import Foundation -enum SDKVersion { - static let version = "3.2.2" -} +public let version = "3.2.2" diff --git a/Sources/ElevenLabsWidget/Audio/ConversationAudioLevelMonitor.swift b/Sources/ElevenLabsWidget/Audio/ConversationAudioLevelMonitor.swift new file mode 100644 index 00000000..21c33fc0 --- /dev/null +++ b/Sources/ElevenLabsWidget/Audio/ConversationAudioLevelMonitor.swift @@ -0,0 +1,62 @@ +import Accelerate +@preconcurrency import AVFoundation +import ElevenLabs + +/// Tracks a normalized loudness level for one conversation audio stream. +/// +/// Pull-based: `didReceive` runs on the audio thread and only holds the loudest +/// level seen. Reading drains it, which is also what paces the decay, so the +/// level keeps falling once a stream goes quiet and stops delivering buffers. +final class ConversationAudioLevelMonitor: ConversationAudioObserver, @unchecked Sendable { + /// Fraction of the level kept per read: fast attack, slow release. + private static let release: Float = 0.75 + + private let lock = NSLock() + private var storedLevel: Float = 0 + /// Only touched from the audio thread, which delivers buffers serially. + private var scratch: [Float] = [] + + /// The loudest level since the last read, leaving one release step behind. + func sample() -> Float { + lock.withLock { + defer { storedLevel *= Self.release } + return storedLevel + } + } + + func didReceive(_ buffer: AVAudioPCMBuffer) { + guard let level = normalizedRMS(of: buffer) else { return } + lock.withLock { storedLevel = max(level, storedLevel) } + } + + func reset() { + lock.withLock { storedLevel = 0 } + } + + /// RMS of the first channel, mapped from the top 60 dB of headroom onto 0...1. + private func normalizedRMS(of buffer: AVAudioPCMBuffer) -> Float? { + let frames = Int(buffer.frameLength) + guard frames > 0 else { return nil } + let stride = buffer.format.isInterleaved ? vDSP_Stride(buffer.format.channelCount) : 1 + + var rms: Float = 0 + if let channel = buffer.floatChannelData?[0] { + vDSP_rmsqv(channel, stride, &rms, vDSP_Length(frames)) + } else if let channel = buffer.int16ChannelData?[0] { + // Grown, never reallocated per buffer: this is the audio thread. + if scratch.count < frames { scratch = [Float](repeating: 0, count: frames) } + scratch.withUnsafeMutableBufferPointer { floats in + guard let samples = floats.baseAddress else { return } + vDSP_vflt16(channel, stride, samples, 1, vDSP_Length(frames)) + var scale = 1 / Float(Int16.max) + vDSP_vsmul(samples, 1, &scale, samples, 1, vDSP_Length(frames)) + vDSP_rmsqv(samples, 1, &rms, vDSP_Length(frames)) + } + } else { + return nil + } + + let decibels = 20 * log10(max(rms, 0.000_001)) + return min(max((decibels + 60) / 60, 0), 1) + } +} diff --git a/Sources/ElevenLabsWidget/Audio/OrbAudioLevels.swift b/Sources/ElevenLabsWidget/Audio/OrbAudioLevels.swift new file mode 100644 index 00000000..4bcc384f --- /dev/null +++ b/Sources/ElevenLabsWidget/Audio/OrbAudioLevels.swift @@ -0,0 +1,58 @@ +import Combine +import Foundation + +/// Samples the mic and agent level monitors at a display-friendly rate. +/// +/// The monitors are updated at PCM rate on the audio thread; publishing from +/// there would re-render SwiftUI thousands of times a second, so the levels are +/// polled on a timer while a conversation is live instead. +@MainActor +final class OrbAudioLevels: ObservableObject { + @Published private(set) var input: Float = 0 + @Published private(set) var output: Float = 0 + + let micMonitor = ConversationAudioLevelMonitor() + let agentMonitor = ConversationAudioLevelMonitor() + + private static let interval: TimeInterval = 1.0 / 30 + private var timer: Timer? + + var isActive: Bool = false { + didSet { + guard isActive != oldValue else { return } + isActive ? start() : stop() + } + } + + deinit { timer?.invalidate() } + + private func start() { + // Clears anything the audio thread wrote after the last stop, which would + // otherwise surface as leftover loudness from the previous call. + micMonitor.reset() + agentMonitor.reset() + let timer = Timer(timeInterval: Self.interval, repeats: true) { [weak self] _ in + MainActor.assumeIsolated { self?.sample() } + } + // Common mode, so the levels keep updating while the transcript scrolls. + RunLoop.main.add(timer, forMode: .common) + self.timer = timer + } + + private func stop() { + timer?.invalidate() + timer = nil + micMonitor.reset() + agentMonitor.reset() + input = 0 + output = 0 + } + + /// Only publishes on change, so a silent conversation doesn't redraw the orb. + private func sample() { + let mic = micMonitor.sample() + let agent = agentMonitor.sample() + if mic != input { input = mic } + if agent != output { output = agent } + } +} diff --git a/Sources/ElevenLabsWidget/ChatWidgetController.swift b/Sources/ElevenLabsWidget/ChatWidgetController.swift new file mode 100644 index 00000000..b303b059 --- /dev/null +++ b/Sources/ElevenLabsWidget/ChatWidgetController.swift @@ -0,0 +1,80 @@ +#if canImport(UIKit) +import ElevenLabs +import Foundation + +/// Host-owned handle for observing widget state and driving it from outside. +/// +/// Optional — pass one to ``ChatWidget/init(authProvider:widgetConfig:conversationConfig:controller:onClientToolCall:)`` +/// to read state through the `@Published` mirrors and to issue commands. +/// +/// The controller may outlive the widget: once the widget view is gone, commands +/// become no-ops and the mirrors stop updating. Mirrors are read-only from the +/// host so state only ever changes through a command. +@available(iOS 16, macCatalyst 16, *) +@MainActor +public final class ChatWidgetController: ObservableObject { + @Published public internal(set) var state: ConversationState = .idle + @Published public internal(set) var isMicMuted = false + @Published public internal(set) var isOpen = false + /// Server-assigned conversation id; `nil` until the agent confirms the session. + @Published public internal(set) var conversationId: String? + @Published public internal(set) var messageCount = 0 + + public init() {} + + public func open() { + binding?.open() + } + + public func close() { + binding?.close() + } + + public func toggleOpen() { + binding?.toggleOpen() + } + + /// Start a conversation, awaiting the connection. No-op if one is live. + public func startConversation() async throws { + try await binding?.startConversationAndWait() + } + + public func endConversation() async { + await binding?.client.endConversation() + } + + /// Send a message as the user, connecting first if needed. + public func sendMessage(_ text: String) async throws { + try await binding?.send(text) + } + + public func setMicMuted(_ muted: Bool) async throws { + try await binding?.client.setMicMuted(muted) + } + + /// Snapshot of the conversation's messages. Not reactive — read it when you + /// need it (e.g. on end of call). + public func messages() -> [Message] { + binding?.client.messages ?? [] + } + + /// Set by the widget on attach. Weak so the controller can outlive the widget. + weak var binding: ChatWidgetViewModel? +} + +@available(iOS 16, macCatalyst 16, *) +extension ChatWidgetViewModel { + func attach(to controller: ChatWidgetController) { + guard controller.binding !== self else { return } + controller.binding = self + + $conversationState.assign(to: &controller.$state) + $isMicMuted.assign(to: &controller.$isMicMuted) + $isOpen.assign(to: &controller.$isOpen) + $messages.map(\.count).assign(to: &controller.$messageCount) + client.$conversationMetadata + .map { $0?.conversationId } + .assign(to: &controller.$conversationId) + } +} +#endif diff --git a/Sources/ElevenLabsWidget/Configuration/ChatWidgetConfig.swift b/Sources/ElevenLabsWidget/Configuration/ChatWidgetConfig.swift new file mode 100644 index 00000000..85023876 --- /dev/null +++ b/Sources/ElevenLabsWidget/Configuration/ChatWidgetConfig.swift @@ -0,0 +1,31 @@ +#if canImport(UIKit) +import Foundation + +/// Behavior and appearance of ``ChatWidget``. +public struct ChatWidgetConfig: Equatable, Sendable { + /// Whether the user talks, types, or both. + public var conversationMode: WidgetConversationMode + /// Dim the host UI behind the open drawer. + public var showBackdrop: Bool + /// Show the microphone mute button while a conversation is live. + public var enableMicMuteControl: Bool + public var strings: ChatWidgetStrings + public var theme: ChatWidgetTheme + + public init( + conversationMode: WidgetConversationMode = .voiceAndText, + showBackdrop: Bool = true, + enableMicMuteControl: Bool = true, + strings: ChatWidgetStrings = .default, + theme: ChatWidgetTheme = .default + ) { + self.conversationMode = conversationMode + self.showBackdrop = showBackdrop + self.enableMicMuteControl = enableMicMuteControl + self.strings = strings + self.theme = theme + } + + public static let `default` = ChatWidgetConfig() +} +#endif diff --git a/Sources/ElevenLabsWidget/Configuration/ChatWidgetStrings.swift b/Sources/ElevenLabsWidget/Configuration/ChatWidgetStrings.swift new file mode 100644 index 00000000..5f1cca1a --- /dev/null +++ b/Sources/ElevenLabsWidget/Configuration/ChatWidgetStrings.swift @@ -0,0 +1,44 @@ +#if canImport(UIKit) +import Foundation + +/// User-facing copy for ``ChatWidget``. Override individual fields to localize +/// or reword; every field has an English default. +public struct ChatWidgetStrings: Equatable, Sendable { + public var title: String + public var mainLabel: String + public var inputPlaceholder: String + public var openChatLabel: String + public var closeChatLabel: String + public var sendMessageLabel: String + public var startConversationLabel: String + public var endConversationLabel: String + public var muteMicrophoneLabel: String + public var unmuteMicrophoneLabel: String + + public init( + title: String = "Chat", + mainLabel: String = "Powered by ElevenLabs", + inputPlaceholder: String = "Type a message…", + openChatLabel: String = "Open chat", + closeChatLabel: String = "Close chat", + sendMessageLabel: String = "Send message", + startConversationLabel: String = "Start voice conversation", + endConversationLabel: String = "End conversation", + muteMicrophoneLabel: String = "Mute microphone", + unmuteMicrophoneLabel: String = "Unmute microphone" + ) { + self.title = title + self.mainLabel = mainLabel + self.inputPlaceholder = inputPlaceholder + self.openChatLabel = openChatLabel + self.closeChatLabel = closeChatLabel + self.sendMessageLabel = sendMessageLabel + self.startConversationLabel = startConversationLabel + self.endConversationLabel = endConversationLabel + self.muteMicrophoneLabel = muteMicrophoneLabel + self.unmuteMicrophoneLabel = unmuteMicrophoneLabel + } + + public static let `default` = ChatWidgetStrings() +} +#endif diff --git a/Sources/ElevenLabsWidget/Configuration/ChatWidgetTheme.swift b/Sources/ElevenLabsWidget/Configuration/ChatWidgetTheme.swift new file mode 100644 index 00000000..4baa739d --- /dev/null +++ b/Sources/ElevenLabsWidget/Configuration/ChatWidgetTheme.swift @@ -0,0 +1,53 @@ +#if canImport(UIKit) +import SwiftUI + +/// Visual styling for ``ChatWidget``. Defaults mirror the ElevenLabs web widget. +public struct ChatWidgetTheme: Equatable, Sendable { + /// Hairline borders around buttons and the composer. + public var border: Color + /// Saturated red used for the end-call glyph. + public var destructive: Color + /// Soft fill behind the end-call button. + public var destructiveTint: Color + /// Primary (darker) color in the orb gradient. + public var orbPrimary: Color + /// Secondary (lighter) color in the orb gradient. + public var orbSecondary: Color + + public init( + border: Color = ChatWidgetTheme.default.border, + destructive: Color = ChatWidgetTheme.default.destructive, + destructiveTint: Color = ChatWidgetTheme.default.destructiveTint, + orbPrimary: Color = ChatWidgetTheme.default.orbPrimary, + orbSecondary: Color = ChatWidgetTheme.default.orbSecondary + ) { + self.border = border + self.destructive = destructive + self.destructiveTint = destructiveTint + self.orbPrimary = orbPrimary + self.orbSecondary = orbSecondary + } + + public static let `default` = ChatWidgetTheme( + border: Color(hex: 0xE1E1E1), + destructive: Color(hex: 0xFF1900), + destructiveTint: Color(hex: 0xFDE4E3), + orbPrimary: Color(hex: 0x2792DC), + orbSecondary: Color(hex: 0x9CE6E6) + ) +} + +extension Color { + /// Build a `Color` from a 24-bit `0xRRGGBB` value, matching the hex strings + /// used by the web widget config (e.g. `#2792dc` → `0x2792DC`). + init(hex: UInt32) { + self.init( + .sRGB, + red: Double((hex >> 16) & 0xFF) / 255, + green: Double((hex >> 8) & 0xFF) / 255, + blue: Double(hex & 0xFF) / 255, + opacity: 1 + ) + } +} +#endif diff --git a/Sources/ElevenLabsWidget/Configuration/WidgetConversationMode.swift b/Sources/ElevenLabsWidget/Configuration/WidgetConversationMode.swift new file mode 100644 index 00000000..b9201bfe --- /dev/null +++ b/Sources/ElevenLabsWidget/Configuration/WidgetConversationMode.swift @@ -0,0 +1,25 @@ +#if canImport(UIKit) +import Foundation + +/// Which ways the user can talk to the agent through ``ChatWidget``. +public enum WidgetConversationMode: String, CaseIterable, Identifiable, Sendable { + /// Typed messages only; the conversation runs without audio. + case textOnly + /// A voice call with no composer. + case voiceOnly + /// A voice call the user can also type into. + case voiceAndText + + public var id: String { + rawValue + } + + public var supportsVoice: Bool { + self != .textOnly + } + + public var supportsTextInput: Bool { + self != .voiceOnly + } +} +#endif diff --git a/Sources/ElevenLabsWidget/Models/ChatMessage.swift b/Sources/ElevenLabsWidget/Models/ChatMessage.swift new file mode 100644 index 00000000..3e0b1a66 --- /dev/null +++ b/Sources/ElevenLabsWidget/Models/ChatMessage.swift @@ -0,0 +1,21 @@ +#if canImport(UIKit) +import ElevenLabs +import Foundation + +/// A transcript bubble, projected from the SDK's ``Message``. +struct ChatMessage: Identifiable, Equatable { + enum Role: Equatable { case user, agent } + + let id: String + let role: Role + let content: String + let timestamp: Date + + init(_ message: Message) { + id = message.id + role = message.role == .user ? .user : .agent + content = message.content + timestamp = message.timestamp + } +} +#endif diff --git a/Sources/ElevenLabsWidget/Rendering/Orb.swift b/Sources/ElevenLabsWidget/Rendering/Orb.swift new file mode 100644 index 00000000..bea09afd --- /dev/null +++ b/Sources/ElevenLabsWidget/Rendering/Orb.swift @@ -0,0 +1,288 @@ +// Ported from ElevenLabs components-swift (OrbVisualizer.swift), trimmed to the +// volume-driven orb; the LiveKit track-based visualizer is not carried over. + +#if canImport(UIKit) +import Foundation +import MetalKit +import simd +import SwiftUI +import UIKit + +/// CPU-side uniforms must match `OrbUniforms` in `OrbShader.metal` byte‑for‑byte. +/// Stride = 96 bytes. +struct OrbUniforms { + var time: Float = 0 + var animation: Float = 0 + var inverted: Float = 0 + var _pad0: Float = 0 // 16‑byte align + var offsets: simd_float8 = .zero // only first 7 used + var color1: simd_float4 = .zero + var color2: simd_float4 = .zero + var inputVolume: Float = 0 + var outputVolume: Float = 0 + var _pad1: SIMD2 = .zero // to 96 bytes + + init() {} +} + +/// Convert SwiftUI `Color` -> linear‑space simd_float4. +@inline(__always) +@available(iOS 14, macCatalyst 14, *) +private func colorToSIMD4(_ color: Color) -> simd_float4 { + let ui = UIColor(color) + var r: CGFloat = 0, g: CGFloat = 0, b: CGFloat = 0, a: CGFloat = 1 + ui.getRed(&r, green: &g, blue: &b, alpha: &a) + func sRGBToLinear(_ v: CGFloat) -> Float { + if v <= 0.04045 { return Float(v / 12.92) } + return Float(pow((v + 0.055) / 1.055, 2.4)) + } + return .init(sRGBToLinear(r), sRGBToLinear(g), sRGBToLinear(b), Float(a)) +} + +/// Shared Metal renderer backing the SwiftUI representables. +@available(iOS 14, macCatalyst 14, *) +class MetalOrbRenderer: NSObject, MTKViewDelegate { + private let device: MTLDevice + private let commandQueue: MTLCommandQueue + private var pipeline: MTLRenderPipelineState! + private var vertexBuffer: MTLBuffer! + + private var animationTime: Float = 0 + /// Wall-clock-driven time fed to the shader. Unlike `CACurrentMediaTime` + /// it only advances while the orb is active, so a `.disconnected` orb + /// freezes instead of swirling as if it were live. + private var displayTime: Float = 0 + private var lastDrawTime: CFTimeInterval = CACurrentMediaTime() + + private var uniforms = OrbUniforms() + private var randomOffsets: [Float] = [] + private var currentAgentState: VisualizerAgentState = .unknown + + // MARK: - Init + + override init() { + guard let d = MTLCreateSystemDefaultDevice(), let q = d.makeCommandQueue() else { + fatalError("Metal not available") + } + device = d + commandQueue = q + super.init() + generateRandomOffsets() + buildBuffers() + buildPipeline() + } + + // MARK: - Public updaters + + func updateColors(color1: Color, color2: Color) { + uniforms.color1 = colorToSIMD4(color1) + uniforms.color2 = colorToSIMD4(color2) + } + + func updateVolumes(input: Float, output: Float) { + uniforms.inputVolume = max(0, min(1, input)) + uniforms.outputVolume = max(0, min(1, output)) + } + + func updateAgentState(_ state: VisualizerAgentState) { + // No longer inverting colors for thinking state + uniforms.inverted = 0 + currentAgentState = state + } + + // MARK: - MTKViewDelegate + + func mtkView(_: MTKView, drawableSizeWillChange _: CGSize) {} + + func draw(in view: MTKView) { + guard let drawable = view.currentDrawable, + let rpd = view.currentRenderPassDescriptor, + let cmd = commandQueue.makeCommandBuffer(), + let enc = cmd.makeRenderCommandEncoder(descriptor: rpd) else { return } + + let now = CACurrentMediaTime() + let fps = max(view.preferredFramesPerSecond, 1) + // Capped at one frame: an idle orb stops drawing, and the gap must not + // land on the clock as a jump when the next call resumes it. + let dt = min(Float(now - lastDrawTime), 1 / Float(fps)) + lastDrawTime = now + + // A disconnected orb is idle: freeze both animation clocks so it stops + // swirling (otherwise it looks "active" / live even after a call ends). + let isIdle = currentAgentState == .disconnected + if !isIdle { + // Slow down animation when thinking (0.02x speed instead of 0.1x) + let animationSpeed: Float = currentAgentState == .thinking ? 0.02 : 0.1 + animationTime += (1.0 / Float(fps)) * animationSpeed + displayTime += dt + } + uniforms.time = displayTime + uniforms.animation = animationTime + uniforms.offsets = simd_float8(randomOffsets + [0]) + + enc.setRenderPipelineState(pipeline) + enc.setVertexBuffer(vertexBuffer, offset: 0, index: 0) + + var u = uniforms + if isIdle { + // Don't let any residual level keep the idle orb pulsing. + u.inputVolume = 0 + u.outputVolume = 0 + } + enc.setFragmentBytes(&u, length: MemoryLayout.stride, index: 0) + + enc.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4) + enc.endEncoding() + cmd.present(drawable) + cmd.commit() + } + + // MARK: - Private + + private func generateRandomOffsets() { + randomOffsets = (0 ..< 7).map { _ in Float.random(in: 0 ... (Float.pi * 2)) } + } + + private func buildBuffers() { + // full‑screen quad + let verts: [Float] = [ + -1, 1, + -1, -1, + 1, 1, + 1, -1 + ] + vertexBuffer = device.makeBuffer(bytes: verts, length: verts.count * MemoryLayout.size, options: []) + } + + private func buildPipeline() { + // Try to load the Metal library from various sources + var lib: MTLLibrary? + + // For Swift Package Manager, try Bundle.module first (contains package resources) + #if SWIFT_PACKAGE + lib = try? device.makeDefaultLibrary(bundle: Bundle.module) + #endif + + // Try default library (works when Metal files are in the main target) + if lib == nil { + lib = device.makeDefaultLibrary() + } + + // If that fails, try the class bundle + if lib == nil { + lib = try? device.makeDefaultLibrary(bundle: Bundle(for: type(of: self))) + } + + // If not found, try the main bundle + if lib == nil { + lib = try? device.makeDefaultLibrary(bundle: .main) + } + + guard let library = lib else { + fatalError("Unable to load Metal library – ensure OrbShader.metal is included in the target") + } + + guard let vfn = library.makeFunction(name: "orbVertexShader"), + let ffn = library.makeFunction(name: "orbFragmentShader") + else { + fatalError("Unable to find shader functions in Metal library") + } + + let desc = MTLRenderPipelineDescriptor() + desc.vertexFunction = vfn + desc.fragmentFunction = ffn + desc.colorAttachments[0].pixelFormat = .bgra8Unorm + + do { + pipeline = try device.makeRenderPipelineState(descriptor: desc) + } catch { + fatalError("Orb pipeline creation failed: \(error)") + } + } +} + +@available(iOS 14, macCatalyst 14, *) +struct OrbMetalView: UIViewRepresentable { + var color1: Color + var color2: Color + var inputVolume: Float + var outputVolume: Float + var agentState: VisualizerAgentState + + func makeUIView(context: Context) -> MTKView { + let view = MTKView() + view.device = MTLCreateSystemDefaultDevice() + view.delegate = context.coordinator + configure(view: view) + context.coordinator.updateAll(color1: color1, color2: color2, input: inputVolume, output: outputVolume, state: agentState) + return view + } + + func updateUIView(_ view: MTKView, context: Context) { + context.coordinator.updateAll(color1: color1, color2: color2, input: inputVolume, output: outputVolume, state: agentState) + setIdle(isIdle, on: view) + } + + /// A disconnected, silent orb renders a still frame, so the launcher isn't + /// driving the GPU at 60fps while it sits in the host's UI. + private var isIdle: Bool { + agentState == .disconnected && inputVolume == 0 && outputVolume == 0 + } + + private func setIdle(_ idle: Bool, on view: MTKView) { + view.isPaused = idle + // On demand rather than never, so layout changes still get a frame. + view.enableSetNeedsDisplay = idle + if idle { view.setNeedsDisplay() } + } + + func makeCoordinator() -> Coordinator { + Coordinator() + } + + private func configure(view: MTKView) { + view.framebufferOnly = false + view.isPaused = false + view.enableSetNeedsDisplay = false + view.preferredFramesPerSecond = 60 + view.clearColor = .init(red: 0, green: 0, blue: 0, alpha: 0) + view.colorPixelFormat = .bgra8Unorm + view.autoResizeDrawable = true + } + + final class Coordinator: MetalOrbRenderer { + func updateAll(color1: Color, color2: Color, input: Float, output: Float, state: VisualizerAgentState) { + updateColors(color1: color1, color2: color2) + updateVolumes(input: input, output: output) + updateAgentState(state) + } + } +} + +@available(iOS 14, macCatalyst 14, *) +struct Orb: View { + var color1: Color + var color2: Color + var inputVolume: Float + var outputVolume: Float + var agentState: VisualizerAgentState = .unknown + + var body: some View { + GeometryReader { geo in + let side = max(1, min(geo.size.width, geo.size.height)) + OrbMetalView( + color1: color1, + color2: color2, + inputVolume: inputVolume, + outputVolume: outputVolume, + agentState: agentState + ) + .frame(width: side, height: side) + .clipShape(Circle()) + .frame(maxWidth: .infinity, maxHeight: .infinity) + .accessibilityLabel(Text("Orb visualizer")) + } + .aspectRatio(1, contentMode: .fit) + } +} +#endif diff --git a/Sources/ElevenLabsWidget/Rendering/VisualizerAgentState.swift b/Sources/ElevenLabsWidget/Rendering/VisualizerAgentState.swift new file mode 100644 index 00000000..49cb7c50 --- /dev/null +++ b/Sources/ElevenLabsWidget/Rendering/VisualizerAgentState.swift @@ -0,0 +1,23 @@ +#if canImport(UIKit) +import Foundation + +/// A richer agent state enum specifically for visualizer animations. +/// This enum preserves the full set of states needed for detailed visual feedback, +/// independent of the simplified SDK's `AgentState` enum. +enum VisualizerAgentState: Sendable, Equatable { + /// Agent is connecting to the session + case connecting + /// Agent is initializing + case initializing + /// Agent is listening to user input + case listening + /// Agent is processing/thinking + case thinking + /// Agent is speaking + case speaking + /// Agent is disconnected + case disconnected + /// Unknown or unspecified state + case unknown +} +#endif diff --git a/Sources/ElevenLabsWidget/Resources/OrbShader.metal b/Sources/ElevenLabsWidget/Resources/OrbShader.metal new file mode 100644 index 00000000..65201325 --- /dev/null +++ b/Sources/ElevenLabsWidget/Resources/OrbShader.metal @@ -0,0 +1,203 @@ +#include +using namespace metal; + +constant float PI = 3.14159265358979323846; + +struct VertexOut { + float4 position [[position]]; + float2 uv; +}; + +struct OrbUniforms { + float time; + float animation; + float inverted; + float _pad0; // padding to 16 bytes alignment + float offsets[8]; // 8 offsets for alignment + float4 color1; + float4 color2; + float inputVolume; + float outputVolume; + float2 _pad1; // 8 bytes padding to reach 96 bytes +}; + +vertex VertexOut orbVertexShader(uint vertexID [[vertex_id]], + constant float2* vertices [[buffer(0)]]) { + VertexOut out; + float2 pos = vertices[vertexID]; + out.position = float4(pos, 0.0, 1.0); + out.uv = pos * 0.5 + 0.5; // Convert from [-1,1] to [0,1] + return out; +} + +float2 hash2(float2 p) { + return fract(sin(float2(dot(p, float2(127.1, 311.7)), dot(p, float2(269.5, 183.3)))) * 43758.5453); +} + +float noise2D(float2 p) { + float2 i = floor(p); + float2 f = fract(p); + + float2 u = f * f * (3.0 - 2.0 * f); + float n = mix( + mix(dot(hash2(i + float2(0.0, 0.0)), f - float2(0.0, 0.0)), + dot(hash2(i + float2(1.0, 0.0)), f - float2(1.0, 0.0)), u.x), + mix(dot(hash2(i + float2(0.0, 1.0)), f - float2(0.0, 1.0)), + dot(hash2(i + float2(1.0, 1.0)), f - float2(1.0, 1.0)), u.x), + u.y + ); + + return 0.5 + 0.5 * n; +} + +float perlinTexture(float2 uv) { + return noise2D(uv * 8.0); +} + +bool drawOval(float2 polarUv, float2 polarCenter, float a, float b, bool reverseGradient, float softness, thread float4& color) { + float2 p = polarUv - polarCenter; + float oval = (p.x * p.x) / (a * a) + (p.y * p.y) / (b * b); + + float edge = smoothstep(1.0, 1.0 - softness, oval); + + if (edge > 0.0) { + float gradient = reverseGradient ? (1.0 - (p.x / a + 1.0) / 2.0) : ((p.x / a + 1.0) / 2.0); + color = float4(float3(gradient), min(1.2 * edge, 1.0)); + return true; + } + return false; +} + +float3 colorRamp(float grayscale, float3 color1, float3 color2, float3 color3, float3 color4) { + if (grayscale < 0.33) { + return mix(color1, color2, grayscale * 3.0); + } else if (grayscale < 0.66) { + return mix(color2, color3, (grayscale - 0.33) * 3.0); + } else { + return mix(color3, color4, (grayscale - 0.66) * 3.0); + } +} + +float sharpRing(float3 decomposed, float time) { + float ringStart = 1.0; + float ringWidth = 0.5; + float noiseScale = 5.0; + + float noise = mix( + noise2D(float2(decomposed.x, time) * noiseScale), + noise2D(float2(decomposed.y, time) * noiseScale), + decomposed.z + ); + + noise = (noise - 0.5) * 4.0; + + return ringStart + noise * ringWidth * 1.5; +} + +float smoothRing(float3 decomposed, float time) { + float ringStart = 0.9; + float ringWidth = 0.3; + float noiseScale = 6.0; + + float noise = mix( + noise2D(float2(decomposed.x, time) * noiseScale), + noise2D(float2(decomposed.y, time) * noiseScale), + decomposed.z + ); + + noise = (noise - 0.5) * 8.0; + + return ringStart + noise * ringWidth; +} + +float flow(float3 decomposed, float time) { + return mix( + perlinTexture(float2(time, decomposed.x / 2.0)), + perlinTexture(float2(time, decomposed.y / 2.0)), + decomposed.z + ); +} + +fragment float4 orbFragmentShader(VertexOut in [[stage_in]], + constant OrbUniforms& uniforms [[buffer(0)]]) { + + float2 uv = in.uv * 2.0 - 1.0; + + float radius = length(uv); + float theta = atan2(uv.y, uv.x); + if (theta < 0.0) theta += 2.0 * PI; + + float3 decomposed = float3( + theta / (2.0 * PI), + fmod(theta / (2.0 * PI) + 0.5, 1.0) + 1.0, + abs(theta / PI - 1.0) + ); + + float noise = flow(decomposed, radius * 0.03 - uniforms.animation * 0.2) - 0.5; + theta += noise * mix(0.5, 1.0, uniforms.outputVolume); + + float4 color = float4(1.0, 1.0, 1.0, 1.0); + + float originalCenters[7] = {0.0, 0.5 * PI, 1.0 * PI, 1.5 * PI, 2.0 * PI, 2.5 * PI, 3.0 * PI}; + + float centers[7]; + for (int i = 0; i < 7; i++) { + centers[i] = originalCenters[i] + 0.5 * sin(uniforms.time / 20.0 + uniforms.offsets[i]); + } + + float a, b; + float4 ovalColor; + + for (int i = 0; i < 7; i++) { + float noise = perlinTexture(float2(fmod(centers[i] + uniforms.time * 0.05, 1.0), 0.5)); + a = 0.5 + noise * 0.5; // Increased variance: goes from 0.0 to 1.0 + b = noise * mix(4.5, 3.0, uniforms.inputVolume); // Tall semi-minor axis + bool reverseGradient = (i % 2 == 1); // Reverse gradient for every second oval + + // Calculate the distance in polar coordinates + float distTheta = min( + abs(theta - centers[i]), + min( + abs(theta + 2.0 * PI - centers[i]), + abs(theta - 2.0 * PI - centers[i]) + ) + ); + float distRadius = radius; + + float softness = 0.4; // Controls edge softness + + if (drawOval(float2(distTheta, distRadius), float2(0.0, 0.0), a, b, reverseGradient, softness, ovalColor)) { + color.rgb = mix(color.rgb, ovalColor.rgb, ovalColor.a); + } + } + + float ringRadius1 = sharpRing(decomposed, uniforms.time * 0.1); + float ringRadius2 = smoothRing(decomposed, uniforms.time * 0.1); + + float inputRadius1 = radius + uniforms.inputVolume * 0.3; + float inputRadius2 = radius + uniforms.inputVolume * 0.2; + float opacity1 = mix(0.3, 0.8, uniforms.inputVolume); + float opacity2 = mix(0.25, 0.6, uniforms.inputVolume); + + float ringAlpha1 = (inputRadius2 >= ringRadius1) ? opacity1 : 0.0; + float ringAlpha2 = smoothstep(ringRadius2 - 0.05, ringRadius2 + 0.05, inputRadius1) * opacity2; + + float totalRingAlpha = max(ringAlpha1, ringAlpha2); + + float3 ringColor = float3(1.0); + color.rgb = 1.0 - (1.0 - color.rgb) * (1.0 - ringColor * totalRingAlpha); + + float3 color1 = float3(0.0, 0.0, 0.0); // Black + float3 color2 = uniforms.color1.xyz; // Darker Color + float3 color3 = uniforms.color2.xyz; // Lighter Color + float3 color4 = float3(1.0, 1.0, 1.0); // White + + // Convert grayscale color to the color ramp + float luminance = mix(color.r, 1.0 - color.r, uniforms.inverted); + color.rgb = colorRamp(luminance, color1, color2, color3, color4); + + // Always fully opaque for the orb + color.a = 1.0; + + return color; +} diff --git a/Sources/ElevenLabsWidget/ViewModels/ChatWidgetViewModel.swift b/Sources/ElevenLabsWidget/ViewModels/ChatWidgetViewModel.swift new file mode 100644 index 00000000..13e7ad67 --- /dev/null +++ b/Sources/ElevenLabsWidget/ViewModels/ChatWidgetViewModel.swift @@ -0,0 +1,228 @@ +#if canImport(UIKit) +import Combine +import ElevenLabs +import Foundation +import SwiftUI +import UIKit + +@available(iOS 16, macCatalyst 16, *) +@MainActor +final class ChatWidgetViewModel: ObservableObject { + @Published var isOpen = false + @Published var input = "" + @Published private(set) var messages: [ChatMessage] = [] + @Published private(set) var conversationState: ConversationState = .idle + @Published private(set) var agentState: AgentState = .listening + @Published private(set) var isMicMuted = false + @Published private(set) var isSending = false + + /// Mirrors the config the host passes to ``ChatWidget``, so changes take + /// effect without recreating the conversation. + @Published var widgetConfig: ChatWidgetConfig + let client: ConversationClient + let audioLevels = OrbAudioLevels() + + private let authProvider: () async throws -> ConversationCredentials + private let conversationConfig: ConversationConfig + private let onClientToolCall: (@MainActor (ClientToolCallEvent) async -> ClientToolResultEvent)? + /// Tool calls already dispatched, so a re-published snapshot doesn't run them twice. + private var dispatchedToolCallIds: Set = [] + /// The in-flight startup, so overlapping callers join it instead of racing. + private var startTask: Task? + private var cancellables = Set() + + init( + authProvider: @escaping () async throws -> ConversationCredentials, + widgetConfig: ChatWidgetConfig, + client: ConversationClient, + conversationConfig: ConversationConfig, + onClientToolCall: (@MainActor (ClientToolCallEvent) async -> ClientToolResultEvent)? + ) { + self.authProvider = authProvider + self.widgetConfig = widgetConfig + self.client = client + self.conversationConfig = conversationConfig + self.onClientToolCall = onClientToolCall + + // The client is durable across sessions, so every subscription and + // observer is wired once. + client.addMicAudioObserver(audioLevels.micMonitor) + client.addAgentAudioObserver(audioLevels.agentMonitor) + + client.$state.assign(to: &$conversationState) + client.$state + .sink { [weak self] in self?.audioLevels.isActive = $0.isConnected } + .store(in: &cancellables) + client.$agentState.assign(to: &$agentState) + client.$isMicMuted.assign(to: &$isMicMuted) + client.$messages + .map { $0.map(ChatMessage.init) } + .assign(to: &$messages) + client.$pendingToolCalls + .sink { [weak self] in self?.dispatchNewToolCalls($0) } + .store(in: &cancellables) + } + + /// The host can drop the widget mid-call, and nothing else owns the client, + /// so the session and the microphone would otherwise stay live. + deinit { + let client = client + Task { @MainActor in await client.endConversation() } + } + + // MARK: - Derived state + + var hasActiveConversation: Bool { + conversationState.isConnecting || conversationState.isConnected + } + + var canSend: Bool { + !isSending && !input.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + } + + var canToggleMicMute: Bool { + widgetConfig.enableMicMuteControl && mode.supportsVoice && conversationState.isConnected + } + + var canShowTextInput: Bool { + mode.supportsTextInput + } + + var canStartVoiceConversation: Bool { + mode.supportsVoice && !hasActiveConversation + } + + private var mode: WidgetConversationMode { + widgetConfig.conversationMode + } + + /// Text-only widgets run the conversation without audio. + private var effectiveConversationConfig: ConversationConfig { + guard !mode.supportsVoice else { return conversationConfig } + var config = conversationConfig + config.conversationOverrides.textOnly = true + return config + } + + var orbState: ChatOrbState { + switch conversationState { + case .idle, .ended, .error: .disconnected + case .connecting: .connecting + case .connected: + switch agentState { + case .speaking: .speaking + case .thinking: .thinking + case .listening: .listening + } + } + } + + // MARK: - Presentation + + func open() { + setOpen(true) + } + + func close() { + setOpen(false) + } + + func toggleOpen() { + setOpen(!isOpen) + } + + /// Every open and close goes through here, including the host's, so the + /// drawer animates and the keyboard retracts with it either way. + private func setOpen(_ open: Bool) { + guard open != isOpen else { return } + if !open { + UIApplication.shared.sendAction( + #selector(UIResponder.resignFirstResponder), to: nil, from: nil, for: nil + ) + } + withAnimation(.spring(response: 0.42, dampingFraction: 0.82)) { isOpen = open } + } + + // MARK: - Conversation + + func startConversation() { + Task { try? await startConversationAndWait() } + } + + func startConversationAndWait() async throws { + // Joining first matters while connecting: returning early there would let + // a send reach the agent before the session is up. + if let startTask { return try await startTask.value } + guard !hasActiveConversation else { return } + let task = Task { + _ = try await client.startConversation(auth: authProvider(), config: effectiveConversationConfig) + } + startTask = task + defer { startTask = nil } + try await task.value + } + + func endConversation() { + Task { await client.endConversation() } + } + + func send() { + let text = input.trimmingCharacters(in: .whitespacesAndNewlines) + guard !text.isEmpty, !isSending else { return } + input = "" + isSending = true + Task { + do { + try await send(text) + } catch { + // Hand the text back so the user can retry rather than losing it. + if input.isEmpty { input = text } + } + isSending = false + } + } + + func send(_ text: String) async throws { + try await startConversationAndWait() + try await client.sendMessage(text) + } + + func toggleMicMute() { + Task { try? await client.setMicMuted(!isMicMuted) } + } + + // MARK: - Client tools + + private func dispatchNewToolCalls(_ calls: [ClientToolCallEvent]) { + let ids = Set(calls.map(\.toolCallId)) + let added = ids.subtracting(dispatchedToolCallIds) + dispatchedToolCallIds = ids + for call in calls where added.contains(call.toolCallId) { + dispatch(call) + } + } + + private func dispatch(_ call: ClientToolCallEvent) { + guard let onClientToolCall else { + respond(to: call, with: .init( + toolCallId: call.toolCallId, + result: "No client tool handler is configured in the app.", + isError: true + )) + return + } + Task { @MainActor [weak self] in + let result = await onClientToolCall(call) + self?.respond(to: call, with: result) + } + } + + private func respond(to call: ClientToolCallEvent, with result: ClientToolResultEvent) { + guard call.expectsResponse else { + client.markToolCallCompleted(call.toolCallId) + return + } + Task { try? await client.sendToolResult(result) } + } +} +#endif diff --git a/Sources/ElevenLabsWidget/Views/ChatInputBar.swift b/Sources/ElevenLabsWidget/Views/ChatInputBar.swift new file mode 100644 index 00000000..06df005a --- /dev/null +++ b/Sources/ElevenLabsWidget/Views/ChatInputBar.swift @@ -0,0 +1,103 @@ +#if canImport(UIKit) +import SwiftUI + +/// Composer plus the mic / start / end / send controls. +@available(iOS 16, macCatalyst 16, *) +struct ChatInputBar: View { + @ObservedObject var vm: ChatWidgetViewModel + var isInputFocused: FocusState.Binding + + private var strings: ChatWidgetStrings { + vm.widgetConfig.strings + } + + private var theme: ChatWidgetTheme { + vm.widgetConfig.theme + } + + var body: some View { + VStack(spacing: 10) { + if vm.canShowTextInput { + TextField(strings.inputPlaceholder, text: $vm.input, axis: .vertical) + .textFieldStyle(.plain) + .lineLimit(1 ... 3) + .padding(.horizontal, 8) + .frame(minHeight: 36) + .focused(isInputFocused) + .onSubmit(vm.send) + } + + HStack(spacing: 10) { + if vm.canToggleMicMute { + ChatMicButton(vm: vm, levels: vm.audioLevels, diameter: 38, theme: theme) + } + // Without a composer there is nothing to push the call controls + // away from, so they sit centered instead. + Spacer(minLength: 0) + if vm.hasActiveConversation { + endConversationButton + } else if vm.canStartVoiceConversation { + startConversationButton + } + if vm.canShowTextInput { + sendButton + } + if !vm.canShowTextInput { + Spacer(minLength: 0) + } + } + } + .padding(.horizontal, 14) + .padding(.vertical, 12) + .background( + RoundedRectangle(cornerRadius: 28, style: .continuous) + .fill(Color(.systemBackground)) + .shadow(color: .black.opacity(0.08), radius: 8, y: 2) + ) + .overlay( + RoundedRectangle(cornerRadius: 28, style: .continuous) + .strokeBorder(theme.border, lineWidth: 1) + ) + .contentShape(Rectangle()) + .onTapGesture { if vm.canShowTextInput { isInputFocused.wrappedValue = true } } + } + + private var startConversationButton: some View { + Button(action: vm.startConversation) { + circle(fill: Color(.systemBackground), glyph: "phone.fill", tint: .black) + .overlay(Circle().strokeBorder(theme.border, lineWidth: 1)) + } + .buttonStyle(.plain) + .accessibilityLabel(strings.startConversationLabel) + } + + private var endConversationButton: some View { + Button(action: vm.endConversation) { + circle(fill: theme.destructiveTint, glyph: "phone.down.fill", tint: theme.destructive) + } + .buttonStyle(.plain) + .accessibilityLabel(strings.endConversationLabel) + } + + private var sendButton: some View { + Button(action: vm.send) { + circle( + fill: vm.canSend ? Color.black : Color.secondary.opacity(0.4), + glyph: "paperplane.fill", + tint: .white + ) + } + .buttonStyle(.plain) + .disabled(!vm.canSend) + .accessibilityLabel(strings.sendMessageLabel) + } + + private func circle(fill: Color, glyph: String, tint: Color) -> some View { + Image(systemName: glyph) + .font(.system(size: 15, weight: .semibold)) + .foregroundColor(tint) + .frame(width: 38, height: 38) + .background(Circle().fill(fill)) + } +} +#endif diff --git a/Sources/ElevenLabsWidget/Views/ChatMicButton.swift b/Sources/ElevenLabsWidget/Views/ChatMicButton.swift new file mode 100644 index 00000000..57785524 --- /dev/null +++ b/Sources/ElevenLabsWidget/Views/ChatMicButton.swift @@ -0,0 +1,49 @@ +#if canImport(UIKit) +import SwiftUI + +/// Circular microphone toggle with a live input-level fill and a muted slash. +@available(iOS 16, macCatalyst 16, *) +struct ChatMicButton: View { + @ObservedObject var vm: ChatWidgetViewModel + /// Observed directly so the fill tracks the mic while redrawing only this button. + @ObservedObject var levels: OrbAudioLevels + let diameter: CGFloat + var theme: ChatWidgetTheme = .default + + private var micLevel: CGFloat { + vm.isMicMuted ? 0 : CGFloat(levels.input) + } + + var body: some View { + Button(action: vm.toggleMicMute) { + ZStack { + Circle().fill(Color(.systemBackground)) + + GeometryReader { geometry in + VStack(spacing: 0) { + Spacer(minLength: 0) + Color.accentColor + .opacity(0.18 + micLevel * 0.25) + .frame(height: geometry.size.height * micLevel) + } + } + .clipShape(Circle()) + + Circle().strokeBorder(theme.border, lineWidth: 1) + + Image(systemName: vm.isMicMuted ? "mic.slash.fill" : "mic.fill") + .font(.system(size: diameter * 0.42)) + .foregroundColor(.black) + } + .frame(width: diameter, height: diameter) + .animation(.linear(duration: 0.06), value: micLevel) + } + .buttonStyle(.plain) + .accessibilityLabel( + vm.isMicMuted + ? vm.widgetConfig.strings.unmuteMicrophoneLabel + : vm.widgetConfig.strings.muteMicrophoneLabel + ) + } +} +#endif diff --git a/Sources/ElevenLabsWidget/Views/ChatOrbView.swift b/Sources/ElevenLabsWidget/Views/ChatOrbView.swift new file mode 100644 index 00000000..8a3417c7 --- /dev/null +++ b/Sources/ElevenLabsWidget/Views/ChatOrbView.swift @@ -0,0 +1,42 @@ +#if canImport(UIKit) +import SwiftUI + +/// The audio-reactive orb, driven by the sampled mic and agent levels. +@available(iOS 16, macCatalyst 16, *) +struct ChatOrbView: View { + /// Observed here so level changes redraw only the orb, not the whole widget. + @ObservedObject var levels: OrbAudioLevels + let state: ChatOrbState + let size: CGFloat + var theme: ChatWidgetTheme = .default + + var body: some View { + Orb( + color1: theme.orbPrimary, + color2: theme.orbSecondary, + inputVolume: levels.input, + outputVolume: levels.output, + agentState: state.visualizerState + ) + .frame(width: size, height: size) + } +} + +enum ChatOrbState { + case connecting + case listening + case thinking + case speaking + case disconnected + + var visualizerState: VisualizerAgentState { + switch self { + case .connecting: .connecting + case .listening: .listening + case .thinking: .thinking + case .speaking: .speaking + case .disconnected: .disconnected + } + } +} +#endif diff --git a/Sources/ElevenLabsWidget/Views/ChatPopupView.swift b/Sources/ElevenLabsWidget/Views/ChatPopupView.swift new file mode 100644 index 00000000..f49d4c4f --- /dev/null +++ b/Sources/ElevenLabsWidget/Views/ChatPopupView.swift @@ -0,0 +1,70 @@ +#if canImport(UIKit) +import SwiftUI + +/// The drawer: header with the orb, transcript, composer. +@available(iOS 16, macCatalyst 16, *) +struct ChatPopupView: View { + @ObservedObject var vm: ChatWidgetViewModel + let onClose: () -> Void + + @FocusState private var isInputFocused: Bool + + private var strings: ChatWidgetStrings { + vm.widgetConfig.strings + } + + var body: some View { + VStack(spacing: 0) { + header + Divider() + if vm.messages.isEmpty { + Spacer(minLength: 0) + orb(size: 128) + Spacer(minLength: 0) + } else { + // A voice-only drawer has no composer to anchor it, so the orb + // stays visible above the transcript. + if !vm.canShowTextInput { + orb(size: 96).padding(.vertical, 12) + } + ChatTranscriptView(vm: vm) + } + ChatInputBar(vm: vm, isInputFocused: $isInputFocused) + .padding(.horizontal, 10) + .padding(.top, 10) + Text(strings.mainLabel) + .font(.caption2) + .foregroundColor(.secondary) + .padding(.vertical, 6) + } + .background(Color(.systemBackground)) + .clipShape(RoundedRectangle(cornerRadius: 28, style: .continuous)) + .shadow(color: .black.opacity(0.16), radius: 20, y: 6) + .padding(.horizontal, 8) + .padding(.bottom, 8) + } + + private func orb(size: CGFloat) -> some View { + ChatOrbView(levels: vm.audioLevels, state: vm.orbState, size: size, theme: vm.widgetConfig.theme) + } + + private var header: some View { + HStack(spacing: 12) { + orb(size: 36) + Text(strings.title) + .font(.headline) + Spacer(minLength: 0) + Button(action: onClose) { + Image(systemName: "xmark") + .font(.system(size: 14, weight: .semibold)) + .foregroundColor(.secondary) + .frame(width: 32, height: 32) + } + .buttonStyle(.plain) + .accessibilityLabel(strings.closeChatLabel) + } + .padding(.horizontal, 16) + .padding(.vertical, 12) + } +} +#endif diff --git a/Sources/ElevenLabsWidget/Views/ChatTranscriptView.swift b/Sources/ElevenLabsWidget/Views/ChatTranscriptView.swift new file mode 100644 index 00000000..ac37866f --- /dev/null +++ b/Sources/ElevenLabsWidget/Views/ChatTranscriptView.swift @@ -0,0 +1,27 @@ +#if canImport(UIKit) +import SwiftUI + +@available(iOS 16, macCatalyst 16, *) +struct ChatTranscriptView: View { + @ObservedObject var vm: ChatWidgetViewModel + + var body: some View { + ScrollViewReader { proxy in + ScrollView { + LazyVStack(spacing: 10) { + ForEach(vm.messages) { message in + MessageBubble(message: message) + .id(message.id) + } + } + .padding(.horizontal, 16) + .padding(.vertical, 12) + } + .onChange(of: vm.messages.last?.content) { _ in + guard let last = vm.messages.last else { return } + withAnimation(.easeOut(duration: 0.2)) { proxy.scrollTo(last.id, anchor: .bottom) } + } + } + } +} +#endif diff --git a/Sources/ElevenLabsWidget/Views/ChatWidget.swift b/Sources/ElevenLabsWidget/Views/ChatWidget.swift new file mode 100644 index 00000000..45521201 --- /dev/null +++ b/Sources/ElevenLabsWidget/Views/ChatWidget.swift @@ -0,0 +1,89 @@ +#if canImport(UIKit) +import ElevenLabs +import SwiftUI + +/// A drop-in chat widget: a launcher in the corner of your UI that opens a +/// drawer with the agent conversation. +/// +/// Overlay it on your own content, typically in a `ZStack`: +/// +/// ```swift +/// ChatWidget(authProvider: { .publicAgent(id: "agent_id") }) +/// ``` +@available(iOS 16, macCatalyst 16, *) +public struct ChatWidget: View { + @StateObject private var vm: ChatWidgetViewModel + private let controller: ChatWidgetController? + private let widgetConfig: ChatWidgetConfig + private let launcher: (() -> AnyView)? + + /// - Parameters: + /// - authProvider: Called before each conversation starts, so short-lived + /// tokens can be minted on demand. + /// - controller: Optional handle for driving the widget from the host. + /// - launcher: Replaces the default orb launcher. + /// - onClientToolCall: Handles client tool calls from the agent. + @MainActor public init( + authProvider: @escaping () async throws -> ConversationCredentials, + widgetConfig: ChatWidgetConfig = .default, + conversationConfig: ConversationConfig = .init(), + controller: ChatWidgetController? = nil, + launcher: (() -> AnyView)? = nil, + onClientToolCall: (@MainActor (ClientToolCallEvent) async -> ClientToolResultEvent)? = nil + ) { + self.controller = controller + self.widgetConfig = widgetConfig + self.launcher = launcher + // Built inside the autoclosure so SwiftUI only creates it once, rather + // than on every re-init of this view. + _vm = StateObject(wrappedValue: ChatWidgetViewModel( + authProvider: authProvider, + widgetConfig: widgetConfig, + client: ConversationClient(), + conversationConfig: conversationConfig, + onClientToolCall: onClientToolCall + )) + } + + public var body: some View { + ZStack(alignment: .bottomTrailing) { + if vm.widgetConfig.showBackdrop { + // Always mounted so hit-testing switches off the instant we close; + // a removal transition would keep swallowing taps while fading out. + Color.black + .opacity(vm.isOpen ? 0.2 : 0) + .ignoresSafeArea() + .allowsHitTesting(vm.isOpen) + .onTapGesture(perform: vm.close) + } + + if vm.isOpen { + ChatPopupView(vm: vm, onClose: vm.close) + .padding(.top, 64) + .transition(.move(edge: .bottom).combined(with: .opacity)) + } else { + launcherView + .padding(16) + .transition(.scale.combined(with: .opacity)) + } + } + // Attaching seeds the controller's mirrors, so it has to happen outside a + // view update — otherwise the host's controller invalidates the view that + // is still being built. + .task { if let controller { vm.attach(to: controller) } } + // The view model outlives every re-init, so config changes are pushed to it. + .onChange(of: widgetConfig) { vm.widgetConfig = $0 } + } + + @ViewBuilder + private var launcherView: some View { + if let launcher { + Button(action: vm.toggleOpen) { launcher() } + .buttonStyle(.plain) + .accessibilityLabel(vm.widgetConfig.strings.openChatLabel) + } else { + FloatingChatButton(vm: vm, onTap: vm.toggleOpen) + } + } +} +#endif diff --git a/Sources/ElevenLabsWidget/Views/FloatingChatButton.swift b/Sources/ElevenLabsWidget/Views/FloatingChatButton.swift new file mode 100644 index 00000000..20cc224d --- /dev/null +++ b/Sources/ElevenLabsWidget/Views/FloatingChatButton.swift @@ -0,0 +1,24 @@ +#if canImport(UIKit) +import SwiftUI + +/// Default launcher: a mini orb in the corner of the host UI. +@available(iOS 16, macCatalyst 16, *) +struct FloatingChatButton: View { + @ObservedObject var vm: ChatWidgetViewModel + let onTap: () -> Void + + var body: some View { + Button(action: onTap) { + ChatOrbView( + levels: vm.audioLevels, + state: vm.orbState, + size: 58, + theme: vm.widgetConfig.theme + ) + .shadow(color: .black.opacity(0.18), radius: 10, y: 4) + } + .buttonStyle(.plain) + .accessibilityLabel(vm.widgetConfig.strings.openChatLabel) + } +} +#endif diff --git a/Sources/ElevenLabsWidget/Views/MessageBubble.swift b/Sources/ElevenLabsWidget/Views/MessageBubble.swift new file mode 100644 index 00000000..c0496a5c --- /dev/null +++ b/Sources/ElevenLabsWidget/Views/MessageBubble.swift @@ -0,0 +1,24 @@ +#if canImport(UIKit) +import SwiftUI + +@available(iOS 16, macCatalyst 16, *) +struct MessageBubble: View { + let message: ChatMessage + + var body: some View { + HStack { + if message.role == .user { Spacer(minLength: 40) } + Text(message.content) + .font(.body) + .foregroundColor(message.role == .user ? .white : .primary) + .padding(.horizontal, 14) + .padding(.vertical, 10) + .background( + RoundedRectangle(cornerRadius: 18, style: .continuous) + .fill(message.role == .user ? Color.black : Color(.secondarySystemBackground)) + ) + if message.role == .agent { Spacer(minLength: 40) } + } + } +} +#endif diff --git a/Tests/ElevenLabsTests/ElevenLabsTests.swift b/Tests/ElevenLabsTests/ElevenLabsTests.swift index 1e73a495..bbda99c0 100644 --- a/Tests/ElevenLabsTests/ElevenLabsTests.swift +++ b/Tests/ElevenLabsTests/ElevenLabsTests.swift @@ -2,17 +2,11 @@ import XCTest final class ElevenLabsTests: XCTestCase { - func testConfigurationDefault() { - let config = ElevenLabs.Configuration.default - XCTAssertNil(config.apiEndpoint) - XCTAssertEqual(config.logLevel, .warning) - XCTAssertFalse(config.debugMode) - } - func testConversationConfigInit() { let config = ConversationConfig() XCTAssertNil(config.agentOverrides) XCTAssertNil(config.ttsOverrides) XCTAssertFalse(config.conversationOverrides.textOnly) + XCTAssertEqual(config.endpoints, .production) } } diff --git a/Tests/ElevenLabsTests/Integration/ConversationIntegrationTests.swift b/Tests/ElevenLabsTests/Integration/ConversationIntegrationTests.swift index 4b947895..69e977f4 100644 --- a/Tests/ElevenLabsTests/Integration/ConversationIntegrationTests.swift +++ b/Tests/ElevenLabsTests/Integration/ConversationIntegrationTests.swift @@ -40,11 +40,6 @@ final class ConversationIntegrationTests: XCTestCase { try await conv.sendMessage("Hello") } - // Mute operations also require connected state - await assertThrowsConversationError(.notConnected) { - try await conv.toggleMicMute() - } - await assertThrowsConversationError(.notConnected) { try await conv.interruptAgent() } @@ -63,21 +58,13 @@ final class ConversationIntegrationTests: XCTestCase { // - Verify message ordering } - func testAudioIntegration() async { - let conversation = Conversation(dependencyProvider: Dependencies()) + func testAudioIntegration() async throws { + let client = ConversationClient() - // Test initial mute state - XCTAssertTrue(conversation.isMicMuted) + XCTAssertFalse(client.isMicMuted) - // Test mute operations when not connected - should throw - do { - try await conversation.setMicMuted(true) - XCTFail("Should throw error when not connected") - } catch let error as ConversationError { - XCTAssertEqual(error, .notConnected) - } catch { - XCTFail("Unexpected error type") - } + try await client.setMicMuted(true) + XCTAssertTrue(client.isMicMuted) // In a real integration test: // - Test microphone permissions @@ -93,9 +80,10 @@ final class ConversationIntegrationTests: XCTestCase { let conv2 = conversation await assertThrowsConversationError(.notConnected) { try await conv2.sendToolResult( - for: "test-tool-call", - result: "Tool executed successfully", - isError: false + .init( + toolCallId: "test-tool-call", + result: "Tool executed successfully" + ) ) } diff --git a/Tests/ElevenLabsTests/Mocks/MockWebRTCConnectionManager.swift b/Tests/ElevenLabsTests/Mocks/MockWebRTCConnectionManager.swift index 3ff90cf8..54e77868 100644 --- a/Tests/ElevenLabsTests/Mocks/MockWebRTCConnectionManager.swift +++ b/Tests/ElevenLabsTests/Mocks/MockWebRTCConnectionManager.swift @@ -19,28 +19,34 @@ final class MockWebRTCConnectionManager: WebRTCConnectionManaging { } var onRemoteSpeakingChanged: (@Sendable (Bool) -> Void)? + var onTracksChanged: (@Sendable () -> Void)? + private var initiationMetadataWaiter: ConversationInitiationMetadataWaiter? private var eventHandlerInstalled: CheckedContinuation? var room: Room? - var inputTrack: LocalAudioTrack? - var agentAudioTrack: RemoteAudioTrack? + var inputTrack: (any AudioTrackProtocol)? + var agentAudioTrack: (any AudioTrackProtocol)? var isMicrophoneMuted = true var errorHandler: ((Swift.Error?) -> Void)? + var onDisconnectStarted: (@MainActor () async -> Void)? var shouldFailConnection = false var connectionError: Swift.Error = Error.connectionFailed var tokenError: ConversationError? var publishError: Swift.Error? var microphoneError: Swift.Error? + var autoDeliverInitiationMetadata = true + var initiationMetadataConversationId = "test-conversation-id" private(set) var connectCallCount = 0 private(set) var disconnectCallCount = 0 private(set) var lastNetworkConfiguration: WebRTCConfiguration = .default private(set) var lastWaitTimeout: TimeInterval = 0 private(set) var publishedPayloads: [Data] = [] + private var startupStateChange: ((ConversationStartupState) -> Void)? private var waitContinuation: CheckedContinuation? private var pendingWaitResult: AgentReadyWaitResult? @@ -57,20 +63,27 @@ final class MockWebRTCConnectionManager: WebRTCConnectionManaging { auth: ConversationCredentials, config: ConversationConfig, onStartupStateChange: @escaping (ConversationStartupState) -> Void - ) async throws -> StartupResult { + ) async throws -> ConversationStartResult { + startupStateChange = onStartupStateChange + await initiationMetadataWaiter?.cancel() + let waiter = ConversationInitiationMetadataWaiter( + timeout: config.startupConfiguration.initiationMetadataTimeout + ) + initiationMetadataWaiter = waiter + let startTime = Date() connectCallCount += 1 lastNetworkConfiguration = config.networkConfiguration var metrics = ConversationStartupMetrics() onStartupStateChange(.resolvingToken) if let tokenError { - throw StartupFailure.token(tokenError, metrics) + throw tokenError } onStartupStateChange(.connectingRoom) if shouldFailConnection { errorHandler?(connectionError) - throw StartupFailure.room(connectionError as? ConversationError ?? .connectionFailed(connectionError), metrics) + throw connectionError as? ConversationError ?? .connectionFailed(connectionError) } room = Room() @@ -78,32 +91,63 @@ final class MockWebRTCConnectionManager: WebRTCConnectionManaging { switch await waitForAgentReady(timeout: config.startupConfiguration.agentReadyTimeout) { case let .success(elapsed): metrics.agentReady = elapsed - onStartupStateChange(.agentReady(ConversationAgentReadyReport(elapsed: elapsed))) + onStartupStateChange(.agentReady(elapsed: elapsed)) case let .timedOut(elapsed): metrics.agentReady = elapsed - throw StartupFailure.agentTimeout(metrics) + throw ConversationError.agentTimeout + case let .cancelled(elapsed): + metrics.agentReady = elapsed + throw CancellationError() } - onStartupStateChange(.sendingConversationInit(attempt: 1)) + onStartupStateChange(.sendingConversationInit) do { try await send(event: .conversationInit(ConversationInitEvent(config: config))) } catch { - throw StartupFailure.conversationInit(error as? ConversationError ?? .connectionFailed(error), metrics) + throw error as? ConversationError ?? ConversationError.connectionFailed(error) + } + + if autoDeliverInitiationMetadata { + let metadata = makeInitiationMetadata() + await waiter.observe(metadata) + onEventReceived?(.conversationMetadata(metadata)) } - metrics.conversationInitAttempts = 1 - return StartupResult(agentId: auth.agentId, metrics: metrics) + let metadata = try await waitForInitiationMetadata( + config: config, + metrics: &metrics, + startTime: startTime, + metadataWaiter: waiter, + onStartupStateChange: onStartupStateChange + ) + return ConversationStartResult( + callInfo: CallInfo(agentId: auth.agentId, conversationId: metadata.conversationId), + metrics: metrics + ) + } + + func deliverStartupState(_ state: ConversationStartupState) { + startupStateChange?(state) } func disconnect() async { disconnectCallCount += 1 + await onDisconnectStarted?() onEventReceived = nil onDisconnected = nil errorHandler = nil onRemoteSpeakingChanged = nil + onTracksChanged = nil room = nil + // Only cancel an in-flight agent-ready wait — don't stash `.cancelled` for the next connect. + if waitContinuation != nil { + await cancelAgentReady(elapsed: 0) + } + await initiationMetadataWaiter?.cancel() + initiationMetadataWaiter = nil } + @MainActor func waitForAgentReady(timeout: TimeInterval) async -> AgentReadyWaitResult { lastWaitTimeout = timeout if let pending = pendingWaitResult { @@ -144,7 +188,14 @@ final class MockWebRTCConnectionManager: WebRTCConnectionManaging { // MARK: - Helpers func receive(data: Data) { - handleIncomingData(data, logger: SDKLogger(logLevel: .error)) + guard let initiationMetadataWaiter else { + preconditionFailure("Cannot receive data before connecting") + } + handleIncomingData( + data, + metadataWaiter: initiationMetadataWaiter, + logger: SDKLogger(logLevel: .error) + ) } /// Delivers an already-decoded event directly to the installed handler @@ -153,19 +204,28 @@ final class MockWebRTCConnectionManager: WebRTCConnectionManaging { } /// Suspends until `onEventReceived` is set. + @MainActor func waitForEventHandlerInstalled() async { if onEventReceived != nil { return } await withCheckedContinuation { eventHandlerInstalled = $0 } } + @MainActor func succeedAgentReady(elapsed: TimeInterval = 0.1) { resumeWait(with: .success(elapsed: elapsed)) } + @MainActor func timeoutAgentReady(elapsed: TimeInterval = 0.1) { resumeWait(with: .timedOut(elapsed: elapsed)) } + @MainActor + func cancelAgentReady(elapsed: TimeInterval = 0.1) { + resumeWait(with: .cancelled(elapsed: elapsed)) + } + + @MainActor private func resumeWait(with result: AgentReadyWaitResult) { if let continuation = waitContinuation { waitContinuation = nil @@ -174,4 +234,12 @@ final class MockWebRTCConnectionManager: WebRTCConnectionManaging { pendingWaitResult = result } } + + private func makeInitiationMetadata() -> ConversationMetadataEvent { + ConversationMetadataEvent( + conversationId: initiationMetadataConversationId, + agentOutputAudioFormat: "pcm_16000", + userInputAudioFormat: "pcm_16000" + ) + } } diff --git a/Tests/ElevenLabsTests/Mocks/MockWebSocketConnectionManager.swift b/Tests/ElevenLabsTests/Mocks/MockWebSocketConnectionManager.swift index 8fbb4cbf..5445c9c0 100644 --- a/Tests/ElevenLabsTests/Mocks/MockWebSocketConnectionManager.swift +++ b/Tests/ElevenLabsTests/Mocks/MockWebSocketConnectionManager.swift @@ -5,9 +5,12 @@ final class MockWebSocketConnectionManager: WebSocketConnectionManaging { var onEventReceived: (@Sendable (IncomingEvent) -> Void)? var onDisconnected: (() async -> Void)? var errorHandler: ((Swift.Error?) -> Void)? + private var initiationMetadataWaiter: ConversationInitiationMetadataWaiter? var connectError: Error? var sendError: Error? + var autoDeliverInitiationMetadata = true + var initiationMetadataConversationId = "test-conversation-id" private(set) var connectCallCount = 0 private(set) var disconnectCallCount = 0 @@ -15,28 +18,38 @@ final class MockWebSocketConnectionManager: WebSocketConnectionManaging { private(set) var sentPayloads: [Data] = [] private(set) var isConnected = false - func connect(auth: ConversationCredentials, config: ConversationConfig) async throws -> StartupResult { + @MainActor + func connect( + auth: ConversationCredentials, + config: ConversationConfig, + onStartupStateChange: @escaping (ConversationStartupState) -> Void + ) async throws -> ConversationStartResult { + await initiationMetadataWaiter?.cancel() + let waiter = ConversationInitiationMetadataWaiter( + timeout: config.startupConfiguration.initiationMetadataTimeout + ) + initiationMetadataWaiter = waiter connectCallCount += 1 let startTime = Date() var metrics = ConversationStartupMetrics() do { - lastConnectedURL = try WebSocketConnectionManager.url(for: auth) + lastConnectedURL = try WebSocketConnectionManager.websocketUrl(for: auth, endpoints: config.endpoints) } catch { metrics.total = Date().timeIntervalSince(startTime) let convError = error as? ConversationError ?? .authenticationFailed(error.localizedDescription) - throw StartupFailure.token(convError, metrics) + throw convError } if let connectError { errorHandler?(connectError) metrics.total = Date().timeIntervalSince(startTime) - let convError = connectError as? ConversationError ?? .connectionFailed(connectError) - throw StartupFailure.conversationInit(convError, metrics) + throw connectError as? ConversationError ?? ConversationError.connectionFailed(connectError) } isConnected = true + onStartupStateChange(.sendingConversationInit) do { let initEvent = ConversationInitEvent(config: config) try await send(data: EventSerializer.serializeOutgoingEvent(.conversationInit(initEvent))) @@ -44,13 +57,26 @@ final class MockWebSocketConnectionManager: WebSocketConnectionManaging { throw CancellationError() } catch { metrics.total = Date().timeIntervalSince(startTime) - let convError = error as? ConversationError ?? .connectionFailed(error) - throw StartupFailure.conversationInit(convError, metrics) + throw error as? ConversationError ?? ConversationError.connectionFailed(error) } - metrics.conversationInitAttempts = 1 - metrics.total = Date().timeIntervalSince(startTime) - return StartupResult(agentId: auth.agentId, metrics: metrics) + if autoDeliverInitiationMetadata { + let metadata = makeInitiationMetadata() + await waiter.observe(metadata) + onEventReceived?(.conversationMetadata(metadata)) + } + + let metadata = try await waitForInitiationMetadata( + config: config, + metrics: &metrics, + startTime: startTime, + metadataWaiter: waiter, + onStartupStateChange: onStartupStateChange + ) + return ConversationStartResult( + callInfo: CallInfo(agentId: auth.agentId, conversationId: metadata.conversationId), + metrics: metrics + ) } func disconnect() async { @@ -59,6 +85,8 @@ final class MockWebSocketConnectionManager: WebSocketConnectionManaging { onDisconnected = nil errorHandler = nil isConnected = false + await initiationMetadataWaiter?.cancel() + initiationMetadataWaiter = nil } func send(data: Data) async throws { @@ -73,6 +101,21 @@ final class MockWebSocketConnectionManager: WebSocketConnectionManaging { } func receive(data: Data) { - handleIncomingData(data, logger: SDKLogger(logLevel: .error)) + guard let initiationMetadataWaiter else { + preconditionFailure("Cannot receive data before connecting") + } + handleIncomingData( + data, + metadataWaiter: initiationMetadataWaiter, + logger: SDKLogger(logLevel: .error) + ) + } + + private func makeInitiationMetadata() -> ConversationMetadataEvent { + ConversationMetadataEvent( + conversationId: initiationMetadataConversationId, + agentOutputAudioFormat: "pcm_16000", + userInputAudioFormat: "pcm_16000" + ) } } diff --git a/Tests/ElevenLabsTests/StartupPerformanceTest.swift b/Tests/ElevenLabsTests/StartupPerformanceTest.swift index 6454d820..9d75b2df 100644 --- a/Tests/ElevenLabsTests/StartupPerformanceTest.swift +++ b/Tests/ElevenLabsTests/StartupPerformanceTest.swift @@ -48,28 +48,28 @@ final class StartupPerformanceTest: XCTestCase { // Monitor state changes - create these before starting conversation var hasConnected = false var hasReceivedFirstMessage = false - var conversation: Conversation! + let client = ConversationClient() - // Start the conversation using the static API + // Start the conversation print(" [\(String(format: "%.3f", 0.0))s] Starting conversation...") - conversation = try await ElevenLabs.startConversation( + let result = try await client.startConversation( agentId: "agent_4601k18km8yde6ftyzzwfdk6jvez" ) - // Since the static API already handles the startup, just monitor the result + // Since starting already handles the startup, just monitor the result // Check the current state immediately let elapsed = Date().timeIntervalSince(testStart) print(" [\(String(format: "%.3f", elapsed))s] Conversation created") - switch conversation.state { + switch client.state { case .idle: print(" [\(String(format: "%.3f", elapsed))s] State: idle") - case .connecting: - print(" [\(String(format: "%.3f", elapsed))s] State: connecting") + case let .connecting(stage): + print(" [\(String(format: "%.3f", elapsed))s] State: connecting (\(stage))") case let .connected(info): hasConnected = true print(" [\(String(format: "%.3f", elapsed))s] State: connected (agent: \(info.agentId))") - print(" 🎯 ACTIVE STATE REACHED in \(String(format: "%.3f", elapsed))s") + print(" 🎯 CONNECTED STATE REACHED in \(String(format: "%.3f", elapsed))s") case let .ended(reason): print(" [\(String(format: "%.3f", elapsed))s] State: ended (reason: \(reason))") case let .error(error): @@ -77,24 +77,21 @@ final class StartupPerformanceTest: XCTestCase { } // Check for existing messages - if !conversation.messages.isEmpty { + if !client.messages.isEmpty { hasReceivedFirstMessage = true - print(" [\(String(format: "%.3f", elapsed))s] Messages already present: \(conversation.messages.count)") - if let firstMessage = conversation.messages.first { + print(" [\(String(format: "%.3f", elapsed))s] Messages already present: \(client.messages.count)") + if let firstMessage = client.messages.first { print(" 📨 Message: \(firstMessage.content)") } } - print(" [\(String(format: "%.3f", elapsed))s] Agent state: \(conversation.agentState)") + print(" [\(String(format: "%.3f", elapsed))s] Agent state: \(client.agentState)") - // The static API should return an already-connected conversation - // But let's give it a moment and measure the total time when we called the API let totalTime = Date().timeIntervalSince(testStart) - // Mark as connected since the API returned successfully if !hasConnected { hasConnected = true - print(" [\(String(format: "%.3f", totalTime))s] ✅ Conversation returned from API") + print(" [\(String(format: "%.3f", totalTime))s] ✅ Conversation returned from startConversation") } // Wait a bit for first message @@ -105,20 +102,20 @@ final class StartupPerformanceTest: XCTestCase { // Clean up print(" [\(String(format: "%.3f", Date().timeIntervalSince(testStart)))s] Ending conversation...") - await conversation.endConversation() + await client.endConversation() // Wait for cleanup try await Task.sleep(nanoseconds: 500_000_000) // 0.5s // Check if we reached connected state - let reachedConnected = conversation.state.isConnected - if case .ended(reason: .userEnded) = conversation.state { + let reachedConnected = hasConnected || client.state.isConnected + if case .ended(reason: .userEnded) = client.state { // This is fine - we ended it ourselves } else if !reachedConnected { throw NSError(domain: "StartupTest", code: 1, userInfo: [NSLocalizedDescriptionKey: "Failed to reach connected state"]) } - return totalTime + return try XCTUnwrap(result.metrics.total) } private func printSummary(timings: [TimeInterval]) { diff --git a/Tests/ElevenLabsTests/TestSupport.swift b/Tests/ElevenLabsTests/TestSupport.swift index 4e04a596..1e92985e 100644 --- a/Tests/ElevenLabsTests/TestSupport.swift +++ b/Tests/ElevenLabsTests/TestSupport.swift @@ -1,5 +1,7 @@ +import AVFoundation import Combine @testable import ElevenLabs +import LiveKit import XCTest extension XCTestCase { @@ -51,19 +53,6 @@ extension XCTestCase { return await recorder.last() } - /// Waits until the mock's `onEventReceived` handler is installed. - func waitForEventHandlerInstalled( - on mock: MockWebRTCConnectionManager, - timeout: TimeInterval = 1.0 - ) async { - let done = expectation(description: "event handler installed") - Task { - await mock.waitForEventHandlerInstalled() - done.fulfill() - } - await fulfillment(of: [done], timeout: timeout) - } - func XCTAssertThrowsErrorAsync( _ expression: () async throws -> some Sendable, _ message: @autoclosure () -> String = "", @@ -129,3 +118,40 @@ actor ValueRecorder { await withCheckedContinuation { valueWaiters.append((predicate, $0)) } } } + +final class SpyAudioTrack: NSObject, AudioTrackProtocol, @unchecked Sendable { + private var renderers: [any AudioRenderer] = [] + + private(set) var addCallCount = 0 + private(set) var removeCallCount = 0 + + var attachedRendererCount: Int { + renderers.count + } + + func add(audioRenderer: AudioRenderer) { + addCallCount += 1 + renderers.append(audioRenderer) + } + + func remove(audioRenderer: AudioRenderer) { + removeCallCount += 1 + let removedRenderer = audioRenderer as AnyObject + renderers.removeAll { ($0 as AnyObject) === removedRenderer } + } + + func render() { + let format = AVAudioFormat(standardFormatWithSampleRate: 16000, channels: 1)! + let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: 160)! + buffer.frameLength = 160 + renderers.forEach { $0.render(pcmBuffer: buffer) } + } +} + +final class RecordingAudioObserver: ConversationAudioObserver, @unchecked Sendable { + private(set) var receivedBufferCount = 0 + + func didReceive(_: AVAudioPCMBuffer) { + receivedBufferCount += 1 + } +} diff --git a/Tests/ElevenLabsTests/Unit/AudioObserverRegistryTests.swift b/Tests/ElevenLabsTests/Unit/AudioObserverRegistryTests.swift new file mode 100644 index 00000000..3e0587f2 --- /dev/null +++ b/Tests/ElevenLabsTests/Unit/AudioObserverRegistryTests.swift @@ -0,0 +1,63 @@ +import AVFoundation +@testable import ElevenLabs +import XCTest + +@MainActor +final class AudioObserverRegistryTests: XCTestCase { + func testAddRemoveIsIdempotentWithoutTrack() { + let registry = AudioObserverRegistry() + let observer = SpyAudioObserver() + + registry.add(observer) + registry.add(observer) + XCTAssertEqual(registry.registeredCount, 1) + + registry.remove(observer) + registry.remove(observer) + XCTAssertEqual(registry.registeredCount, 0) + } + + func testAttachToNilKeepsRegisteredObservers() { + let registry = AudioObserverRegistry() + let observer = SpyAudioObserver() + + registry.add(observer) + registry.attach(to: nil) + XCTAssertEqual(registry.registeredCount, 1) + + registry.remove(observer) + XCTAssertEqual(registry.registeredCount, 0) + } + + func testResetClearsRegistrations() { + let registry = AudioObserverRegistry() + registry.add(SpyAudioObserver()) + registry.reset() + XCTAssertEqual(registry.registeredCount, 0) + } + + func testAttachDeliversBuffersAndResetDetachesFromTrack() { + let registry = AudioObserverRegistry() + let track = SpyAudioTrack() + let observer = RecordingAudioObserver() + + registry.add(observer) + registry.attach(to: track) + track.render() + + XCTAssertEqual(observer.receivedBufferCount, 1) + XCTAssertEqual(track.addCallCount, 1) + XCTAssertEqual(track.attachedRendererCount, 1) + + registry.reset() + track.render() + + XCTAssertEqual(observer.receivedBufferCount, 1) + XCTAssertEqual(track.removeCallCount, 1) + XCTAssertEqual(track.attachedRendererCount, 0) + } +} + +private final class SpyAudioObserver: ConversationAudioObserver, @unchecked Sendable { + func didReceive(_: AVAudioPCMBuffer) {} +} diff --git a/Tests/ElevenLabsTests/Unit/BusinessLogicTests.swift b/Tests/ElevenLabsTests/Unit/BusinessLogicTests.swift deleted file mode 100644 index 3a0661b2..00000000 --- a/Tests/ElevenLabsTests/Unit/BusinessLogicTests.swift +++ /dev/null @@ -1,183 +0,0 @@ -import Combine -@testable import ElevenLabs -import Foundation -import LiveKit -import XCTest - -@MainActor -final class ElevenLabsBusinessLogicTests: XCTestCase { - private var conversation: Conversation! - private var mockWebRTCConnectionManager: MockWebRTCConnectionManager! - private var dependencyProvider: TestDependencyProvider! - - override func setUp() async throws { - mockWebRTCConnectionManager = MockWebRTCConnectionManager() - dependencyProvider = TestDependencyProvider( - webRTCConnectionManager: mockWebRTCConnectionManager - ) - conversation = Conversation(dependencyProvider: dependencyProvider) - } - - override func tearDown() async throws { - conversation = nil - mockWebRTCConnectionManager = nil - dependencyProvider = nil - } - - // MARK: - Tool Call Tests - - func testToolCallLifecycle() async throws { - try await conversation.startConversation(auth: .publicAgent(id: "test")) - - // 1. Receive a tool call - let toolCall = try ClientToolCallEvent( - toolName: "test_tool", - toolCallId: "call_123", - parametersData: JSONSerialization.data(withJSONObject: ["arg": "val"]), - eventId: 1, - expectsResponse: false - ) - mockWebRTCConnectionManager.deliver(.clientToolCall(toolCall)) - await waitForPublished(conversation.$pendingToolCalls) { $0.contains { $0.toolCallId == "call_123" } } - - XCTAssertEqual(conversation.pendingToolCalls.count, 1) - XCTAssertEqual(conversation.pendingToolCalls.first?.toolCallId, "call_123") - - // 2. Send result - let payloadCountBeforeResult = mockWebRTCConnectionManager.publishedPayloads.count - try await conversation.sendToolResult(for: "call_123", result: "success") - - // 3. Verify tool is removed from pending list - XCTAssertTrue(conversation.pendingToolCalls.isEmpty) - - // 4. Verify result was published - XCTAssertEqual(mockWebRTCConnectionManager.publishedPayloads.count, payloadCountBeforeResult + 1) - let lastPayload = mockWebRTCConnectionManager.publishedPayloads.last ?? Data() - let lastPayloadString = String(data: lastPayload, encoding: .utf8) ?? "" - XCTAssertTrue(lastPayloadString.contains("call_123")) - XCTAssertTrue(lastPayloadString.contains("success")) - } - - func testSendToolResultEncodesEncodableResult() async throws { - struct Weather: Encodable { - let temperature: Int - let condition: String - } - - try await conversation.startConversation(auth: .publicAgent(id: "test")) - - try await conversation.sendToolResult( - for: "call_42", - result: Weather(temperature: 25, condition: "Sunny") - ) - - let payload = try XCTUnwrap(mockWebRTCConnectionManager.publishedPayloads.last) - let envelope = try XCTUnwrap(JSONSerialization.jsonObject(with: payload) as? [String: Any]) - XCTAssertEqual(envelope["type"] as? String, "client_tool_result") - // The Encodable value is JSON-encoded into the `result` string. - let resultString = try XCTUnwrap(envelope["result"] as? String) - let parsed = try JSONSerialization.jsonObject(with: XCTUnwrap(resultString.data(using: .utf8))) as? [String: Any] - XCTAssertEqual(parsed?["temperature"] as? Int, 25) - XCTAssertEqual(parsed?["condition"] as? String, "Sunny") - } - - // MARK: - Streaming Message Tests - - func testAgentStreamingMessages() async throws { - try await conversation.startConversation(auth: .publicAgent(id: "test")) - - // 1. Start streaming - mockWebRTCConnectionManager.deliver(.agentChatResponsePart( - AgentChatResponsePartEvent(text: "Hello", type: .start, eventId: 1) - )) - await waitForPublished(conversation.$messages) { $0.first?.content == "Hello" } - - XCTAssertEqual(conversation.messages.count, 1) - XCTAssertEqual(conversation.messages.first?.content, "Hello") - XCTAssertEqual(conversation.messages.first?.role, .agent) - - // 2. Delta update - mockWebRTCConnectionManager.deliver(.agentChatResponsePart( - AgentChatResponsePartEvent(text: " world", type: .delta, eventId: 1) - )) - await waitForPublished(conversation.$messages) { $0.first?.content == "Hello world" } - - XCTAssertEqual(conversation.messages.count, 1, "Should still have only 1 message, just updated") - XCTAssertEqual(conversation.messages.first?.content, "Hello world") - - // 3. Stop streaming - mockWebRTCConnectionManager.deliver(.agentChatResponsePart( - AgentChatResponsePartEvent(text: "!", type: .stop, eventId: 1) - )) - await waitForPublished(conversation.$messages) { $0.first?.content == "Hello world!" } - - XCTAssertEqual(conversation.messages.count, 1) - XCTAssertEqual(conversation.messages.first?.content, "Hello world!") - } - - // MARK: - End Call Logic - - func testAutomaticEndCallHandling() async throws { - try await conversation.startConversation(auth: .publicAgent(id: "test")) - - mockWebRTCConnectionManager.deliver(.agentToolResponse(AgentToolResponseEvent( - toolName: "end_call", toolCallId: "id", toolType: "action", isError: false, eventId: 1 - ))) - await waitForPublished(conversation.$state) { $0 == .ended(reason: .userEnded) } - - XCTAssertEqual(conversation.state, .ended(reason: .userEnded)) - } - - // MARK: - Concurrency & Responsiveness - - func testStateTransitionsImmediatelyToConnecting() async throws { - try await conversation.startConversation(auth: .publicAgent(id: "old-agent")) - await conversation.endConversation() - - // Hold agent-ready so we can observe `.connecting` before startup finishes. - mockWebRTCConnectionManager.autoSucceedAgentReady = false - let startTask = Task { - try await conversation.startConversation(auth: .publicAgent(id: "new-agent")) - } - - await waitForPublished(conversation.$state) { $0 == .connecting } - XCTAssertEqual(conversation.state, .connecting, "Should be connecting immediately, even if disconnect() is slow") - - mockWebRTCConnectionManager.succeedAgentReady() - try await startTask.value - - XCTAssertEqual(conversation.state, .connected(CallInfo(agentId: "new-agent"))) - } - - // MARK: - Audio Alignment - - func testAudioAlignmentUpdatesProperty() async throws { - try await conversation.startConversation(auth: .publicAgent(id: "test")) - - let alignment = AudioAlignment( - chars: ["H", "e", "l", "l", "o"], - charStartTimesMs: [0, 100, 200, 300, 400], - charDurationsMs: [100, 100, 100, 100, 100] - ) - mockWebRTCConnectionManager.deliver(.audio(AudioEvent(audioBase64: "base64", eventId: 1, alignment: alignment))) - await waitForPublished(conversation.$latestAudioAlignment) { $0 != nil } - - XCTAssertEqual(conversation.latestAudioAlignment?.chars, ["H", "e", "l", "l", "o"]) - } - - func testEndConversationClearsLatestAudioState() async throws { - try await conversation.startConversation(auth: .publicAgent(id: "test")) - - let alignment = AudioAlignment(chars: ["H"], charStartTimesMs: [0], charDurationsMs: [100]) - mockWebRTCConnectionManager.deliver(.audio(AudioEvent(audioBase64: "base64", eventId: 1, alignment: alignment))) - await waitForPublished(conversation.$latestAudioAlignment) { $0 != nil } - - XCTAssertNotNil(conversation.latestAudioEvent) - XCTAssertNotNil(conversation.latestAudioAlignment) - - await conversation.endConversation() - - XCTAssertNil(conversation.latestAudioEvent) - XCTAssertNil(conversation.latestAudioAlignment) - } -} diff --git a/Tests/ElevenLabsTests/Unit/ConversationAudioObserverTests.swift b/Tests/ElevenLabsTests/Unit/ConversationAudioObserverTests.swift new file mode 100644 index 00000000..ba1416db --- /dev/null +++ b/Tests/ElevenLabsTests/Unit/ConversationAudioObserverTests.swift @@ -0,0 +1,175 @@ +@testable import ElevenLabs +import XCTest + +@MainActor +final class ConversationAudioObserverTests: XCTestCase { + private var conversation: Conversation! + private var mockWebRTCConnectionManager: MockWebRTCConnectionManager! + private var dependencyProvider: TestDependencyProvider! + + override func setUp() async throws { + mockWebRTCConnectionManager = MockWebRTCConnectionManager() + dependencyProvider = TestDependencyProvider( + webRTCConnectionManager: mockWebRTCConnectionManager, + webSocketConnectionManager: MockWebSocketConnectionManager() + ) + conversation = Conversation(dependencyProvider: dependencyProvider) + } + + override func tearDown() async throws { + conversation = nil + mockWebRTCConnectionManager = nil + dependencyProvider = nil + } + + func testSessionKeepsObserversThroughStartAndClearsThemOnEnd() async throws { + let agentObserver = RecordingAudioObserver() + let micObserver = RecordingAudioObserver() + let agentTrack = SpyAudioTrack() + let micTrack = SpyAudioTrack() + mockWebRTCConnectionManager.agentAudioTrack = agentTrack + mockWebRTCConnectionManager.inputTrack = micTrack + + conversation.addAgentAudioObserver(agentObserver) + conversation.addMicAudioObserver(micObserver) + XCTAssertEqual(conversation.agentObserverRegistry.registeredCount, 1) + XCTAssertEqual(conversation.micObserverRegistry.registeredCount, 1) + + _ = try await conversation.start(auth: .publicAgent(id: "test-agent")) + XCTAssertNotNil(mockWebRTCConnectionManager.onTracksChanged) + XCTAssertEqual(conversation.agentObserverRegistry.registeredCount, 1) + XCTAssertEqual(conversation.micObserverRegistry.registeredCount, 1) + + agentTrack.render() + micTrack.render() + XCTAssertEqual(agentObserver.receivedBufferCount, 1) + XCTAssertEqual(micObserver.receivedBufferCount, 1) + + let staleTracksChanged = mockWebRTCConnectionManager.onTracksChanged + mockWebRTCConnectionManager.onDisconnectStarted = { + agentTrack.render() + micTrack.render() + } + + await conversation.endConversation() + XCTAssertEqual(conversation.agentObserverRegistry.registeredCount, 0) + XCTAssertEqual(conversation.micObserverRegistry.registeredCount, 0) + XCTAssertEqual(agentObserver.receivedBufferCount, 1) + XCTAssertEqual(micObserver.receivedBufferCount, 1) + XCTAssertEqual(agentTrack.attachedRendererCount, 0) + XCTAssertEqual(micTrack.attachedRendererCount, 0) + + staleTracksChanged?() + await Task { @MainActor in }.value + agentTrack.render() + micTrack.render() + XCTAssertEqual(agentObserver.receivedBufferCount, 1) + XCTAssertEqual(micObserver.receivedBufferCount, 1) + } + + func testIdleEndPreventsLaterObserverAttachment() async { + let observer = RecordingAudioObserver() + let track = SpyAudioTrack() + + await conversation.endConversation() + conversation.addAgentAudioObserver(observer) + conversation.agentObserverRegistry.attach(to: track) + track.render() + + XCTAssertEqual(observer.receivedBufferCount, 0) + XCTAssertEqual(track.attachedRendererCount, 0) + } + + func testStartupFailureIgnoresStaleTrackChangesAfterCleanup() async { + let lateObserver = RecordingAudioObserver() + let agentTrack = SpyAudioTrack() + mockWebRTCConnectionManager.agentAudioTrack = agentTrack + mockWebRTCConnectionManager.autoSucceedAgentReady = false + + let startTask = Task { + try await conversation.start(auth: .publicAgent(id: "test-agent")) + } + + await waitForPublished(conversation.$state) { + guard case let .connecting(stage) = $0, + case .waitingForAgent = stage + else { + return false + } + return true + } + + let staleTracksChanged = mockWebRTCConnectionManager.onTracksChanged + mockWebRTCConnectionManager.onDisconnectStarted = { [conversation] in + conversation?.addAgentAudioObserver(lateObserver) + staleTracksChanged?() + await Task { @MainActor in }.value + agentTrack.render() + } + + mockWebRTCConnectionManager.timeoutAgentReady() + await XCTAssertThrowsErrorAsync { + try await startTask.value + } errorHandler: { error in + XCTAssertEqual(error as? ConversationError, .agentTimeout) + } + + XCTAssertEqual(lateObserver.receivedBufferCount, 0) + XCTAssertEqual(agentTrack.attachedRendererCount, 0) + } + + func testStartupCancellationDetachesObserversBeforeDisconnect() async { + let observer = RecordingAudioObserver() + let agentTrack = SpyAudioTrack() + mockWebRTCConnectionManager.agentAudioTrack = agentTrack + mockWebRTCConnectionManager.autoDeliverInitiationMetadata = false + conversation.addAgentAudioObserver(observer) + + let startTask = Task { + try await conversation.start(auth: .publicAgent(id: "test-agent")) + } + + await waitForPublished(conversation.$state) { + $0 == .connecting(.waitingForInitiationMetadata(timeout: 5)) + } + mockWebRTCConnectionManager.onTracksChanged?() + await Task { @MainActor in }.value + agentTrack.render() + XCTAssertEqual(observer.receivedBufferCount, 1) + + mockWebRTCConnectionManager.onDisconnectStarted = { + agentTrack.render() + } + startTask.cancel() + + await XCTAssertThrowsErrorAsync { + try await startTask.value + } errorHandler: { error in + XCTAssertTrue(error is CancellationError) + } + XCTAssertEqual(observer.receivedBufferCount, 1) + XCTAssertEqual(agentTrack.attachedRendererCount, 0) + } + + func testClientReusesDurableObserversAcrossResetWithoutReadding() async throws { + let client = ConversationClient(dependencyProvider: dependencyProvider) + let agentObserver = RecordingAudioObserver() + let micObserver = RecordingAudioObserver() + + client.addAgentAudioObserver(agentObserver) + client.addMicAudioObserver(micObserver) + client.addAgentAudioObserver(agentObserver) // idempotent + + _ = try await client.startConversation(auth: .publicAgent(id: "first-agent")) + XCTAssertTrue(client.state.isConnected) + + await client.reset() + _ = try await client.startConversation(auth: .publicAgent(id: "second-agent")) + XCTAssertTrue(client.state.isConnected) + + // Public remove path remains valid after the second session binds. + client.removeAgentAudioObserver(agentObserver) + client.removeMicAudioObserver(micObserver) + await client.endConversation() + } +} diff --git a/Tests/ElevenLabsTests/Unit/ConversationClientTests.swift b/Tests/ElevenLabsTests/Unit/ConversationClientTests.swift new file mode 100644 index 00000000..abf65b61 --- /dev/null +++ b/Tests/ElevenLabsTests/Unit/ConversationClientTests.swift @@ -0,0 +1,242 @@ +@testable import ElevenLabs +import Foundation +import XCTest + +@MainActor +final class ConversationClientTests: XCTestCase { + private var client: ConversationClient! + private var mockWebRTCConnectionManager: MockWebRTCConnectionManager! + private var mockWebSocketConnectionManager: MockWebSocketConnectionManager! + private var dependencyProvider: TestDependencyProvider! + + override func setUp() async throws { + mockWebRTCConnectionManager = MockWebRTCConnectionManager() + mockWebSocketConnectionManager = MockWebSocketConnectionManager() + dependencyProvider = TestDependencyProvider( + webRTCConnectionManager: mockWebRTCConnectionManager, + webSocketConnectionManager: mockWebSocketConnectionManager + ) + client = ConversationClient(dependencyProvider: dependencyProvider) + } + + override func tearDown() async throws { + client = nil + mockWebRTCConnectionManager = nil + mockWebSocketConnectionManager = nil + dependencyProvider = nil + } + + func testInitialStateIsIdleBeforeAnyStart() { + XCTAssertEqual(client.state, .idle) + XCTAssertTrue(client.messages.isEmpty) + XCTAssertFalse(client.isMicMuted) + } + + func testStartConversationMirrorsSessionStateThroughActiveAndEnd() async throws { + let result = try await client.startConversation(auth: .publicAgent(id: "test-agent-id")) + + assertConnected(agentId: "test-agent-id") + XCTAssertEqual(result.callInfo.agentId, "test-agent-id") + XCTAssertEqual(result.callInfo.conversationId, "test-conversation-id") + XCTAssertNotNil(result.metrics.initiationMetadata) + XCTAssertFalse(client.isMicMuted) + XCTAssertFalse(mockWebRTCConnectionManager.isMicrophoneMuted) + + let payload: [String: Any] = [ + "type": "agent_response", + "agent_response_event": [ + "agent_response": "Hello from the agent", + "event_id": 1 + ] + ] + let data = try JSONSerialization.data(withJSONObject: payload) + mockWebRTCConnectionManager.receive(data: data) + await waitForPublished(client.$messages) { $0.last?.content == "Hello from the agent" } + + XCTAssertEqual(client.messages.last?.content, "Hello from the agent") + + await client.endConversation() + XCTAssertEqual(client.state, .ended(reason: .userEnded)) + } + + func testRestartEndsPreviousSessionAndStartsFresh() async throws { + _ = try await client.startConversation(auth: .publicAgent(id: "first-agent")) + assertConnected(agentId: "first-agent") + XCTAssertEqual(mockWebRTCConnectionManager.connectCallCount, 1) + XCTAssertEqual(mockWebRTCConnectionManager.disconnectCallCount, 0) + + await client.endConversation() + XCTAssertEqual(client.state, .ended(reason: .userEnded)) + XCTAssertEqual(mockWebRTCConnectionManager.disconnectCallCount, 1) + + _ = try await client.startConversation(auth: .publicAgent(id: "second-agent")) + + assertConnected(agentId: "second-agent") + XCTAssertTrue(client.messages.isEmpty) + XCTAssertEqual(mockWebRTCConnectionManager.connectCallCount, 2) + XCTAssertEqual(mockWebRTCConnectionManager.disconnectCallCount, 1) + } + + func testStartWhileActiveEndsPreviousAndStartsNext() async throws { + _ = try await client.startConversation(auth: .publicAgent(id: "first-agent")) + assertConnected(agentId: "first-agent") + + _ = try await client.startConversation(auth: .publicAgent(id: "second-agent")) + + assertConnected(agentId: "second-agent") + XCTAssertTrue(client.messages.isEmpty) + XCTAssertEqual(mockWebRTCConnectionManager.connectCallCount, 2) + XCTAssertEqual(mockWebRTCConnectionManager.disconnectCallCount, 1) + } + + func testLatestOverlappingStartWins() async throws { + mockWebRTCConnectionManager.autoSucceedAgentReady = false + + let firstStart = Task { + try await client.startConversation(auth: .publicAgent(id: "first-agent")) + } + await waitForPublished(client.$state) { + guard case let .connecting(stage) = $0, + case .waitingForAgent = stage + else { + return false + } + return true + } + + var textConfig = ConversationConfig() + textConfig.conversationOverrides = ConversationOverrides(textOnly: true) + _ = try await client.startConversation( + auth: .publicAgent(id: "second-agent"), + config: textConfig + ) + + mockWebRTCConnectionManager.succeedAgentReady() + await XCTAssertThrowsErrorAsync { + _ = try await firstStart.value + } + + assertConnected(agentId: "second-agent") + } + + func testResetEndsSessionAndClearsMirroredState() async throws { + _ = try await client.startConversation(auth: .publicAgent(id: "test-agent")) + + let payload: [String: Any] = [ + "type": "agent_response", + "agent_response_event": [ + "agent_response": "Hello", + "event_id": 1 + ] + ] + try mockWebRTCConnectionManager.receive(data: JSONSerialization.data(withJSONObject: payload)) + await waitForPublished(client.$messages) { $0.last?.content == "Hello" } + + await client.reset() + + XCTAssertEqual(client.state, .idle) + XCTAssertTrue(client.messages.isEmpty) + XCTAssertTrue(client.pendingToolCalls.isEmpty) + XCTAssertNil(client.conversationMetadata) + XCTAssertFalse(client.isMicMuted) + + // Client is reusable after reset. + _ = try await client.startConversation(auth: .publicAgent(id: "after-reset")) + assertConnected(agentId: "after-reset") + } + + func testVoiceThenTextOnlyUsesFreshSessionPerTransport() async throws { + _ = try await client.startConversation(auth: .publicAgent(id: "voice-agent")) + XCTAssertEqual(mockWebRTCConnectionManager.connectCallCount, 1) + + var textConfig = ConversationConfig() + textConfig.conversationOverrides = ConversationOverrides(textOnly: true) + _ = try await client.startConversation(auth: .publicAgent(id: "text-agent"), config: textConfig) + + assertConnected(agentId: "text-agent") + XCTAssertEqual(mockWebSocketConnectionManager.connectCallCount, 1) + XCTAssertNil(mockWebRTCConnectionManager.onEventReceived) + XCTAssertNil(mockWebRTCConnectionManager.onDisconnected) + } + + func testStartFailureAfterBindMirrorsErrorAndCommandsRemainUnavailable() async throws { + mockWebRTCConnectionManager.tokenError = .authenticationFailed("Mock authentication failed") + + await XCTAssertThrowsErrorAsync { + _ = try await client.startConversation(auth: .publicAgent(id: "test-agent")) + } errorHandler: { error in + XCTAssertEqual(error as? ConversationError, .authenticationFailed("Mock authentication failed")) + } + + guard case let .error(error) = client.state else { + return XCTFail("Expected mirrored startup error after bind") + } + XCTAssertEqual(error, .authenticationFailed("Mock authentication failed")) + + await XCTAssertThrowsErrorAsync { + try await client.sendMessage("hello") + } errorHandler: { error in + XCTAssertEqual(error as? ConversationError, .notConnected) + } + + // A later start should still work on the same client. + mockWebRTCConnectionManager.tokenError = nil + _ = try await client.startConversation(auth: .publicAgent(id: "recovered-agent")) + assertConnected(agentId: "recovered-agent") + } + + func testCommandThrowsNotConnectedWithNoSession() async throws { + do { + try await client.sendMessage("hello") + XCTFail("Expected notConnected to be thrown") + } catch let error as ConversationError { + XCTAssertEqual(error, .notConnected) + } + } + + func testSetMicMutedUpdatesStateWithNoSession() async throws { + await client.endConversation() + try await client.setMicMuted(true) + XCTAssertTrue(client.isMicMuted) + + try await client.setMicMuted(false) + XCTAssertFalse(client.isMicMuted) + + client.markToolCallCompleted("missing-id") + + XCTAssertEqual(client.state, .idle) + } + + func testPreStartMuteIsAppliedToConversation() async throws { + mockWebRTCConnectionManager.isMicrophoneMuted = false + + try await client.setMicMuted(true) + _ = try await client.startConversation(auth: .publicAgent(id: "test-agent")) + + XCTAssertTrue(client.isMicMuted) + XCTAssertTrue(mockWebRTCConnectionManager.isMicrophoneMuted) + } + + func testSetMicMutedIsHarmlessAfterEnd() async throws { + _ = try await client.startConversation(auth: .publicAgent(id: "test-agent")) + await client.endConversation() + + try await client.setMicMuted(true) + + XCTAssertTrue(client.isMicMuted) + XCTAssertEqual(client.state, .ended(reason: .userEnded)) + } + + private func assertConnected( + agentId: String, + conversationId: String = "test-conversation-id", + file: StaticString = #filePath, + line: UInt = #line + ) { + guard case let .connected(info) = client.state else { + return XCTFail("Expected connected state, got \(client.state)", file: file, line: line) + } + XCTAssertEqual(info.agentId, agentId, file: file, line: line) + XCTAssertEqual(info.conversationId, conversationId, file: file, line: line) + } +} diff --git a/Tests/ElevenLabsTests/Unit/ConversationEventHandlerTests.swift b/Tests/ElevenLabsTests/Unit/ConversationEventHandlerTests.swift index 65dfb331..2e350224 100644 --- a/Tests/ElevenLabsTests/Unit/ConversationEventHandlerTests.swift +++ b/Tests/ElevenLabsTests/Unit/ConversationEventHandlerTests.swift @@ -1,4 +1,5 @@ @testable import ElevenLabs +import Foundation import XCTest @MainActor @@ -79,7 +80,6 @@ final class ConversationEventHandlerTests: XCTestCase { XCTAssertEqual(conversation.messages.last?.content, "I am an AI") XCTAssertEqual(conversation.messages.last?.role, .agent) XCTAssertEqual(conversation.messages.last?.eventId, 456) - XCTAssertEqual(conversation.lastAgentEventId, 456) } func testAgentResponseFinalizesStreamedMessageInsteadOfDuplicating() async { @@ -240,4 +240,53 @@ final class ConversationEventHandlerTests: XCTestCase { XCTAssertEqual(conversation.messages.last?.content, "Hello World!") XCTAssertEqual(conversation.messages.last?.eventId, 13) } + + // MARK: - End Call + + func testAutomaticEndCallHandling() async { + await conversation.handleIncomingEvent(.agentToolResponse(AgentToolResponseEvent( + toolName: "end_call", toolCallId: "id", toolType: "action", isError: false, eventId: 1 + ))) + + XCTAssertEqual(conversation.state, .ended(reason: .userEnded)) + } + + // MARK: - Audio Alignment + + func testAudioAlignmentCallback() async { + let receivedAlignment = expectation(description: "Audio alignment callback") + conversation = Conversation( + dependencyProvider: mockDependencyProvider, + callbacks: ConversationCallbacks(onAudioAlignment: { alignment in + XCTAssertEqual(alignment.chars, ["H", "e", "l", "l", "o"]) + receivedAlignment.fulfill() + }) + ) + + let alignment = AudioAlignment( + chars: ["H", "e", "l", "l", "o"], + charStartTimesMs: [0, 100, 200, 300, 400], + charDurationsMs: [100, 100, 100, 100, 100] + ) + await conversation.handleIncomingEvent( + .audio(AudioEvent(audioBase64: "base64", eventId: 1, alignment: alignment)) + ) + await fulfillment(of: [receivedAlignment], timeout: 1) + } + + // MARK: - Client Tool Calls + + func testClientToolCallIsQueuedInPendingToolCalls() async throws { + let toolCall = try ClientToolCallEvent( + toolName: "test_tool", + toolCallId: "call_123", + parametersData: JSONSerialization.data(withJSONObject: ["arg": "val"]), + eventId: 1, + expectsResponse: false + ) + await conversation.handleIncomingEvent(.clientToolCall(toolCall)) + + XCTAssertEqual(conversation.pendingToolCalls.count, 1) + XCTAssertEqual(conversation.pendingToolCalls.first?.toolCallId, "call_123") + } } diff --git a/Tests/ElevenLabsTests/Unit/ConversationInitiationMetadataWaiterTests.swift b/Tests/ElevenLabsTests/Unit/ConversationInitiationMetadataWaiterTests.swift new file mode 100644 index 00000000..316d7761 --- /dev/null +++ b/Tests/ElevenLabsTests/Unit/ConversationInitiationMetadataWaiterTests.swift @@ -0,0 +1,76 @@ +@testable import ElevenLabs +import XCTest + +final class ConversationInitiationMetadataWaiterTests: XCTestCase { + func testMetadataReceivedBeforeWait() async throws { + let waiter = ConversationInitiationMetadataWaiter(timeout: 1) + await waiter.observe(makeMetadata(conversationId: "early")) + + let metadata = try await waiter.wait() + + XCTAssertEqual(metadata.conversationId, "early") + } + + func testFirstMetadataWins() async throws { + let waiter = ConversationInitiationMetadataWaiter(timeout: 1) + await waiter.observe(makeMetadata(conversationId: "first")) + await waiter.observe(makeMetadata(conversationId: "second")) + + let metadata = try await waiter.wait() + + XCTAssertEqual(metadata.conversationId, "first") + } + + func testTimeout() async { + let waiter = ConversationInitiationMetadataWaiter(timeout: 0.01) + + await XCTAssertThrowsErrorAsync { + try await waiter.wait() + } errorHandler: { error in + XCTAssertEqual(error as? ConversationError, .initiationMetadataTimeout) + } + } + + func testTimeoutIsTerminal() async { + let waiter = ConversationInitiationMetadataWaiter(timeout: 0) + + await XCTAssertThrowsErrorAsync { + try await waiter.wait() + } + await waiter.observe(makeMetadata(conversationId: "late")) + await XCTAssertThrowsErrorAsync { + try await waiter.wait() + } errorHandler: { error in + XCTAssertEqual(error as? ConversationError, .initiationMetadataTimeout) + } + } + + func testCancellationIsTerminal() async { + let waiter = ConversationInitiationMetadataWaiter(timeout: 1) + let waitTask = Task { try await waiter.wait() } + + waitTask.cancel() + + do { + _ = try await waitTask.value + XCTFail("Expected cancellation") + } catch { + XCTAssertTrue(error is CancellationError) + } + + await waiter.observe(makeMetadata(conversationId: "late")) + await XCTAssertThrowsErrorAsync { + try await waiter.wait() + } errorHandler: { error in + XCTAssertTrue(error is CancellationError) + } + } + + private func makeMetadata(conversationId: String) -> ConversationMetadataEvent { + ConversationMetadataEvent( + conversationId: conversationId, + agentOutputAudioFormat: "pcm_16000", + userInputAudioFormat: "pcm_16000" + ) + } +} diff --git a/Tests/ElevenLabsTests/Unit/ConversationTests.swift b/Tests/ElevenLabsTests/Unit/ConversationTests.swift index cb27741f..26ad514f 100644 --- a/Tests/ElevenLabsTests/Unit/ConversationTests.swift +++ b/Tests/ElevenLabsTests/Unit/ConversationTests.swift @@ -22,7 +22,11 @@ final class ConversationTests: XCTestCase { webRTCConnectionManager: mockWebRTCConnectionManager, webSocketConnectionManager: mockWebSocketConnectionManager ) - conversation = Conversation(dependencyProvider: dependencyProvider, callbacks: makeCallbacks()) + conversation = Conversation( + dependencyProvider: dependencyProvider, + config: makeConfig(), + callbacks: makeCallbacks() + ) await capturedErrors.reset() } @@ -37,45 +41,47 @@ final class ConversationTests: XCTestCase { @MainActor func testConversationInitialState() { XCTAssertEqual(conversation.state, .idle) - XCTAssertTrue(conversation.isMicMuted) XCTAssertTrue(conversation.messages.isEmpty) } - func testStartConversationSuccessUpdatesStartupState() async throws { - let stateExpectation = expectation(description: "startup becomes connected") - + func testStartConversationSuccessReturnsResult() async throws { let config = makeConfig() - let callbacks = makeCallbacks(onStartupStateChange: { state in - if case .connected = state { - stateExpectation.fulfill() - } - }) - let conversation = Conversation(dependencyProvider: dependencyProvider, config: config, callbacks: callbacks) + let conversation = Conversation(dependencyProvider: dependencyProvider, config: config, callbacks: makeCallbacks()) - try await conversation.startConversation( - auth: ConversationCredentials.publicAgent(id: "test-agent-id"), - config: config + let result = try await conversation.start( + auth: ConversationCredentials.publicAgent(id: "test-agent-id") ) - await fulfillment(of: [stateExpectation], timeout: 1.0) - XCTAssertEqual(mockWebRTCConnectionManager.connectCallCount, 1) XCTAssertFalse(mockWebRTCConnectionManager.publishedPayloads.isEmpty) - XCTAssertEqual(conversation.state, ConversationState.connected(.init(agentId: "test-agent-id"))) - guard case let .connected(callInfo, metrics) = conversation.startupState else { - return XCTFail("Expected connected startup state") + guard case let .connected(callInfo) = conversation.state else { + return XCTFail("Expected connected state") } XCTAssertEqual(callInfo.agentId, "test-agent-id") - XCTAssertEqual(metrics.conversationInitAttempts, 1) - XCTAssertEqual(conversation.startupMetrics?.total, metrics.total) + XCTAssertEqual(callInfo.conversationId, "test-conversation-id") + XCTAssertEqual(result.callInfo, callInfo) + XCTAssertEqual(result.metrics.agentReady, 0) + XCTAssertNotNil(result.metrics.initiationMetadata) let errorsAfterSuccess = await capturedErrors.values() XCTAssertTrue(errorsAfterSuccess.isEmpty) } + func testConversationCanOnlyStartOnce() async throws { + _ = try await conversation.start(auth: .publicAgent(id: "first-agent")) + await conversation.endConversation() + + await XCTAssertThrowsErrorAsync { + _ = try await conversation.start(auth: .publicAgent(id: "second-agent")) + } errorHandler: { error in + XCTAssertEqual(error as? ConversationError, .alreadyStarted) + } + + XCTAssertEqual(mockWebRTCConnectionManager.connectCallCount, 1) + } + func testStartConversationConfiguresIncomingEventHandler() async throws { - try await conversation.startConversation( - auth: ConversationCredentials.publicAgent(id: "test-agent-id"), - config: makeConfig() + _ = try await conversation.start( + auth: ConversationCredentials.publicAgent(id: "test-agent-id") ) XCTAssertNotNil(mockWebRTCConnectionManager.onEventReceived) @@ -99,16 +105,16 @@ final class ConversationTests: XCTestCase { func testStartConversationHandlesIncomingDataBeforeAgentReady() async throws { // Hold agent-ready so we can deliver protocol data while still connecting. mockWebRTCConnectionManager.autoSucceedAgentReady = false + mockWebRTCConnectionManager.autoDeliverInitiationMetadata = false let startTask = Task { guard let conversation = self.conversation else { return } - try await conversation.startConversation( - auth: ConversationCredentials.publicAgent(id: "test-agent-id"), - config: makeConfig() + _ = try await conversation.start( + auth: ConversationCredentials.publicAgent(id: "test-agent-id") ) } - await waitForEventHandlerInstalled(on: mockWebRTCConnectionManager) + await mockWebRTCConnectionManager.waitForEventHandlerInstalled() guard mockWebRTCConnectionManager.onEventReceived != nil else { mockWebRTCConnectionManager.succeedAgentReady() @@ -133,12 +139,16 @@ final class ConversationTests: XCTestCase { mockWebRTCConnectionManager.succeedAgentReady() try await startTask.value + + guard case let .connected(callInfo) = conversation.state else { + return XCTFail("Expected connected state") + } + XCTAssertEqual(callInfo.conversationId, "conversation-before-ready") } func testStaleProtocolDataHandlerDoesNotMutateEndedConversation() async throws { - try await conversation.startConversation( - auth: ConversationCredentials.publicAgent(id: "test-agent-id"), - config: makeConfig() + _ = try await conversation.start( + auth: ConversationCredentials.publicAgent(id: "test-agent-id") ) let staleHandler = try XCTUnwrap(mockWebRTCConnectionManager.onEventReceived) @@ -155,13 +165,11 @@ final class ConversationTests: XCTestCase { func testStartConversationConfiguresRoomObservationHandlers() async throws { mockWebRTCConnectionManager.isMicrophoneMuted = false - try await conversation.startConversation( - auth: ConversationCredentials.publicAgent(id: "test-agent-id"), - config: makeConfig() + _ = try await conversation.start( + auth: ConversationCredentials.publicAgent(id: "test-agent-id") ) XCTAssertNotNil(mockWebRTCConnectionManager.onRemoteSpeakingChanged) - XCTAssertFalse(conversation.isMicMuted) mockWebRTCConnectionManager.onRemoteSpeakingChanged?(true) await waitForPublished(conversation.$agentState) { $0 == .speaking } @@ -170,15 +178,32 @@ final class ConversationTests: XCTestCase { } func testStartTextOnlyPublicAgentUsesWebSocketConnectionManager() async throws { + let startupStages = ValueRecorder() let config = makeConfig(configure: { config in config.conversationOverrides = ConversationOverrides(textOnly: true) }) + conversation = Conversation( + dependencyProvider: dependencyProvider, + config: config, + callbacks: makeCallbacks() + ) + var cancellable: AnyCancellable? + cancellable = conversation.$state.sink { state in + if case let .connecting(stage) = state { + Task { await startupStages.append(stage) } + } + } - try await conversation.startConversation( - auth: ConversationCredentials.publicAgent(id: "test-agent-id"), - config: config + _ = try await conversation.start( + auth: ConversationCredentials.publicAgent(id: "test-agent-id") ) + cancellable?.cancel() + let reportedStartupStages = await waitForValues(startupStages, count: 3) + XCTAssertEqual( + reportedStartupStages, + [.preparing, .sendingConversationInit, .waitingForInitiationMetadata(timeout: 5.0)] + ) XCTAssertEqual(mockWebRTCConnectionManager.connectCallCount, 0) XCTAssertEqual(mockWebSocketConnectionManager.connectCallCount, 1) XCTAssertEqual(mockWebSocketConnectionManager.lastConnectedURL?.scheme, "wss") @@ -190,7 +215,7 @@ final class ConversationTests: XCTestCase { ) XCTAssertFalse(mockWebSocketConnectionManager.sentPayloads.isEmpty) XCTAssertEqual(try sentEventType(from: mockWebSocketConnectionManager.sentPayloads[0]), "conversation_initiation_client_data") - XCTAssertEqual(conversation.state, ConversationState.connected(.init(agentId: "test-agent-id"))) + XCTAssertTrue(conversation.state.isConnected) let payload: [String: Any] = [ "type": "agent_response", @@ -207,50 +232,27 @@ final class ConversationTests: XCTestCase { XCTAssertEqual(conversation.messages.last?.role, .agent) } - func testTextOnlyStartDisconnectsPreviousActiveManagerBeforeSwitchingTransports() async throws { - try await conversation.startConversation(auth: .publicAgent(id: "test-agent-id"), config: makeConfig()) - await conversation.endConversation() - - // Re-arm stale handlers so the transport switch has something to clear. - mockWebRTCConnectionManager.onEventReceived = { _ in } - mockWebRTCConnectionManager.onDisconnected = {} - mockWebRTCConnectionManager.onRemoteSpeakingChanged = { _ in } - - let config = makeConfig(configure: { config in - config.conversationOverrides = ConversationOverrides(textOnly: true) - }) - - try await conversation.startConversation( - auth: ConversationCredentials.publicAgent(id: "test-agent-id"), - config: config - ) - - // 1 (reset-before-connect on the first start) + 1 (real endConversation) + 1 (stale - // manager disconnected when the second start switches transport) = 3. - XCTAssertEqual(mockWebRTCConnectionManager.disconnectCallCount, 3) - XCTAssertNil(mockWebRTCConnectionManager.onEventReceived) - XCTAssertNil(mockWebRTCConnectionManager.onDisconnected) - XCTAssertNil(mockWebRTCConnectionManager.onRemoteSpeakingChanged) - XCTAssertEqual(mockWebSocketConnectionManager.connectCallCount, 1) - XCTAssertEqual(conversation.state, ConversationState.connected(.init(agentId: "test-agent-id"))) - } - func testStartTextOnlySignedURLUsesProvidedWebSocketURL() async throws { let signedURL = "wss://api.elevenlabs.io/v1/convai/conversation?agent_id=agent-private&conversation_signature=sig" let config = makeConfig(configure: { config in config.conversationOverrides = ConversationOverrides(textOnly: true) }) - - try await conversation.startConversation( - auth: .signedWebSocketURL(signedURL), - config: config + conversation = Conversation( + dependencyProvider: dependencyProvider, + config: config, + callbacks: makeCallbacks() ) + _ = try await conversation.start(auth: .signedWebSocketURL(signedURL)) + XCTAssertEqual(mockWebRTCConnectionManager.connectCallCount, 0) XCTAssertEqual(mockWebSocketConnectionManager.connectCallCount, 1) XCTAssertEqual(mockWebSocketConnectionManager.lastConnectedURL?.absoluteString, signedURL) XCTAssertFalse(mockWebSocketConnectionManager.sentPayloads.isEmpty) - XCTAssertEqual(conversation.state, ConversationState.connected(.init(agentId: "agent-private"))) + guard case let .connected(info) = conversation.state else { + return XCTFail("Expected connected state") + } + XCTAssertEqual(info.agentId, "agent-private") } func testSignedWebSocketURLRejectsURLWithoutAgentId() { @@ -268,12 +270,14 @@ final class ConversationTests: XCTestCase { let config = makeConfig(configure: { config in config.conversationOverrides = ConversationOverrides(textOnly: true) }) + conversation = Conversation( + dependencyProvider: dependencyProvider, + config: config, + callbacks: makeCallbacks() + ) do { - try await conversation.startConversation( - auth: .conversationToken("livekit-token"), - config: config - ) + _ = try await conversation.start(auth: .conversationToken("livekit-token")) XCTFail("Expected text-only startup to reject LiveKit token auth") } catch let error as ConversationError { guard case .authenticationFailed = error else { @@ -296,39 +300,24 @@ final class ConversationTests: XCTestCase { } @MainActor - func testToggleMicMuteWhenNotConnected() async { - do { - try await conversation.toggleMicMute() - XCTFail("Should throw error when not connected") - } catch let error as ConversationError { - XCTAssertEqual(error, .notConnected) - } catch { - XCTFail("Unexpected error type") - } - } + func testSetMicMutedWhileIdleAppliesOnStart() async throws { + mockWebRTCConnectionManager.isMicrophoneMuted = false - @MainActor - func testSetMicMutedWhenNotConnected() async { - do { - try await conversation.setMicMuted(true) - XCTFail("Should throw error when not connected") - } catch let error as ConversationError { - XCTAssertEqual(error, .notConnected) - } catch { - XCTFail("Unexpected error type") - } + try await conversation.setMicMuted(true) + _ = try await conversation.start(auth: .publicAgent(id: "test-agent")) + + XCTAssertTrue(mockWebRTCConnectionManager.isMicrophoneMuted) } @MainActor func testSetHardwareMicMutedUsesConnectionManagerAudioControl() async throws { mockWebRTCConnectionManager.isMicrophoneMuted = false - try await conversation.startConversation(auth: .publicAgent(id: "test-agent"), config: makeConfig()) + _ = try await conversation.start(auth: .publicAgent(id: "test-agent")) try await conversation.setHardwareMicMuted(true) XCTAssertTrue(mockWebRTCConnectionManager.isMicrophoneMuted) - XCTAssertTrue(conversation.isMicMuted) } @MainActor @@ -337,13 +326,17 @@ final class ConversationTests: XCTestCase { let config = makeConfig(configure: { config in config.audioConfiguration = AudioPipelineConfiguration(microphoneMuteMode: .software()) }) + conversation = Conversation( + dependencyProvider: dependencyProvider, + config: config, + callbacks: makeCallbacks() + ) - try await conversation.startConversation(auth: .publicAgent(id: "test-agent"), config: config) + _ = try await conversation.start(auth: .publicAgent(id: "test-agent")) try await conversation.setMicMuted(true) XCTAssertFalse(mockWebRTCConnectionManager.isMicrophoneMuted) - XCTAssertTrue(conversation.isMicMuted) } @MainActor @@ -385,25 +378,18 @@ final class ConversationTests: XCTestCase { func testStartConversationTokenFailure() async { mockWebRTCConnectionManager.tokenError = .authenticationFailed("Mock authentication failed") - let config = makeConfig() - guard let conversation else { return } await XCTAssertThrowsErrorAsync { - try await conversation.startConversation( - auth: .publicAgent(id: "test-agent"), - config: config - ) + _ = try await conversation.start(auth: .publicAgent(id: "test-agent")) } errorHandler: { error in XCTAssertEqual(error as? ConversationError, .authenticationFailed("Mock authentication failed")) } - guard case let .failed(.token(conversationError), metrics) = conversation.startupState else { - return XCTFail("Expected startup failure due to token") + guard case let .error(conversationError) = conversation.state else { + return XCTFail("Expected error state due to token failure") } XCTAssertEqual(conversationError, .authenticationFailed("Mock authentication failed")) - XCTAssertEqual(conversation.state, .idle) - XCTAssertEqual(conversation.startupMetrics?.tokenFetch, metrics.tokenFetch) let errorsAfterTokenFailure = await waitForValues(capturedErrors, count: 1) XCTAssertEqual(errorsAfterTokenFailure, [.authenticationFailed("Mock authentication failed")]) } @@ -411,59 +397,46 @@ final class ConversationTests: XCTestCase { func testStartConversationConnectionFailure() async { mockWebRTCConnectionManager.shouldFailConnection = true - let config = makeConfig() - guard let conversation else { return } await XCTAssertThrowsErrorAsync { - try await conversation.startConversation( - auth: .publicAgent(id: "test-agent"), - config: config - ) + _ = try await conversation.start(auth: .publicAgent(id: "test-agent")) } errorHandler: { error in XCTAssertEqual(error as? ConversationError, .connectionFailed("Mock connection failed")) } - guard case let .failed(.room(conversationError), metrics) = conversation.startupState else { - return XCTFail("Expected startup failure due to room connect") + guard case let .error(conversationError) = conversation.state else { + return XCTFail("Expected error state due to room connect failure") } XCTAssertEqual(conversationError, .connectionFailed("Mock connection failed")) - XCTAssertEqual(conversation.state, .idle) - XCTAssertEqual(conversation.startupMetrics?.roomConnect, metrics.roomConnect) + mockWebRTCConnectionManager.deliverStartupState(.waitingForAgent(timeout: 1)) + XCTAssertEqual(conversation.state, .error(conversationError)) + let errorsAfterConnectionFailure = await waitForValues(capturedErrors, count: 1) XCTAssertEqual(errorsAfterConnectionFailure, [.connectionFailed("Mock connection failed")]) } func testStartConversationAgentTimeoutFailure() async { mockWebRTCConnectionManager.autoSucceedAgentReady = false + mockWebRTCConnectionManager.timeoutAgentReady() let startupConfig = ConversationStartupConfiguration(agentReadyTimeout: 0.05) let config = makeConfig(startupConfiguration: startupConfig) - - let startTask = Task { - guard let conversation = self.conversation else { return } - try await conversation.startConversation( - auth: .publicAgent(id: "test-agent"), - config: config - ) - } - - await waitForEventHandlerInstalled(on: mockWebRTCConnectionManager) - try? await conversation.setMicMuted(false) - XCTAssertFalse(conversation.isMicMuted) - mockWebRTCConnectionManager.timeoutAgentReady() + conversation = Conversation( + dependencyProvider: dependencyProvider, + config: config, + callbacks: makeCallbacks() + ) await XCTAssertThrowsErrorAsync { - try await startTask.value + _ = try await conversation.start(auth: .publicAgent(id: "test-agent")) } errorHandler: { error in XCTAssertEqual(error as? ConversationError, .agentTimeout) } - guard case .failed(.agentTimeout, _) = conversation.startupState else { - return XCTFail("Expected agent timeout failure state") + guard case .error(.agentTimeout) = conversation.state else { + return XCTFail("Expected agent timeout error state") } - XCTAssertEqual(conversation.state, .idle) - XCTAssertTrue(conversation.isMicMuted) XCTAssertNil(mockWebRTCConnectionManager.room) XCTAssertNil(mockWebRTCConnectionManager.onDisconnected) XCTAssertNil(mockWebRTCConnectionManager.onEventReceived) @@ -473,24 +446,105 @@ final class ConversationTests: XCTestCase { XCTAssertEqual(errorsAfterAgentTimeout, [.agentTimeout]) } + func testCancelledStartupEndsConversation() async { + mockWebRTCConnectionManager.autoDeliverInitiationMetadata = false + let startTask = Task { + try await conversation.start(auth: .publicAgent(id: "test-agent")) + } + + await waitForPublished(conversation.$state) { + $0 == .connecting(.waitingForInitiationMetadata(timeout: 5)) + } + startTask.cancel() + + await XCTAssertThrowsErrorAsync { + try await startTask.value + } errorHandler: { error in + XCTAssertTrue(error is CancellationError) + } + XCTAssertEqual(conversation.state, .ended(reason: .userEnded)) + } + + func testEndDuringStartupRemainsEnded() async { + mockWebRTCConnectionManager.autoDeliverInitiationMetadata = false + let startTask = Task { + try await conversation.start(auth: .publicAgent(id: "test-agent")) + } + + await waitForPublished(conversation.$state) { + $0 == .connecting(.waitingForInitiationMetadata(timeout: 5)) + } + await conversation.endConversation() + + await XCTAssertThrowsErrorAsync { + try await startTask.value + } errorHandler: { error in + XCTAssertTrue(error is CancellationError) + } + XCTAssertEqual(conversation.state, .ended(reason: .userEnded)) + } + + func testEndDuringWaitingForAgentDoesNotReportFailure() async { + mockWebRTCConnectionManager.autoSucceedAgentReady = false + let startTask = Task { + try await conversation.start(auth: .publicAgent(id: "test-agent")) + } + + await waitForPublished(conversation.$state) { + if case .connecting(.waitingForAgent) = $0 { return true } + return false + } + await conversation.endConversation() + + await XCTAssertThrowsErrorAsync { + try await startTask.value + } errorHandler: { error in + XCTAssertTrue(error is CancellationError) + } + XCTAssertEqual(conversation.state, .ended(reason: .userEnded)) + // onError appends asynchronously; give it a beat, then confirm End didn't fake a timeout. + try? await Task.sleep(nanoseconds: 100_000_000) + let errors = await capturedErrors.values() + XCTAssertTrue(errors.isEmpty) + } + + func testStartConversationInitiationMetadataTimeout() async { + mockWebRTCConnectionManager.autoDeliverInitiationMetadata = false + + let startupConfig = ConversationStartupConfiguration(initiationMetadataTimeout: 0.01) + let config = makeConfig(startupConfiguration: startupConfig) + conversation = Conversation( + dependencyProvider: dependencyProvider, + config: config, + callbacks: makeCallbacks() + ) + + await XCTAssertThrowsErrorAsync { + _ = try await conversation.start(auth: .publicAgent(id: "test-agent")) + } errorHandler: { error in + XCTAssertEqual(error as? ConversationError, .initiationMetadataTimeout) + } + + guard case .error(.initiationMetadataTimeout) = conversation.state else { + return XCTFail("Expected initiation metadata timeout error state") + } + let errorsAfterTimeout = await waitForValues(capturedErrors, count: 1) + XCTAssertEqual(errorsAfterTimeout, [.initiationMetadataTimeout]) + } + func testStartConversationConversationInitFailure() async { mockWebRTCConnectionManager.publishError = ConversationError.connectionFailed("Publish failed") - let config = makeConfig() - guard let conversation else { return } await XCTAssertThrowsErrorAsync { - try await conversation.startConversation( - auth: .publicAgent(id: "test-agent"), - config: config - ) + _ = try await conversation.start(auth: .publicAgent(id: "test-agent")) } errorHandler: { error in XCTAssertEqual(error as? ConversationError, .connectionFailed("Publish failed")) } - guard case let .failed(.conversationInit(conversationError), _) = conversation.startupState else { - return XCTFail("Expected conversation init failure state") + guard case let .error(conversationError) = conversation.state else { + return XCTFail("Expected conversation init error state") } XCTAssertEqual(conversationError, .connectionFailed("Publish failed")) @@ -503,36 +557,21 @@ final class ConversationTests: XCTestCase { XCTAssertEqual(errorsAfterInitFailure, [.connectionFailed("Publish failed")]) } - func testAgentResponseCallbackTogglesFeedbackAvailability() async throws { - let gotResponse = expectation(description: "agent response") - // Feedback flips more than once (start→false, response→true, feedback→false). - let feedbackStates = ValueRecorder() - - let callbacks = makeCallbacks(configure: { callbacks in - callbacks.onAgentResponse = { text, eventId in - XCTAssertEqual(text, "Hello") - XCTAssertEqual(eventId, 42) - gotResponse.fulfill() - } - callbacks.onCanSendFeedbackChange = { canSend in - Task { await feedbackStates.append(canSend) } - } - }) - - let conversation = Conversation(dependencyProvider: dependencyProvider, callbacks: callbacks) - - try await conversation.startConversation(auth: .publicAgent(id: "test"), config: makeConfig()) - - mockWebRTCConnectionManager.deliver(.agentResponse(AgentResponseEvent(response: "Hello", eventId: 42))) + func testSendFeedbackWhileConnected() async throws { + let conversation = Conversation( + dependencyProvider: dependencyProvider, + config: makeConfig() + ) - await fulfillment(of: [gotResponse], timeout: 1.0) - let initialFeedbackState = await waitForLastValue(feedbackStates) { $0 == true } - XCTAssertEqual(initialFeedbackState, true) + _ = try await conversation.start(auth: .publicAgent(id: "test")) try await conversation.sendFeedback(FeedbackEvent.Score.like, eventId: 42) - let updatedFeedbackState = await waitForLastValue(feedbackStates) { $0 == false } - XCTAssertEqual(updatedFeedbackState, false) + let payload = try XCTUnwrap(mockWebRTCConnectionManager.publishedPayloads.last) + let json = try XCTUnwrap(JSONSerialization.jsonObject(with: payload) as? [String: Any]) + XCTAssertEqual(json["type"] as? String, "feedback") + XCTAssertEqual(json["score"] as? String, "like") + XCTAssertEqual(json["event_id"] as? Int, 42) } func testVadScoreCallbackReceivesScores() async throws { @@ -544,8 +583,12 @@ final class ConversationTests: XCTestCase { } }) - let conversation = Conversation(dependencyProvider: dependencyProvider, callbacks: callbacks) - try await conversation.startConversation(auth: .publicAgent(id: "test"), config: makeConfig()) + let conversation = Conversation( + dependencyProvider: dependencyProvider, + config: makeConfig(), + callbacks: callbacks + ) + _ = try await conversation.start(auth: .publicAgent(id: "test")) mockWebRTCConnectionManager.deliver(.vadScore(VadScoreEvent(vadScore: 0.87))) @@ -561,8 +604,12 @@ final class ConversationTests: XCTestCase { } }) - let conversation = Conversation(dependencyProvider: dependencyProvider, callbacks: callbacks) - try await conversation.startConversation(auth: .publicAgent(id: "test"), config: makeConfig()) + let conversation = Conversation( + dependencyProvider: dependencyProvider, + config: makeConfig(), + callbacks: callbacks + ) + _ = try await conversation.start(auth: .publicAgent(id: "test")) mockWebRTCConnectionManager.deliver(.agentToolResponse(AgentToolResponseEvent( toolName: "lookup_weather", toolCallId: "id", toolType: "action", isError: false, eventId: 10 @@ -571,34 +618,12 @@ final class ConversationTests: XCTestCase { await fulfillment(of: [gotTool], timeout: 1.0) } - func testInterruptionCallbackDisablesFeedback() async throws { - let gotInterruption = expectation(description: "interruption") - let feedbackStates = ValueRecorder() - - let callbacks = makeCallbacks(configure: { callbacks in - callbacks.onInterruption = { id in - XCTAssertEqual(id, 7) - gotInterruption.fulfill() - } - callbacks.onCanSendFeedbackChange = { canSend in - Task { await feedbackStates.append(canSend) } - } - }) - - let conversation = Conversation(dependencyProvider: dependencyProvider, callbacks: callbacks) - try await conversation.startConversation(auth: .publicAgent(id: "test"), config: makeConfig()) - - mockWebRTCConnectionManager.deliver(.interruption(InterruptionEvent(eventId: 7))) - - await fulfillment(of: [gotInterruption], timeout: 1.0) - let interruptionFeedbackState = await waitForLastValue(feedbackStates) { $0 == false } - XCTAssertEqual(interruptionFeedbackState, false) - } - @MainActor func testSendToolResultWhenNotConnected() async { do { - try await conversation.sendToolResult(for: "tool-id", result: "result", isError: false) + try await conversation.sendToolResult( + .init(toolCallId: "tool-id", result: "result") + ) XCTFail("Should throw error when not connected") } catch let error as ConversationError { XCTAssertEqual(error, .notConnected) @@ -607,32 +632,87 @@ final class ConversationTests: XCTestCase { } } + func testToolCallLifecycle() async throws { + _ = try await conversation.start(auth: .publicAgent(id: "test")) + + let toolCall = try ClientToolCallEvent( + toolName: "test_tool", + toolCallId: "call_123", + parametersData: JSONSerialization.data(withJSONObject: ["arg": "val"]), + eventId: 1, + expectsResponse: false + ) + mockWebRTCConnectionManager.deliver(.clientToolCall(toolCall)) + await waitForPublished(conversation.$pendingToolCalls) { $0.contains { $0.toolCallId == "call_123" } } + + XCTAssertEqual(conversation.pendingToolCalls.count, 1) + XCTAssertEqual(conversation.pendingToolCalls.first?.toolCallId, "call_123") + + let payloadCountBeforeResult = mockWebRTCConnectionManager.publishedPayloads.count + try await conversation.sendToolResult(.init(toolCallId: "call_123", result: "success")) + + XCTAssertTrue(conversation.pendingToolCalls.isEmpty) + XCTAssertEqual(mockWebRTCConnectionManager.publishedPayloads.count, payloadCountBeforeResult + 1) + let lastPayload = mockWebRTCConnectionManager.publishedPayloads.last ?? Data() + let lastPayloadString = String(data: lastPayload, encoding: .utf8) ?? "" + XCTAssertTrue(lastPayloadString.contains("call_123")) + XCTAssertTrue(lastPayloadString.contains("success")) + } + + func testSendToolResultEncodesEncodableResult() async throws { + struct Weather: Encodable { + let temperature: Int + let condition: String + } + + _ = try await conversation.start(auth: .publicAgent(id: "test")) + + try await conversation.sendToolResult( + .init( + toolCallId: "call_42", + result: Weather(temperature: 25, condition: "Sunny") + ) + ) + + let payload = try XCTUnwrap(mockWebRTCConnectionManager.publishedPayloads.last) + let envelope = try XCTUnwrap(JSONSerialization.jsonObject(with: payload) as? [String: Any]) + XCTAssertEqual(envelope["type"] as? String, "client_tool_result") + let resultString = try XCTUnwrap(envelope["result"] as? String) + let parsed = try JSONSerialization.jsonObject(with: XCTUnwrap(resultString.data(using: .utf8))) as? [String: Any] + XCTAssertEqual(parsed?["temperature"] as? Int, 25) + XCTAssertEqual(parsed?["condition"] as? String, "Sunny") + } + @MainActor - func testEndConversationWhenNotConnected() async { - // Should not throw error when ending a non-connected conversation + func testEndIdleConversation() async { await conversation.endConversation() - XCTAssertEqual(conversation.state, .idle) + XCTAssertEqual(conversation.state, .ended(reason: .userEnded)) } func testConversationErrorEquality() { XCTAssertEqual(ConversationError.notConnected, ConversationError.notConnected) - XCTAssertEqual(ConversationError.alreadyActive, ConversationError.alreadyActive) + XCTAssertEqual(ConversationError.alreadyStarted, ConversationError.alreadyStarted) XCTAssertEqual(ConversationError.authenticationFailed("test"), ConversationError.authenticationFailed("test")) XCTAssertEqual(ConversationError.connectionFailed("test"), ConversationError.connectionFailed("test")) XCTAssertEqual(ConversationError.agentTimeout, ConversationError.agentTimeout) + XCTAssertEqual(ConversationError.initiationMetadataTimeout, ConversationError.initiationMetadataTimeout) XCTAssertEqual(ConversationError.microphoneToggleFailed("test"), ConversationError.microphoneToggleFailed("test")) - XCTAssertNotEqual(ConversationError.notConnected, ConversationError.alreadyActive) + XCTAssertNotEqual(ConversationError.notConnected, ConversationError.alreadyStarted) } func testConversationStateEnum() { let idleState: ConversationState = .idle - let connectingState: ConversationState = .connecting - let connectedState: ConversationState = .connected(CallInfo(agentId: "test")) + let connectingState: ConversationState = .connecting(.preparing) + let connectedState: ConversationState = .connected( + CallInfo(agentId: "test", conversationId: "conversation") + ) XCTAssertNotEqual(idleState, connectingState) XCTAssertNotEqual(connectingState, connectedState) XCTAssertNotEqual(idleState, connectedState) + XCTAssertTrue(connectingState.isConnecting) + XCTAssertTrue(connectedState.isConnected) } func testFeedbackTypeEnum() { @@ -646,17 +726,16 @@ final class ConversationTests: XCTestCase { let config = makeConfig() let callbacks = makeCallbacks(configure: { callbacks in callbacks.onDisconnect = { reason in - XCTAssertEqual(reason, .agent) + XCTAssertEqual(reason, .remoteDisconnected) gotDisconnect.fulfill() } }) let conversation = Conversation(dependencyProvider: dependencyProvider, config: config, callbacks: callbacks) - try await conversation.startConversation( - auth: ConversationCredentials.publicAgent(id: "test-agent-id"), - config: config + _ = try await conversation.start( + auth: ConversationCredentials.publicAgent(id: "test-agent-id") ) - XCTAssertEqual(conversation.state, ConversationState.connected(.init(agentId: "test-agent-id"))) + XCTAssertTrue(conversation.state.isConnected) let disconnectsBefore = mockWebRTCConnectionManager.disconnectCallCount await mockWebRTCConnectionManager.onDisconnected?() @@ -684,10 +763,9 @@ extension ConversationTests { } private func makeCallbacks( - onStartupStateChange: (@Sendable (ConversationStartupState) -> Void)? = nil, configure: ((inout ConversationCallbacks) -> Void)? = nil ) -> ConversationCallbacks { - var callbacks = ConversationCallbacks(onStartupStateChange: onStartupStateChange) + var callbacks = ConversationCallbacks() callbacks.onError = { [capturedErrors] error in Task { await capturedErrors.append(error) } diff --git a/Tests/ElevenLabsTests/Unit/ElevenLabsSDKTests.swift b/Tests/ElevenLabsTests/Unit/ElevenLabsSDKTests.swift index 4b8e88fe..0ae3dc66 100644 --- a/Tests/ElevenLabsTests/Unit/ElevenLabsSDKTests.swift +++ b/Tests/ElevenLabsTests/Unit/ElevenLabsSDKTests.swift @@ -7,50 +7,13 @@ final class ElevenLabsSDKTests: XCTestCase { XCTAssertFalse(ElevenLabs.version.isEmpty) } - func testDefaultConfiguration() { - let config = ElevenLabs.Configuration.default - - XCTAssertNil(config.apiEndpoint) - XCTAssertEqual(config.logLevel, .warning) - XCTAssertFalse(config.debugMode) - } - - func testCustomConfiguration() { - let config = ElevenLabs.Configuration( - apiEndpoint: URL(string: "https://custom.api.com"), - logLevel: .debug, - debugMode: true - ) - - XCTAssertEqual(config.apiEndpoint, URL(string: "https://custom.api.com")) - XCTAssertEqual(config.logLevel, .debug) - XCTAssertTrue(config.debugMode) - } - @MainActor - func testConfigureSDK() { - let config = ElevenLabs.Configuration( - apiEndpoint: URL(string: "https://test.api.com"), - logLevel: .info, - debugMode: false - ) - - ElevenLabs.configure(config) - - // Verify configuration was applied (in real implementation) - // This would require exposing the internal configuration for testing - } - func testStartConversationWithAgentId() async { let config = ConversationConfig() + let client = ConversationClient() do { - let conversation = try await ElevenLabs.startConversation( - agentId: "test-agent-123", - config: config - ) - - XCTAssertNotNil(conversation) + _ = try await client.startConversation(agentId: "test-agent-123", config: config) // In a proper test environment with mocks, we'd verify connection } catch { // Expected to fail without proper API setup @@ -95,16 +58,37 @@ final class ElevenLabsSDKTests: XCTestCase { } } - func testConfigurationLogLevels() { - let debugConfig = ElevenLabs.Configuration(logLevel: .debug) - let infoConfig = ElevenLabs.Configuration(logLevel: .info) - let warningConfig = ElevenLabs.Configuration(logLevel: .warning) - let errorConfig = ElevenLabs.Configuration(logLevel: .error) + func testEndpointsDefaultToProduction() { + XCTAssertEqual(ConversationConfig().endpoints, .production) + XCTAssertEqual(Endpoints(), .production) + } + + func testEndpointsOverrideIndividualHosts() throws { + let endpoints = try Endpoints(voiceWebSocket: XCTUnwrap(URL(string: "wss://rtc.example.com"))) + XCTAssertEqual(endpoints.voiceWebSocket.absoluteString, "wss://rtc.example.com") + XCTAssertEqual(endpoints.textWebSocket, Endpoints.production.textWebSocket) + XCTAssertEqual(endpoints.apiBase, Endpoints.production.apiBase) + } + + func testConversationTokenDerivesFromAPIBase() throws { + let endpoints = try Endpoints(apiBase: XCTUnwrap(URL(string: "https://proxy.example.com/eleven"))) + XCTAssertEqual( + endpoints.conversationToken.absoluteString, + "https://proxy.example.com/eleven/v1/convai/conversation/token" + ) + XCTAssertEqual( + Endpoints.production.conversationToken.absoluteString, + "https://api.elevenlabs.io/v1/convai/conversation/token" + ) + } - XCTAssertEqual(debugConfig.logLevel, .debug) - XCTAssertEqual(infoConfig.logLevel, .info) - XCTAssertEqual(warningConfig.logLevel, .warning) - XCTAssertEqual(errorConfig.logLevel, .error) + func testWebsocketUrlKeepsEndpointQueryItems() throws { + let endpoints = try Endpoints(textWebSocket: XCTUnwrap(URL(string: "wss://proxy.example.com/ws?tenant=acme"))) + let url = try WebSocketConnectionManager.websocketUrl( + for: .publicAgent(id: "agent-123"), + endpoints: endpoints + ) + XCTAssertEqual(url.absoluteString, "wss://proxy.example.com/ws?tenant=acme&agent_id=agent-123") } func testConversationConfigDefaults() { @@ -146,11 +130,12 @@ final class ElevenLabsSDKTests: XCTestCase { func testSDKModuleImports() { // Verify that all necessary types are accessible - XCTAssertNotNil(ElevenLabs.self) - XCTAssertNotNil(Conversation.self) + XCTAssertNotNil(ConversationClient.self) XCTAssertNotNil(ConversationConfig.self) XCTAssertNotNil(ConversationError.self) XCTAssertNotNil(ConversationState.self) XCTAssertNotNil(Language.self) + XCTAssertNotNil(LogLevel.self) + XCTAssertNotNil(AgentState.self) } } diff --git a/Tests/ElevenLabsTests/Unit/ErrorHandlingIntegrationTests.swift b/Tests/ElevenLabsTests/Unit/ErrorHandlingIntegrationTests.swift index bad8e07e..08ed73ed 100644 --- a/Tests/ElevenLabsTests/Unit/ErrorHandlingIntegrationTests.swift +++ b/Tests/ElevenLabsTests/Unit/ErrorHandlingIntegrationTests.swift @@ -1,4 +1,5 @@ // swiftlint:disable file_length type_body_length function_body_length +import Combine @testable import ElevenLabs import XCTest @@ -37,10 +38,6 @@ final class ErrorHandlingIntegrationTests: XCTestCase { let config = ConversationConfig() let callbacks = ConversationCallbacks( - onStartupStateChange: { state in - print("📊 Startup state: \(state)") - Task { await collector.addState(state) } - }, onError: { error in print("✅ onError callback received: \(error)") Task { await collector.addError(error) } @@ -57,9 +54,8 @@ final class ErrorHandlingIntegrationTests: XCTestCase { // Use an invalid agent ID to trigger an error do { - try await conversation.startConversation( - auth: ConversationCredentials.publicAgent(id: "invalid_agent_id_12345"), - config: config + _ = try await conversation.start( + auth: ConversationCredentials.publicAgent(id: "invalid_agent_id_12345") ) XCTFail("Should have thrown an error") } catch { @@ -105,10 +101,6 @@ final class ErrorHandlingIntegrationTests: XCTestCase { print("✅ Agent ready!") readyExpectation.fulfill() }, - onStartupStateChange: { state in - print("📊 Startup state: \(state)") - Task { await collector.addState(state) } - }, onError: { error in print("❌ Unexpected error in success test: \(error)") Task { await collector.addError(error) } @@ -123,9 +115,8 @@ final class ErrorHandlingIntegrationTests: XCTestCase { self.conversation = conversation do { - try await conversation.startConversation( - auth: ConversationCredentials.publicAgent(id: testAgentId), - config: config + let result = try await conversation.start( + auth: ConversationCredentials.publicAgent(id: testAgentId) ) await fulfillment(of: [readyExpectation], timeout: 15.0) @@ -147,14 +138,11 @@ final class ErrorHandlingIntegrationTests: XCTestCase { print(" \(index + 1). \(state)") } - if case let .connected(_, metrics) = conversation.startupState { - print("\n⏱️ Startup metrics:") - print(" Total: \(String(format: "%.3f", metrics.total ?? 0))s") - print(" Token fetch: \(String(format: "%.3f", metrics.tokenFetch ?? 0))s") - print(" Room connect: \(String(format: "%.3f", metrics.roomConnect ?? 0))s") - print(" Agent ready: \(String(format: "%.3f", metrics.agentReady ?? 0))s") - print(" Init attempts: \(metrics.conversationInitAttempts)") - } + print("\n⏱️ Startup metrics:") + print(" Total: \(String(format: "%.3f", result.metrics.total ?? 0))s") + print(" Token fetch: \(String(format: "%.3f", result.metrics.tokenFetch ?? 0))s") + print(" Room connect: \(String(format: "%.3f", result.metrics.roomConnect ?? 0))s") + print(" Agent ready: \(String(format: "%.3f", result.metrics.agentReady ?? 0))s") // Clean disconnect await conversation.endConversation() @@ -196,9 +184,8 @@ final class ErrorHandlingIntegrationTests: XCTestCase { ) self.conversation = conversation - try await conversation.startConversation( - auth: ConversationCredentials.publicAgent(id: testAgentId), - config: config + _ = try await conversation.start( + auth: ConversationCredentials.publicAgent(id: testAgentId) ) await fulfillment(of: [readyExpectation], timeout: 15.0) @@ -240,9 +227,6 @@ final class ErrorHandlingIntegrationTests: XCTestCase { networkConfiguration: networkConfig ) let callbacks = ConversationCallbacks( - onStartupStateChange: { state in - print(" 📊 State: \(state)") - }, onError: { error in print(" ❌ Error: \(error)") } @@ -255,9 +239,8 @@ final class ErrorHandlingIntegrationTests: XCTestCase { ) do { - try await conversation.startConversation( - auth: ConversationCredentials.publicAgent(id: testAgentId), - config: config + _ = try await conversation.start( + auth: ConversationCredentials.publicAgent(id: testAgentId) ) print(" ✅ Connection \(attempt) successful") @@ -277,23 +260,15 @@ final class ErrorHandlingIntegrationTests: XCTestCase { } } - /// Test that startup state transitions are properly reported including failures + /// Test that startup stages are reported and failures land in `.error`. func testStartupStateTransitions() async throws { - let errorExpectation = expectation(description: "Should receive error state") - let collector = ErrorCollector() + let errorExpectation = expectation(description: "Should receive error callback") let config = ConversationConfig() let callbacks = ConversationCallbacks( - onStartupStateChange: { state in - print("📊 State transition: \(state)") - Task { await collector.addState(state) } - - if case .failed = state { - errorExpectation.fulfill() - } - }, onError: { error in print("❌ Error: \(error)") + errorExpectation.fulfill() } ) @@ -304,35 +279,31 @@ final class ErrorHandlingIntegrationTests: XCTestCase { ) self.conversation = conversation + var observedResolvingToken = false + var cancellable: AnyCancellable? + cancellable = conversation.$state.sink { state in + if case .connecting(.resolvingToken) = state { + observedResolvingToken = true + } + } + // Use invalid agent to trigger failure do { - try await conversation.startConversation( - auth: ConversationCredentials.publicAgent(id: "invalid_agent"), - config: config + _ = try await conversation.start( + auth: ConversationCredentials.publicAgent(id: "invalid_agent") ) } catch { // Expected } + cancellable?.cancel() await fulfillment(of: [errorExpectation], timeout: 5.0) - let capturedStartupStates = await collector.states + XCTAssertTrue(observedResolvingToken, "Should have resolvingToken stage") - print("\n📊 State transition sequence (\(capturedStartupStates.count) states):") - for (index, state) in capturedStartupStates.enumerated() { - print(" \(index + 1). \(state)") + guard case .error = conversation.state else { + return XCTFail("Expected error state after startup failure") } - - // Verify we got expected state transitions - XCTAssertTrue(capturedStartupStates.contains { state in - if case .resolvingToken = state { return true } - return false - }, "Should have resolvingToken state") - - XCTAssertTrue(capturedStartupStates.contains { state in - if case .failed = state { return true } - return false - }, "Should have failed state") } /// Test custom token provider error handling @@ -358,13 +329,12 @@ final class ErrorHandlingIntegrationTests: XCTestCase { // Provide a token provider that throws an error do { - try await conversation.startConversation( + _ = try await conversation.start( auth: .customTokenProvider { throw NSError(domain: "TestError", code: 500, userInfo: [ NSLocalizedDescriptionKey: "Custom token provider failed" ]) - }, - config: config + } ) XCTFail("Should have thrown error") } catch { @@ -386,9 +356,6 @@ final class ErrorHandlingIntegrationTests: XCTestCase { let collector = ErrorCollector() let config = ConversationConfig() let callbacks = ConversationCallbacks( - onStartupStateChange: { state in - print("📊 State: \(state)") - }, onError: { error in print("❌ Network error: \(error)") Task { await collector.addError(error) } @@ -404,9 +371,8 @@ final class ErrorHandlingIntegrationTests: XCTestCase { // Attempt connection - may succeed or fail depending on network do { - try await conversation.startConversation( - auth: ConversationCredentials.publicAgent(id: testAgentId), - config: config + _ = try await conversation.start( + auth: ConversationCredentials.publicAgent(id: testAgentId) ) print("✅ Connection succeeded (network available)") await conversation.endConversation() @@ -458,13 +424,6 @@ extension ErrorHandlingIntegrationTests { let timestamp = Self.formatTimestamp() print("🔌 [\(timestamp)] DISCONNECTED (reason: \(reason))") }, - onStartupStateChange: { state in - let timestamp = Self.formatTimestamp() - print("📊 [\(timestamp)] STARTUP STATE: \(state)") - Task { - await errorCollector.addState(state) - } - }, onError: { error in let timestamp = Self.formatTimestamp() print("\n❌ [\(timestamp)] ERROR CALLBACK INVOKED:") @@ -487,9 +446,8 @@ extension ErrorHandlingIntegrationTests { print("\n🚀 Starting connection...") do { - try await conversation.startConversation( - auth: ConversationCredentials.publicAgent(id: testAgentId), - config: config + _ = try await conversation.start( + auth: ConversationCredentials.publicAgent(id: testAgentId) ) print("\n✅ CONNECTION SUCCESSFUL") diff --git a/Tests/ElevenLabsTests/Unit/LiveKitRoomEventDelegateTests.swift b/Tests/ElevenLabsTests/Unit/LiveKitRoomEventDelegateTests.swift index b57c6f38..8499d0d9 100644 --- a/Tests/ElevenLabsTests/Unit/LiveKitRoomEventDelegateTests.swift +++ b/Tests/ElevenLabsTests/Unit/LiveKitRoomEventDelegateTests.swift @@ -7,12 +7,14 @@ final class LiveKitRoomEventDelegateTests: XCTestCase { private func makeDelegate( onData: @escaping @Sendable (Data) -> Void = { _ in }, onRemoteSpeaking: @escaping @Sendable (Bool) -> Void = { _ in }, - onRemoteDisconnect: @escaping @Sendable () async -> Void = {} + onRemoteDisconnect: @escaping @Sendable () async -> Void = {}, + onTracksChanged: @escaping @Sendable () -> Void = {} ) -> RoomDelegate { LiveKitRoomEventDelegate( onData: onData, onRemoteSpeaking: onRemoteSpeaking, - onRemoteDisconnect: onRemoteDisconnect + onRemoteDisconnect: onRemoteDisconnect, + onTracksChanged: onTracksChanged ) } diff --git a/Tests/ElevenLabsWidgetTests/ConversationAudioLevelMonitorTests.swift b/Tests/ElevenLabsWidgetTests/ConversationAudioLevelMonitorTests.swift new file mode 100644 index 00000000..c5d1c717 --- /dev/null +++ b/Tests/ElevenLabsWidgetTests/ConversationAudioLevelMonitorTests.swift @@ -0,0 +1,81 @@ +import AVFoundation +@testable import ElevenLabsWidget +import XCTest + +final class ConversationAudioLevelMonitorTests: XCTestCase { + private let monitor = ConversationAudioLevelMonitor() + + func testStartsSilent() { + XCTAssertEqual(monitor.sample(), 0) + } + + func testToneRaisesLevel() throws { + try monitor.didReceive(tone(amplitude: 0.5)) + XCTAssertGreaterThan(monitor.sample(), 0.5) + } + + func testLouderToneGivesHigherLevel() throws { + try monitor.didReceive(tone(amplitude: 0.05)) + let quiet = monitor.sample() + monitor.reset() + try monitor.didReceive(tone(amplitude: 1)) + XCTAssertGreaterThan(monitor.sample(), quiet) + } + + func testLevelHoldsThePeakBetweenReads() throws { + try monitor.didReceive(tone(amplitude: 1)) + let loud = monitor.sample() + monitor.reset() + try monitor.didReceive(tone(amplitude: 1)) + try monitor.didReceive(tone(amplitude: 0)) + XCTAssertEqual(monitor.sample(), loud) + } + + /// Decay is paced by reads, so a stream that stops delivering still fades. + func testSilenceDecaysWithoutReachingZero() throws { + try monitor.didReceive(tone(amplitude: 1)) + let loud = monitor.sample() + let decayed = monitor.sample() + XCTAssertLessThan(decayed, loud) + XCTAssertGreaterThan(decayed, 0) + XCTAssertLessThan(monitor.sample(), decayed) + } + + func testResetClearsLevel() throws { + try monitor.didReceive(tone(amplitude: 1)) + monitor.reset() + XCTAssertEqual(monitor.sample(), 0) + } + + func testEmptyBufferIsIgnored() throws { + let buffer = try tone(amplitude: 1) + buffer.frameLength = 0 + monitor.didReceive(buffer) + XCTAssertEqual(monitor.sample(), 0) + } + + func testInt16BufferRaisesLevel() throws { + let format = try XCTUnwrap(AVAudioFormat( + commonFormat: .pcmFormatInt16, sampleRate: 48000, channels: 1, interleaved: false + )) + let buffer = try XCTUnwrap(AVAudioPCMBuffer(pcmFormat: format, frameCapacity: 1024)) + buffer.frameLength = 1024 + let samples = try XCTUnwrap(buffer.int16ChannelData)[0] + for frame in 0 ..< 1024 { + samples[frame] = Int16(Float(Int16.max) * 0.5 * sin(2 * .pi * 440 * Float(frame) / 48000)) + } + monitor.didReceive(buffer) + XCTAssertGreaterThan(monitor.sample(), 0.5) + } + + private func tone(amplitude: Float, frames: AVAudioFrameCount = 1024) throws -> AVAudioPCMBuffer { + let format = try XCTUnwrap(AVAudioFormat(standardFormatWithSampleRate: 48000, channels: 1)) + let buffer = try XCTUnwrap(AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frames)) + buffer.frameLength = frames + let samples = try XCTUnwrap(buffer.floatChannelData)[0] + for frame in 0 ..< Int(frames) { + samples[frame] = amplitude * sin(2 * .pi * 440 * Float(frame) / 48000) + } + return buffer + } +}