Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 10 additions & 9 deletions Documentation/Usage.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,16 +26,16 @@ This document provides in-depth examples and advanced configuration options for

The `Conversation` object provides reactive `@Published` properties for seamless UI integration.

### Message Handling
### Chat History

Monitor real-time transcriptions for both the agent and the user. The `messages` array is automatically updated.
Observe messages and tool calls in one reconciled history.

```swift
conversation.$messages
conversation.$chatHistory
.receive(on: DispatchQueue.main)
.sink { messages in
// messages are of type [Message]
// Each message has role (.user or .agent) and content
.sink { history in
let messages = history.compactMap(\.message)
let toolCalls = history.compactMap(\.toolCall)
}
.store(in: &cancellables)
```
Expand Down Expand Up @@ -106,10 +106,11 @@ let conversation = try await ElevenLabs.startConversation(
try await conversation.sendMessage("Hi! Tell me about the weather.")

// 3) Receive responses (reactive)
conversation.$messages
conversation.$chatHistory
.receive(on: DispatchQueue.main)
.sink { messages in
guard let last = messages.last, last.role == .agent else { return }
.compactMap { $0.compactMap(\.message).last }
.sink { last in
guard last.role == .agent else { return }
print("Agent:", last.content)
}
.store(in: &cancellables)
Expand Down
7 changes: 4 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -76,15 +76,16 @@ struct ChatView: View {

// Real-time transcriptions
ScrollViewReader { proxy in
let messages = conversation.chatHistory.compactMap(\.message)
ScrollView {
ForEach(conversation.messages) { msg in
ForEach(messages) { msg in
Text("**\(msg.role)**: \(msg.content)")
.padding(8).background(Color.gray.opacity(0.1)).cornerRadius(8)
.id(msg.id)
}
}
.onChange(of: conversation.messages.count) { _ in
proxy.scrollTo(conversation.messages.last?.id)
.onChange(of: messages.count) { _ in
proxy.scrollTo(messages.last?.id)
}
}

Expand Down
120 changes: 120 additions & 0 deletions Sources/ElevenLabs/Internal/Conversation/ChatHistoryReconciler.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
struct ChatHistoryReconciler {
private(set) var items: [any ChatHistoryItem] = []

mutating func appendUserMessage(_ content: String) {
items.append(Message(
role: .user,
content: content,
isFinal: true
))
}

mutating func receive(_ event: TentativeUserTranscriptEvent) {
removeTentativeUserMessages()
items.append(Message(
role: .user,
content: event.transcript,
isFinal: false,
eventId: event.eventId
))
}
Comment thread
cursor[bot] marked this conversation as resolved.

mutating func receive(_ event: UserTranscriptEvent) {
if let index = items.firstIndex(where: {
$0.message?.role == .user
&& $0.message?.eventId == event.eventId
}) {
updateMessage(at: index) {
$0.content = event.transcript
$0.isFinal = true
}
return
}

removeTentativeUserMessages()
items.append(Message(
role: .user,
content: event.transcript,
isFinal: true,
eventId: event.eventId
))
}
Comment thread
cursor[bot] marked this conversation as resolved.

mutating func receive(_ event: AgentChatResponsePartEvent) {
let isEnd = event.type == .stop
if let index = items.firstIndex(where: { $0.id == event.responseId }) {
updateMessage(at: index) {
guard !$0.isFinal else { return }
$0.content += event.text
$0.isFinal = isEnd
}
return
}

items.append(Message(
role: .agent,
content: event.text,
isFinal: isEnd,
eventId: event.eventId,
responseId: event.responseId
))
}

mutating func receive(_ event: AgentResponseEvent) {
upsertAgentMessage(
responseId: event.responseId,
eventId: event.eventId,
content: event.response
)
}

mutating func receive(_ event: AgentResponseCorrectionEvent) {
upsertAgentMessage(
responseId: event.responseId,
eventId: event.eventId,
content: event.correctedAgentResponse
)
}

mutating func receive(_ event: AgentToolResponseEvent) {
guard !items.contains(where: {
$0.toolCall?.toolCallId == event.toolCallId
}) else { return }
items.append(ConversationToolCall(
toolCallId: event.toolCallId,
toolName: event.toolName
))
}

private mutating func upsertAgentMessage(responseId: String, eventId: Int, content: String) {
if let index = items.firstIndex(where: { $0.id == responseId }) {
updateMessage(at: index) {
$0.content = content
$0.isFinal = true
$0.eventId = eventId
}
} else {
items.append(Message(
role: .agent,
content: content,
isFinal: true,
eventId: eventId,
responseId: responseId
))
}
}

private mutating func updateMessage(at index: Int, update: (inout Message) -> Void) {
guard var message = items[index] as? Message else { return }
update(&message)
items[index] = message
}

private mutating func removeTentativeUserMessages() {
items.removeAll {
guard let message = $0.message else { return false }
return message.role == .user
&& !message.isFinal
}
}
}
69 changes: 15 additions & 54 deletions Sources/ElevenLabs/Internal/Conversation/Conversation.swift
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,11 @@ final class Conversation: ObservableObject {
// MARK: - State

@Published var state: ConversationState = .idle
@Published var messages: [Message] = []
@Published var chatHistory: [any ChatHistoryItem] = []
@Published var agentState: AgentState = .listening

private var chatHistoryReconciler = ChatHistoryReconciler()

/// Stream of client tool calls that need to be executed by the app
@Published var pendingToolCalls: [ClientToolCallEvent] = []

Expand Down Expand Up @@ -275,7 +277,7 @@ final class Conversation: ObservableObject {
}
let event = OutgoingEvent.userMessage(UserMessageEvent(text: text))
try await publish(event)
appendMessage(role: .user, content: text)
updateChatHistory { $0.appendUserMessage(text) }
}

/// Mute or unmute the local microphone.
Expand Down Expand Up @@ -432,7 +434,7 @@ final class Conversation: ObservableObject {
}

/// Tear down operational state when an active session ends.
/// Preserves user-visible display state (messages, MCP activity, conversation
/// Preserves user-visible display state (history, MCP activity, conversation
/// metadata) so `ConversationClient` can keep the completed transcript visible.
private func tearDownActiveSession() {
cleanupTransientResources()
Expand Down Expand Up @@ -480,17 +482,17 @@ final class Conversation: ObservableObject {
func handleIncomingEvent(_ event: IncomingEvent) async {
switch event {
case let .userTranscript(e):
insertUserTranscript(content: e.transcript, eventId: e.eventId)
updateChatHistory { $0.receive(e) }
agentStateManager?.processSignal(.userTranscript)
callbacks.onUserTranscript?(e.transcript, e.eventId)

case let .agentResponse(e):
upsertAgentMessage(content: e.response, eventId: e.eventId)
updateChatHistory { $0.receive(e) }
agentStateManager?.processSignal(.agentResponse)
callbacks.onAgentResponse?(e.response, e.eventId)

case let .agentResponseCorrection(correction):
upsertAgentMessage(content: correction.correctedAgentResponse, eventId: correction.eventId)
updateChatHistory { $0.receive(correction) }
callbacks.onAgentResponseCorrection?(
correction.originalAgentResponse,
correction.correctedAgentResponse,
Expand All @@ -504,8 +506,7 @@ final class Conversation: ObservableObject {
)

case let .agentChatResponsePart(e):
let existing = messages.last(where: { $0.role == .agent && $0.eventId == e.eventId })?.content ?? ""
upsertAgentMessage(content: existing + e.text, eventId: e.eventId)
updateChatHistory { $0.receive(e) }

case let .audio(audioEvent):
if let alignment = audioEvent.alignment {
Expand Down Expand Up @@ -534,6 +535,7 @@ final class Conversation: ObservableObject {
callbacks.onVadScore?(vad.vadScore)

case let .agentToolResponse(toolResponse):
updateChatHistory { $0.receive(toolResponse) }
applyStateSignal(.agentToolResponse, fallback: .listening)

if toolResponse.toolName == "end_call" {
Expand All @@ -545,8 +547,8 @@ final class Conversation: ObservableObject {
applyStateSignal(.agentToolRequest, fallback: .thinking)
callbacks.onAgentToolRequest?(toolRequest)

case .tentativeUserTranscript:
break
case let .tentativeUserTranscript(transcript):
updateChatHistory { $0.receive(transcript) }

case let .mcpToolCall(toolCall):
if let index = mcpToolCalls.firstIndex(where: { $0.toolCallId == toolCall.toolCallId }) {
Expand All @@ -564,50 +566,9 @@ final class Conversation: ObservableObject {
}
}

// MARK: - Message Helpers

func appendMessage(role: Message.Role, content: String, eventId: Int? = nil) {
messages.append(
Message(
id: UUID().uuidString,
role: role,
content: content,
timestamp: Date(),
eventId: eventId
)
)
}

/// Inserts the user transcript before the agent message with the same `eventId`
/// if one exists, since the agent's response may be received before the transcript.
private func insertUserTranscript(content: String, eventId: Int) {
let message = Message(
id: UUID().uuidString,
role: .user,
content: content,
timestamp: Date(),
eventId: eventId
)
if let agentIdx = messages.firstIndex(where: { $0.role == .agent && $0.eventId == eventId }) {
messages.insert(message, at: agentIdx)
} else {
messages.append(message)
}
}

private func upsertAgentMessage(content: String, eventId: Int) {
if let idx = messages.lastIndex(where: { $0.role == .agent && $0.eventId == eventId }) {
let existing = messages[idx]
messages[idx] = Message(
id: existing.id,
role: .agent,
content: content,
timestamp: existing.timestamp,
eventId: eventId
)
} else {
appendMessage(role: .agent, content: content, eventId: eventId)
}
private func updateChatHistory(_ update: (inout ChatHistoryReconciler) -> Void) {
update(&chatHistoryReconciler)
chatHistory = chatHistoryReconciler.items
}
}

Expand Down
25 changes: 19 additions & 6 deletions Sources/ElevenLabs/Internal/Utilities/EventParser.swift
Original file line number Diff line number Diff line change
Expand Up @@ -27,21 +27,28 @@ enum EventParser {
case "agent_response":
if let event = json["agent_response_event"] as? [String: Any],
let response = event["agent_response"] as? String,
let eventId = event["event_id"] as? Int
let eventId = event["event_id"] as? Int,
let responseId = event["response_id"] as? String
{
return .agentResponse(AgentResponseEvent(response: response, eventId: eventId))
return .agentResponse(AgentResponseEvent(
response: response,
eventId: eventId,
responseId: responseId
))
}

case "agent_response_correction":
if let event = json["agent_response_correction_event"] as? [String: Any],
let originalResponse = event["original_agent_response"] as? String,
let correctedResponse = event["corrected_agent_response"] as? String,
let eventId = event["event_id"] as? Int
let eventId = event["event_id"] as? Int,
let responseId = event["response_id"] as? String
{
return .agentResponseCorrection(AgentResponseCorrectionEvent(
originalAgentResponse: originalResponse,
correctedAgentResponse: correctedResponse,
eventId: eventId
eventId: eventId,
responseId: responseId
))
}

Expand Down Expand Up @@ -246,12 +253,18 @@ enum EventParser {
case "agent_chat_response_part":
if let event = json["text_response_part"] as? [String: Any],
let text = event["text"] as? String,
let eventId = event["event_id"] as? Int
let eventId = event["event_id"] as? Int,
let responseId = event["response_id"] as? String
{
let partTypeStr = event["type"] as? String ?? "delta"
let partType = AgentChatResponsePartType(rawValue: partTypeStr) ?? .delta
return .agentChatResponsePart(
AgentChatResponsePartEvent(text: text, type: partType, eventId: eventId)
AgentChatResponsePartEvent(
text: text,
type: partType,
eventId: eventId,
responseId: responseId
)
)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,13 +45,15 @@ public struct TentativeUserTranscriptEvent: Sendable {
public struct AgentResponseEvent: Sendable {
public let response: String
public let eventId: Int
public let responseId: String
}

/// Agent's response correction
public struct AgentResponseCorrectionEvent: Sendable {
public let originalAgentResponse: String
public let correctedAgentResponse: String
public let eventId: Int
public let responseId: String
}

/// Agent response metadata
Expand All @@ -64,6 +66,7 @@ public struct AgentChatResponsePartEvent: Sendable {
public let text: String
public let type: AgentChatResponsePartType
public let eventId: Int
public let responseId: String
}

/// Audio alignment data showing character-level timing information
Expand Down
Loading
Loading