Skip to content

Commit 3258c52

Browse files
heavygeecursoragentHAPI
authored
fix(web): embed agent text in voice ready event for readback (#682)
* fix(web): embed agent text in voice ready event for readback Voice onReady now extracts the last speakable assistant message and embeds it in the ready inject so ConvAI can summarize without the user re-prompting. Also formats Codex/Cursor stream-json messages for live context updates and session history. Co-authored-by: Cursor <cursoragent@cursor.com> * test(web): use domain-neutral voice formatter fixtures Replace jellybot/subtitle dogfood strings in tests with generic examples. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(web): guard extractSpeakableFromContent for non-arrays in formatMessage extractSpeakableFromContent also handles content arrays (joins text items), so calling it unconditionally before the existing array loop caused mixed text+tool_use payloads to return early without formatting the tool_use item. Guard with !isContentArray so the loop handles arrays as before. Adds regression test: mixed text+tool_use array must produce both the text and the tool-call line (was red before this fix). via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> * fix(web): narrow extractSpeakableFromContent to codex type only The helper matched any object with a string type and a data property, so sendSessionEvent({ type: 'message', message }) events (which arrive as { type: 'event', data: { type: 'message', message } }) were falsely formatted as speakable assistant text and could be selected as the ready readback. Narrow the Codex path to content.type === 'codex' as the comment already states. Adds regression test: session status event must return null from formatMessage. via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> --------- Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: HAPI <noreply@hapi.run>
1 parent 0fa21a1 commit 3258c52

4 files changed

Lines changed: 318 additions & 12 deletions

File tree

‎web/src/realtime/hooks/contextFormatters.test.ts‎

Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
import { describe, expect, it } from 'vitest'
2+
import type { DecryptedMessage } from '@/types/api'
3+
import { extractLastAssistantSpeakable, formatMessage, formatNewMessages, formatReadyEvent } from './contextFormatters'
4+
5+
function msg(partial: Pick<DecryptedMessage, 'id' | 'seq' | 'content'>): DecryptedMessage {
6+
return {
7+
id: partial.id,
8+
seq: partial.seq,
9+
localId: null,
10+
content: partial.content,
11+
createdAt: 0,
12+
sessionId: 'session-1'
13+
} as DecryptedMessage
14+
}
15+
16+
describe('extractLastAssistantSpeakable', () => {
17+
it('returns null for empty history', () => {
18+
expect(extractLastAssistantSpeakable([])).toBeNull()
19+
})
20+
21+
it('returns the latest assistant plain string', () => {
22+
const messages = [
23+
msg({ id: '1', seq: 1, content: { role: 'user', content: 'hello' } }),
24+
msg({ id: '2', seq: 2, content: { role: 'assistant', content: 'first reply' } }),
25+
msg({ id: '3', seq: 3, content: { role: 'assistant', content: ' latest reply ' } })
26+
]
27+
expect(extractLastAssistantSpeakable(messages)).toBe('latest reply')
28+
})
29+
30+
it('skips trailing user messages and reads earlier assistant text', () => {
31+
const messages = [
32+
msg({ id: '1', seq: 1, content: { role: 'assistant', content: 'done with the refactor' } }),
33+
msg({ id: '2', seq: 2, content: { role: 'user', content: 'thanks' } })
34+
]
35+
expect(extractLastAssistantSpeakable(messages)).toBe('done with the refactor')
36+
})
37+
38+
it('extracts text blocks from assistant content arrays', () => {
39+
const messages = [
40+
msg({
41+
id: '1',
42+
seq: 1,
43+
content: {
44+
role: 'assistant',
45+
content: [
46+
{ type: 'text', text: 'Part one.' },
47+
{ type: 'text', text: 'Part two.' }
48+
]
49+
}
50+
})
51+
]
52+
expect(extractLastAssistantSpeakable(messages)).toBe('Part one.\n\nPart two.')
53+
})
54+
55+
it('extracts codex stream-json assistant messages', () => {
56+
const messages = [
57+
msg({
58+
id: '1',
59+
seq: 1,
60+
content: {
61+
role: 'agent',
62+
content: {
63+
type: 'codex',
64+
data: {
65+
type: 'message',
66+
message: 'Indexed 5,018 items in the search database.'
67+
}
68+
}
69+
}
70+
}),
71+
msg({
72+
id: '2',
73+
seq: 2,
74+
content: {
75+
role: 'agent',
76+
content: {
77+
type: 'codex',
78+
data: { type: 'ready' }
79+
}
80+
}
81+
})
82+
]
83+
expect(extractLastAssistantSpeakable(messages)).toBe('Indexed 5,018 items in the search database.')
84+
})
85+
86+
it('unwraps codex-style output envelopes', () => {
87+
const messages = [
88+
msg({
89+
id: '1',
90+
seq: 1,
91+
content: {
92+
type: 'output',
93+
data: {
94+
type: 'assistant',
95+
message: { content: 'Codex finished the refactor.' }
96+
}
97+
}
98+
})
99+
]
100+
expect(extractLastAssistantSpeakable(messages)).toBe('Codex finished the refactor.')
101+
})
102+
})
103+
104+
describe('formatReadyEvent', () => {
105+
const sessionId = '9d04335d-2b90-4941-98a7-eb414823f0e0'
106+
107+
it('embeds assistant text when provided', () => {
108+
const text = 'Added full-text search to the API module.'
109+
const event = formatReadyEvent(sessionId, text)
110+
expect(event).toContain('coding agent finished working')
111+
expect(event).toContain(`<text>${text}</text>`)
112+
expect(event).not.toContain('Claude Code')
113+
})
114+
115+
it('falls back when assistant text is missing', () => {
116+
const event = formatReadyEvent(sessionId, null)
117+
expect(event).toContain('Use the latest agent message already present in context')
118+
expect(event).not.toContain('Claude Code')
119+
})
120+
121+
it('treats blank assistant text as missing', () => {
122+
const event = formatReadyEvent(sessionId, ' ')
123+
expect(event).toContain('Use the latest agent message already present in context')
124+
})
125+
})
126+
127+
describe('formatMessage', () => {
128+
it('formats codex stream-json assistant messages for voice context', () => {
129+
const formatted = formatMessage(msg({
130+
id: '1',
131+
seq: 1,
132+
content: {
133+
role: 'agent',
134+
content: {
135+
type: 'codex',
136+
data: {
137+
type: 'message',
138+
message: 'Indexed 5,018 items in the search database.'
139+
}
140+
}
141+
}
142+
}))
143+
144+
expect(formatted).toContain('Claude Code:')
145+
expect(formatted).toContain('<text>Indexed 5,018 items in the search database.</text>')
146+
})
147+
148+
it('ignores codex ready and tool-call payloads', () => {
149+
expect(formatMessage(msg({
150+
id: '1',
151+
seq: 1,
152+
content: {
153+
role: 'agent',
154+
content: {
155+
type: 'codex',
156+
data: { type: 'ready' }
157+
}
158+
}
159+
}))).toBeNull()
160+
})
161+
162+
it('does not treat session status events as speakable assistant text', () => {
163+
expect(formatMessage(msg({
164+
id: '1',
165+
seq: 1,
166+
content: {
167+
role: 'agent',
168+
content: {
169+
id: 'some-uuid',
170+
type: 'event',
171+
data: { type: 'message', message: 'Aborting task.' }
172+
}
173+
}
174+
}))).toBeNull()
175+
})
176+
177+
it('preserves tool-call context for mixed text+tool_use content array', () => {
178+
const formatted = formatMessage(msg({
179+
id: '1',
180+
seq: 1,
181+
content: {
182+
role: 'assistant',
183+
content: [
184+
{ type: 'text', text: 'Here is the result.' },
185+
{ type: 'tool_use', name: 'Bash', input: { command: 'ls' } }
186+
]
187+
}
188+
}))
189+
190+
expect(formatted).toContain('Here is the result.')
191+
expect(formatted).toContain('Claude Code is using Bash')
192+
})
193+
})
194+
195+
describe('formatNewMessages', () => {
196+
it('includes codex assistant replies in contextual updates', () => {
197+
const update = formatNewMessages('session-1', [
198+
msg({
199+
id: '1',
200+
seq: 1,
201+
content: {
202+
role: 'agent',
203+
content: {
204+
type: 'codex',
205+
data: {
206+
type: 'message',
207+
message: 'Local database file size is 2.43 GiB.'
208+
}
209+
}
210+
}
211+
})
212+
])
213+
214+
expect(update).toContain('New messages in session: session-1')
215+
expect(update).toContain('Local database file size is 2.43 GiB.')
216+
})
217+
})

‎web/src/realtime/hooks/contextFormatters.ts‎

Lines changed: 93 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -92,21 +92,26 @@ export function formatPermissionRequest(
9292
* Format a single message for voice context
9393
*/
9494
export function formatMessage(message: DecryptedMessage): string | null {
95-
const lines: string[] = []
9695
const { role, content: wrappedContent } = unwrapRoleWrappedContent(message)
9796
const { roleOverride, content } = unwrapOutputContent(wrappedContent)
9897
const normalizedRole = roleOverride ?? role
9998

99+
if (isNonSpeakableAgentPayload(wrappedContent) || isNonSpeakableAgentPayload(content)) {
100+
return null
101+
}
102+
103+
const speakable = !isContentArray(content) ? extractSpeakableFromContent(content) : null
104+
if (speakable) {
105+
const roleForFormat = normalizedRole === 'user' ? 'user' : 'assistant'
106+
return formatPlainText(roleForFormat, speakable)
107+
}
108+
100109
if (!isContentArray(content)) {
101-
if (typeof content === 'string') {
102-
return formatPlainText(normalizedRole, content)
103-
}
104-
if (isObject(content) && content.type === 'text' && typeof content.text === 'string') {
105-
return formatPlainText(normalizedRole, content.text)
106-
}
107110
return null
108111
}
109112

113+
const lines: string[] = []
114+
110115
// Determine message type by checking for tool_use (assistant) vs user content
111116
const hasToolUse = content.some(item => item.type === 'tool_use')
112117
const isAssistant = normalizedRole === 'assistant'
@@ -134,6 +139,81 @@ export function formatMessage(message: DecryptedMessage): string | null {
134139
return lines.join('\n\n')
135140
}
136141

142+
function extractSpeakableFromContent(content: unknown): string | null {
143+
if (typeof content === 'string' && content.trim()) {
144+
return content.trim()
145+
}
146+
147+
if (isObject(content) && content.type === 'text' && typeof content.text === 'string' && content.text.trim()) {
148+
return content.text.trim()
149+
}
150+
151+
// Codex / stream-json agent messages: { type: 'codex', data: { type: 'message', message: '...' } }
152+
if (isObject(content) && content.type === 'codex' && isObject(content.data)) {
153+
const data = content.data
154+
if (data.type === 'message' && typeof data.message === 'string' && data.message.trim()) {
155+
return data.message.trim()
156+
}
157+
}
158+
159+
if (!isContentArray(content)) {
160+
return null
161+
}
162+
163+
const textParts = content
164+
.filter((item) => item.type === 'text' && item.text)
165+
.map((item) => item.text!.trim())
166+
.filter(Boolean)
167+
168+
if (textParts.length > 0) {
169+
return textParts.join('\n\n')
170+
}
171+
172+
return null
173+
}
174+
175+
function isNonSpeakableAgentPayload(content: unknown): boolean {
176+
if (!isObject(content) || typeof content.type !== 'string') {
177+
return false
178+
}
179+
180+
if (content.type === 'codex' && isObject(content.data)) {
181+
const eventType = content.data.type
182+
return eventType === 'ready'
183+
|| eventType === 'tool-call'
184+
|| eventType === 'tool-call-result'
185+
|| eventType === 'event'
186+
}
187+
188+
return false
189+
}
190+
191+
export function extractLastAssistantSpeakable(messages: DecryptedMessage[]): string | null {
192+
const sorted = [...messages].sort((a, b) => (a.seq ?? 0) - (b.seq ?? 0))
193+
194+
for (let i = sorted.length - 1; i >= 0; i -= 1) {
195+
const message = sorted[i]
196+
const { role, content: wrappedContent } = unwrapRoleWrappedContent(message)
197+
const { roleOverride, content } = unwrapOutputContent(wrappedContent)
198+
const normalizedRole = roleOverride ?? role
199+
200+
if (normalizedRole === 'user') {
201+
continue
202+
}
203+
204+
if (isNonSpeakableAgentPayload(wrappedContent) || isNonSpeakableAgentPayload(content)) {
205+
continue
206+
}
207+
208+
const speakable = extractSpeakableFromContent(content)
209+
if (speakable) {
210+
return speakable
211+
}
212+
}
213+
214+
return null
215+
}
216+
137217
export function formatNewSingleMessage(sessionId: string, message: DecryptedMessage): string | null {
138218
const formatted = formatMessage(message)
139219
if (!formatted) {
@@ -199,6 +279,10 @@ export function formatSessionFocus(sessionId: string, _metadata?: SessionMetadat
199279
return `Session became focused: ${sessionId}`
200280
}
201281

202-
export function formatReadyEvent(sessionId: string): string {
203-
return `Claude Code done working in session: ${sessionId}. The previous message(s) are the summary of the work done. Report this to the human immediately.`
282+
export function formatReadyEvent(sessionId: string, lastAssistantText?: string | null): string {
283+
const trimmed = lastAssistantText?.trim()
284+
if (trimmed) {
285+
return `The coding agent finished working in session: ${sessionId}. Summarize this for the human immediately:\n<text>${trimmed}</text>`
286+
}
287+
return `The coding agent finished working in session: ${sessionId}. Use the latest agent message already present in context and summarize it for the human immediately.`
204288
}

‎web/src/realtime/hooks/voiceHooks.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@ import {
66
formatSessionFocus,
77
formatSessionFull,
88
formatSessionOffline,
9-
formatSessionOnline
9+
formatSessionOnline,
10+
extractLastAssistantSpeakable
1011
} from './contextFormatters'
1112
import { VOICE_CONFIG } from '../voiceConfig'
1213
import type { DecryptedMessage, Session } from '@/types/api'
@@ -68,6 +69,7 @@ function reportSession(sessionId: string) {
6869
reportContextualUpdate(contextUpdate)
6970
}
7071

72+
7173
export const voiceHooks = {
7274
/**
7375
* Called when a session comes online/connects
@@ -147,7 +149,9 @@ export const voiceHooks = {
147149
if (VOICE_CONFIG.DISABLE_READY_EVENTS) return
148150

149151
reportSession(sessionId)
150-
reportTextUpdate(formatReadyEvent(sessionId))
152+
const messages = messagesGetter?.(sessionId) ?? []
153+
const lastAssistantText = extractLastAssistantSpeakable(messages)
154+
reportTextUpdate(formatReadyEvent(sessionId, lastAssistantText))
151155
},
152156

153157
/**

‎web/src/realtime/index.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,8 @@ export {
3232
formatSessionOffline,
3333
formatSessionFocus,
3434
formatPermissionRequest,
35-
formatReadyEvent
35+
formatReadyEvent,
36+
extractLastAssistantSpeakable
3637
} from './hooks/contextFormatters'
3738

3839
// Config

0 commit comments

Comments
 (0)