Skip to content

Commit 5e08f6c

Browse files
heavygeeHAPI
andcommitted
fix(voice): replace unsupported conversation.item.create with session.update for Qwen text
Qwen's realtime API only supports conversation.item.create for function_call_output. Sending it with type:'message' for greetings/context was invalid and could fail before the user spoke. sendTextMessage and sendContextualUpdate now update session instructions via session.update (accumulating context into the system prompt) and trigger response.create only when a spoken reply is needed — matching Qwen's supported client event surface. via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run>
1 parent 7320f25 commit 5e08f6c

1 file changed

Lines changed: 50 additions & 34 deletions

File tree

web/src/realtime/QwenVoiceSession.tsx

Lines changed: 50 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -77,13 +77,40 @@ function sendEvent(type: string, payload?: Record<string, unknown>): void {
7777
}))
7878
}
7979

80+
interface QwenSessionConfig {
81+
modalities: ['text', 'audio']
82+
voice: string
83+
input_audio_format: 'pcm16'
84+
output_audio_format: 'pcm24'
85+
instructions: string
86+
temperature: number
87+
turn_detection: {
88+
type: 'server_vad'
89+
threshold: number
90+
silence_duration_ms: number
91+
prefix_padding_ms: number
92+
}
93+
tools: Array<unknown>
94+
tool_choice: 'auto'
95+
}
96+
8097
class QwenVoiceSessionImpl implements VoiceSession {
8198
private api: ApiClient
99+
private currentSessionConfig: QwenSessionConfig | null = null
82100

83101
constructor(api: ApiClient) {
84102
this.api = api
85103
}
86104

105+
private updateInstructions(update: string): void {
106+
if (!this.currentSessionConfig) return
107+
this.currentSessionConfig = {
108+
...this.currentSessionConfig,
109+
instructions: `${this.currentSessionConfig.instructions}\n\n${update}`
110+
}
111+
sendEvent('session.update', { session: this.currentSessionConfig })
112+
}
113+
87114
async startSession(config: VoiceSessionConfig): Promise<void> {
88115
cleanup()
89116
state.statusCallback?.('connecting')
@@ -167,24 +194,23 @@ class QwenVoiceSessionImpl implements VoiceSession {
167194
? `${basePrompt}\n\n[Current Context]\n${config.initialContext}`
168195
: basePrompt
169196

170-
sendEvent('session.update', {
171-
session: {
172-
modalities: ['text', 'audio'],
173-
voice: QWEN_REALTIME_VOICE,
174-
input_audio_format: 'pcm16',
175-
output_audio_format: 'pcm24',
176-
instructions,
177-
temperature: 0.7,
178-
turn_detection: {
179-
type: 'server_vad',
180-
threshold: 0.5,
181-
silence_duration_ms: 800,
182-
prefix_padding_ms: 300
183-
},
184-
tools,
185-
tool_choice: 'auto'
186-
}
187-
})
197+
this.currentSessionConfig = {
198+
modalities: ['text', 'audio'],
199+
voice: QWEN_REALTIME_VOICE,
200+
input_audio_format: 'pcm16',
201+
output_audio_format: 'pcm24',
202+
instructions,
203+
temperature: 0.7,
204+
turn_detection: {
205+
type: 'server_vad',
206+
threshold: 0.5,
207+
silence_duration_ms: 800,
208+
prefix_padding_ms: 300
209+
},
210+
tools,
211+
tool_choice: 'auto'
212+
}
213+
sendEvent('session.update', { session: this.currentSessionConfig })
188214
return
189215
}
190216

@@ -324,32 +350,22 @@ class QwenVoiceSessionImpl implements VoiceSession {
324350
}
325351

326352
async endSession(): Promise<void> {
353+
this.currentSessionConfig = null
327354
cleanup()
328355
resetRealtimeSessionState()
329356
state.statusCallback?.('disconnected')
330357
}
331358

332359
sendTextMessage(message: string): void {
333-
// Send text as a user message via conversation.item.create
334-
sendEvent('conversation.item.create', {
335-
item: {
336-
type: 'message',
337-
role: 'user',
338-
content: [{ type: 'input_text', text: message }]
339-
}
340-
})
360+
// Qwen only supports conversation.item.create for function_call_output.
361+
// Inject text as an instruction update then trigger a response.
362+
this.updateInstructions(message)
341363
sendEvent('response.create')
342364
}
343365

344366
sendContextualUpdate(update: string): void {
345-
// Send context as a system-like user message
346-
sendEvent('conversation.item.create', {
347-
item: {
348-
type: 'message',
349-
role: 'user',
350-
content: [{ type: 'input_text', text: `[System Context Update] ${update}` }]
351-
}
352-
})
367+
// Append context silently — no response.create, so model doesn't speak yet.
368+
this.updateInstructions(`[System Context Update] ${update}`)
353369
}
354370
}
355371

0 commit comments

Comments
 (0)