Skip to content

Commit 41fc406

Browse files
InneiInnei
committed
fix(tts): retry Vertex empty-audio 200 responses
Vertex Gemini TTS sometimes returns HTTP 200 JSON with no audio. Treat that as retryable and read inline audio from any response part.
1 parent 1dc493d commit 41fc406

4 files changed

Lines changed: 144 additions & 19 deletions

File tree

‎apps/core/src/modules/ai/ai-tts/tts-protocol.types.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,15 @@ export interface ITtsProtocolAdapter {
1919
export type TtsProtocolAdapterConfig = TtsRuntimeAdapterConfig
2020

2121
export class TtsProtocolHttpError extends Error {
22+
readonly retryable: boolean
23+
2224
constructor(
2325
readonly status: number,
2426
body: string,
27+
retryable?: boolean,
2528
) {
2629
super(`tts request failed (${status}): ${body.slice(0, 300)}`)
2730
this.name = 'TtsProtocolHttpError'
31+
this.retryable = retryable ?? (status >= 500 || status === 429)
2832
}
2933
}

‎apps/core/src/modules/ai/ai-tts/tts-runtime.adapter.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ export class TtsRuntimeAdapter implements ITtsRuntime {
8484

8585
function isRetryable(error: unknown): boolean {
8686
if (error instanceof TtsProtocolHttpError) {
87-
return error.status >= 500 || error.status === 429
87+
return error.retryable
8888
}
8989
return true
9090
}

‎apps/core/src/modules/ai/ai-tts/vertex-tts-protocol.adapter.ts‎

Lines changed: 36 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -60,39 +60,57 @@ export class VertexTtsProtocolAdapter implements ITtsProtocolAdapter {
6060
await readVertexError(response),
6161
)
6262
}
63-
const payload = (await response.json()) as {
64-
candidates?: Array<{
65-
content?: {
66-
parts?: Array<{
67-
inlineData?: { data?: unknown; mimeType?: unknown }
68-
inline_data?: { data?: unknown; mime_type?: unknown }
69-
}>
70-
}
71-
}>
72-
}
73-
const inline = payload.candidates?.[0]?.content?.parts?.[0]
74-
const data = inline?.inlineData?.data ?? inline?.inline_data?.data
75-
const mimeType =
76-
inline?.inlineData?.mimeType ?? inline?.inline_data?.mime_type
77-
if (typeof data !== 'string' || !data) {
63+
const payload = (await response.json()) as VertexTtsResponse
64+
const audio = extractInlineAudio(payload)
65+
if (!audio) {
7866
throw new TtsProtocolHttpError(
7967
response.status,
8068
'response contained no audio',
69+
true,
8170
)
8271
}
83-
const pcm = Buffer.from(data, 'base64')
72+
const pcm = Buffer.from(audio.data, 'base64')
8473
return {
8574
buffer: wrapPcmAsWav(
8675
pcm,
87-
typeof mimeType === 'string'
88-
? mimeType
76+
typeof audio.mimeType === 'string'
77+
? audio.mimeType
8978
: 'audio/pcm;rate=24000;channels=1',
9079
),
9180
mimeType: 'audio/wav',
9281
}
9382
}
9483
}
9584

85+
type VertexTtsPart = {
86+
inlineData?: { data?: unknown; mimeType?: unknown }
87+
inline_data?: { data?: unknown; mime_type?: unknown }
88+
}
89+
90+
type VertexTtsResponse = {
91+
candidates?: Array<{
92+
content?: {
93+
parts?: VertexTtsPart[]
94+
}
95+
}>
96+
}
97+
98+
function extractInlineAudio(
99+
payload: VertexTtsResponse,
100+
): { data: string; mimeType?: unknown } | undefined {
101+
for (const candidate of payload.candidates ?? []) {
102+
for (const part of candidate.content?.parts ?? []) {
103+
const data = part.inlineData?.data ?? part.inline_data?.data
104+
if (typeof data === 'string' && data) {
105+
return {
106+
data,
107+
mimeType: part.inlineData?.mimeType ?? part.inline_data?.mime_type,
108+
}
109+
}
110+
}
111+
}
112+
}
113+
96114
function normalizeVertexLanguage(language: string): string {
97115
const normalized = language.trim().replace('_', '-')
98116
const defaults: Record<string, string> = {

‎apps/core/test/src/modules/ai/ai-tts/tts-runtime.adapter.spec.ts‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,109 @@ describe('TtsRuntimeAdapter', () => {
9090
})
9191
})
9292

93+
it('retries a Vertex 200 with no audio and succeeds on the next attempt', async () => {
94+
const pcm = Buffer.from([1, 0, 2, 0])
95+
const fetchMock = vi
96+
.fn()
97+
.mockResolvedValueOnce(
98+
new Response(
99+
JSON.stringify({ candidates: [{ content: { parts: [] } }] }),
100+
{
101+
status: 200,
102+
headers: { 'content-type': 'application/json' },
103+
},
104+
),
105+
)
106+
.mockResolvedValueOnce(
107+
new Response(
108+
JSON.stringify({
109+
candidates: [
110+
{
111+
content: {
112+
parts: [
113+
{
114+
inlineData: {
115+
data: pcm.toString('base64'),
116+
mimeType: 'audio/pcm;rate=24000;channels=1',
117+
},
118+
},
119+
],
120+
},
121+
},
122+
],
123+
}),
124+
{ status: 200, headers: { 'content-type': 'application/json' } },
125+
),
126+
)
127+
vi.stubGlobal('fetch', fetchMock)
128+
129+
const adapter = new TtsRuntimeAdapter({
130+
provider: 'vertex',
131+
providerType: AIProviderType.GoogleVertex,
132+
projectId: 'example-project',
133+
apiKey: 'vertex-key',
134+
endpoint:
135+
'https://aiplatform.googleapis.com/v1/projects/example-project/locations/global/endpoints/openapi',
136+
model: 'gemini-3.1-flash-tts-preview',
137+
retryDelayMs: 0,
138+
})
139+
const result = await adapter.generateSpeech({
140+
input: '你好',
141+
language: 'zh',
142+
voice: 'Kore',
143+
speed: 1,
144+
})
145+
146+
expect(fetchMock).toHaveBeenCalledTimes(2)
147+
expect(result.buffer.subarray(44)).toEqual(pcm)
148+
})
149+
150+
it('uses Vertex audio from a later part when the first part has no inline data', async () => {
151+
const pcm = Buffer.from([1, 0, 2, 0])
152+
const fetchMock = vi.fn().mockResolvedValue(
153+
new Response(
154+
JSON.stringify({
155+
candidates: [
156+
{
157+
content: {
158+
parts: [
159+
{ text: '' },
160+
{
161+
inlineData: {
162+
data: pcm.toString('base64'),
163+
mimeType: 'audio/pcm;rate=24000;channels=1',
164+
},
165+
},
166+
],
167+
},
168+
},
169+
],
170+
}),
171+
{ status: 200, headers: { 'content-type': 'application/json' } },
172+
),
173+
)
174+
vi.stubGlobal('fetch', fetchMock)
175+
176+
const adapter = new TtsRuntimeAdapter({
177+
provider: 'vertex',
178+
providerType: AIProviderType.GoogleVertex,
179+
projectId: 'example-project',
180+
apiKey: 'vertex-key',
181+
endpoint:
182+
'https://aiplatform.googleapis.com/v1/projects/example-project/locations/global/endpoints/openapi',
183+
model: 'gemini-3.1-flash-tts-preview',
184+
})
185+
const result = await adapter.generateSpeech({
186+
input: '你好',
187+
language: 'zh',
188+
voice: 'Kore',
189+
speed: 1,
190+
})
191+
192+
expect(fetchMock).toHaveBeenCalledTimes(1)
193+
expect(result.buffer.subarray(44)).toEqual(pcm)
194+
})
195+
93196
it('rejects unsupported provider protocols instead of falling back', () => {
94197
expect(
95198
() =>

0 commit comments

Comments
 (0)