Skip to content

Commit 56212d9

Browse files
tylergibbs1claude
tylergibbs1
and
claude
authored
Add configurable 429 retry with Azure retry-after-ms support (#2)
* Add configurable 429 retry with Azure retry-after-ms support Extract shared computeRetryDelay helper that prefers Azure's retry-after-ms header (ms precision) over standard retry-after (seconds), falling back to exponential backoff with jitter. Both AzureResponsesModel and AzureChatCompletionsModel now accept maxRetries config (default 3) and log retries via console.warn. Chat Completions model also gains network error retry parity with Responses model. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * Address review feedback: cap server delays, abortable sleep, fix SSE retry budget - Cap retry-after-ms and retry-after at 30s to prevent indefinite stalls - Add abortableSleep() so backoff respects AbortSignal cancellation - Fix SSE retry budget: use fixed maxSseRetries=3 independent of doFetch retries to prevent quadratic retry multiplication - Remove unused lastError variable (dead code after refactor) - Add negative value test case for retry-after-ms - Add abortableSleep tests (immediate resolve, early abort) - Annotate unreachable throw as TypeScript control-flow requirement Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * Fix lint warnings and test timeouts from retry changes - Replace non-null assertions (this.apiKey!) with `as string` cast in both models — constructor already validates the invariant - Update test mocks to use retry-after-ms: "1" instead of retry-after: "0" since computeRetryDelay correctly rejects zero values and falls through to exponential backoff, causing test timeouts Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent f6532f4 commit 56212d9

5 files changed

Lines changed: 223 additions & 50 deletions

File tree

‎packages/stratus-sdk/src/azure/chat-completions-model.ts‎

Lines changed: 45 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import type {
1010
} from "../core/model";
1111
import type { ChatMessage, HostedToolDefinition, ToolCall, ToolDefinition } from "../core/types";
1212
import { resolveChatCompletionsUrl } from "./endpoint";
13+
import { abortableSleep, computeRetryDelay } from "./retry";
1314
import { parseSSE } from "./sse-parser";
1415

1516
export interface AzureChatCompletionsModelConfig {
@@ -18,6 +19,8 @@ export interface AzureChatCompletionsModelConfig {
1819
azureAdTokenProvider?: () => Promise<string>;
1920
deployment: string;
2021
apiVersion?: string;
22+
/** Maximum number of retries on 429 / network errors (default 3). */
23+
maxRetries?: number;
2124
}
2225

2326
const DEFAULT_API_VERSION = "2025-03-01-preview";
@@ -27,6 +30,7 @@ export class AzureChatCompletionsModel implements Model {
2730
private readonly apiKey?: string;
2831
private readonly tokenProvider?: () => Promise<string>;
2932
private readonly deployment: string;
33+
private readonly maxRetries: number;
3034

3135
constructor(config: AzureChatCompletionsModelConfig) {
3236
if (config.apiKey && config.azureAdTokenProvider) {
@@ -38,6 +42,7 @@ export class AzureChatCompletionsModel implements Model {
3842
this.apiKey = config.apiKey;
3943
this.tokenProvider = config.azureAdTokenProvider;
4044
this.deployment = config.deployment;
45+
this.maxRetries = config.maxRetries ?? 3;
4146
this.url = resolveChatCompletionsUrl(
4247
config.endpoint,
4348
config.deployment,
@@ -50,7 +55,8 @@ export class AzureChatCompletionsModel implements Model {
5055
const token = await this.tokenProvider();
5156
return { Authorization: `Bearer ${token}` };
5257
}
53-
return { "api-key": this.apiKey! };
58+
// Constructor validates exactly one of apiKey/tokenProvider is set.
59+
return { "api-key": this.apiKey as string };
5460
}
5561

5662
async getResponse(request: ModelRequest, options?: ModelRequestOptions): Promise<ModelResponse> {
@@ -205,25 +211,42 @@ export class AzureChatCompletionsModel implements Model {
205211
}
206212

207213
private async doFetch(body: Record<string, unknown>, signal?: AbortSignal): Promise<Response> {
208-
const maxRetries = 3;
209-
for (let attempt = 0; attempt <= maxRetries; attempt++) {
210-
const authHeaders = await this.getAuthHeaders();
211-
const response = await fetch(this.url, {
212-
method: "POST",
213-
headers: {
214-
"Content-Type": "application/json",
215-
...authHeaders,
216-
},
217-
body: JSON.stringify(body),
218-
signal,
219-
});
220-
221-
if (response.status === 429 && attempt < maxRetries) {
222-
const retryAfter = response.headers.get("retry-after");
223-
const waitMs = retryAfter
224-
? Number.parseInt(retryAfter, 10) * 1000
225-
: Math.min(1000 * 2 ** attempt, 30000);
226-
await new Promise((r) => setTimeout(r, waitMs));
214+
for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
215+
let response: Response;
216+
try {
217+
const authHeaders = await this.getAuthHeaders();
218+
response = await fetch(this.url, {
219+
method: "POST",
220+
headers: {
221+
"Content-Type": "application/json",
222+
...authHeaders,
223+
},
224+
body: JSON.stringify(body),
225+
signal,
226+
});
227+
} catch (fetchErr) {
228+
if (signal?.aborted) throw fetchErr;
229+
if (attempt < this.maxRetries) {
230+
const waitMs = Math.min(1000 * 2 ** attempt + Math.random() * 1000, 30000);
231+
await abortableSleep(waitMs, signal);
232+
if (signal?.aborted) throw fetchErr;
233+
continue;
234+
}
235+
throw new ModelError(
236+
`Azure API network error after ${this.maxRetries + 1} attempts: ${fetchErr instanceof Error ? fetchErr.message : String(fetchErr)}`,
237+
{ cause: fetchErr },
238+
);
239+
}
240+
241+
if (response.status === 429 && attempt < this.maxRetries) {
242+
const waitMs = computeRetryDelay(response.headers, attempt);
243+
console.warn(
244+
`[AzureChatCompletionsModel] 429 rate limited, retrying in ${(waitMs / 1000).toFixed(1)}s (attempt ${attempt + 1}/${this.maxRetries})`,
245+
);
246+
await abortableSleep(waitMs, signal);
247+
if (signal?.aborted) {
248+
throw new ModelError("Azure API request aborted during retry backoff", { status: 429 });
249+
}
227250
continue;
228251
}
229252

@@ -234,6 +257,8 @@ export class AzureChatCompletionsModel implements Model {
234257
return response;
235258
}
236259

260+
// Unreachable: the final iteration always returns or throws above.
261+
// Kept for TypeScript's control-flow analysis.
237262
throw new ModelError("Max retries exceeded for Azure API request");
238263
}
239264

‎packages/stratus-sdk/src/azure/responses-model.ts‎

Lines changed: 37 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import type {
1818
ToolDefinition,
1919
} from "../core/types";
2020
import { resolveResponsesUrl } from "./endpoint";
21+
import { abortableSleep, computeRetryDelay } from "./retry";
2122
import { parseSSE } from "./sse-parser";
2223

2324
export interface AzureResponsesModelConfig {
@@ -27,6 +28,8 @@ export interface AzureResponsesModelConfig {
2728
deployment: string;
2829
apiVersion?: string;
2930
store?: boolean;
31+
/** Maximum number of retries on 429 / network errors (default 3). */
32+
maxRetries?: number;
3033
}
3134

3235
const DEFAULT_API_VERSION = "2025-04-01-preview";
@@ -71,6 +74,7 @@ export class AzureResponsesModel implements Model {
7174
private readonly tokenProvider?: () => Promise<string>;
7275
private readonly deployment: string;
7376
private readonly store: boolean;
77+
private readonly maxRetries: number;
7478

7579
constructor(config: AzureResponsesModelConfig) {
7680
if (config.apiKey && config.azureAdTokenProvider) {
@@ -83,6 +87,7 @@ export class AzureResponsesModel implements Model {
8387
this.tokenProvider = config.azureAdTokenProvider;
8488
this.deployment = config.deployment;
8589
this.store = config.store ?? false;
90+
this.maxRetries = config.maxRetries ?? 3;
8691
this.url = resolveResponsesUrl(config.endpoint, config.apiVersion ?? DEFAULT_API_VERSION);
8792
}
8893

@@ -91,7 +96,8 @@ export class AzureResponsesModel implements Model {
9196
const token = await this.tokenProvider();
9297
return { Authorization: `Bearer ${token}` };
9398
}
94-
return { "api-key": this.apiKey! };
99+
// Constructor validates exactly one of apiKey/tokenProvider is set.
100+
return { "api-key": this.apiKey as string };
95101
}
96102

97103
async getResponse(request: ModelRequest, options?: ModelRequestOptions): Promise<ModelResponse> {
@@ -106,8 +112,9 @@ export class AzureResponsesModel implements Model {
106112
options?: ModelRequestOptions,
107113
): AsyncGenerator<StreamEvent> {
108114
const body = this.buildRequestBody(request, true);
115+
// SSE-level retries are separate from doFetch's HTTP-level retries.
116+
// Use a fixed budget of 3 to avoid quadratic retry multiplication.
109117
const maxSseRetries = 3;
110-
111118
for (let sseAttempt = 0; sseAttempt <= maxSseRetries; sseAttempt++) {
112119
const response = await this.doFetch(body, options?.signal);
113120

@@ -256,11 +263,16 @@ export class AzureResponsesModel implements Model {
256263

257264
// SSE rate limited before any events were yielded — retry with backoff
258265
if (sseRateLimited) {
259-
const retryAfterHeader = response.headers.get("retry-after");
260-
const waitMs = retryAfterHeader
261-
? Number.parseInt(retryAfterHeader, 10) * 1000
262-
: Math.min(10000 * 2 ** sseAttempt, 60000);
263-
await new Promise((r) => setTimeout(r, waitMs));
266+
const waitMs = computeRetryDelay(response.headers, sseAttempt);
267+
console.warn(
268+
`[AzureResponsesModel] 429 rate limited (SSE), retrying in ${(waitMs / 1000).toFixed(1)}s (attempt ${sseAttempt + 1}/${maxSseRetries})`,
269+
);
270+
await abortableSleep(waitMs, options?.signal);
271+
if (options?.signal?.aborted) {
272+
throw new ModelError("Azure API request aborted during SSE retry backoff", {
273+
status: 429,
274+
});
275+
}
264276
continue;
265277
}
266278

@@ -358,9 +370,7 @@ export class AzureResponsesModel implements Model {
358370
}
359371

360372
private async doFetch(body: Record<string, unknown>, signal?: AbortSignal): Promise<Response> {
361-
const maxRetries = 3;
362-
let lastError: unknown;
363-
for (let attempt = 0; attempt <= maxRetries; attempt++) {
373+
for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
364374
let response: Response;
365375
try {
366376
const authHeaders = await this.getAuthHeaders();
@@ -377,24 +387,27 @@ export class AzureResponsesModel implements Model {
377387
// Network errors (timeout, connection reset, DNS failure)
378388
// are retryable unless the caller aborted.
379389
if (signal?.aborted) throw fetchErr;
380-
lastError = fetchErr;
381-
if (attempt < maxRetries) {
382-
const waitMs = Math.min(5000 * 2 ** attempt, 30000);
383-
await new Promise((r) => setTimeout(r, waitMs));
390+
if (attempt < this.maxRetries) {
391+
const waitMs = Math.min(1000 * 2 ** attempt + Math.random() * 1000, 30000);
392+
await abortableSleep(waitMs, signal);
393+
if (signal?.aborted) throw fetchErr;
384394
continue;
385395
}
386396
throw new ModelError(
387-
`Azure API network error after ${maxRetries + 1} attempts: ${fetchErr instanceof Error ? fetchErr.message : String(fetchErr)}`,
397+
`Azure API network error after ${this.maxRetries + 1} attempts: ${fetchErr instanceof Error ? fetchErr.message : String(fetchErr)}`,
388398
{ cause: fetchErr },
389399
);
390400
}
391401

392-
if (response.status === 429 && attempt < maxRetries) {
393-
const retryAfter = response.headers.get("retry-after");
394-
const waitMs = retryAfter
395-
? Number.parseInt(retryAfter, 10) * 1000
396-
: Math.min(5000 * 2 ** attempt, 30000);
397-
await new Promise((r) => setTimeout(r, waitMs));
402+
if (response.status === 429 && attempt < this.maxRetries) {
403+
const waitMs = computeRetryDelay(response.headers, attempt);
404+
console.warn(
405+
`[AzureResponsesModel] 429 rate limited, retrying in ${(waitMs / 1000).toFixed(1)}s (attempt ${attempt + 1}/${this.maxRetries})`,
406+
);
407+
await abortableSleep(waitMs, signal);
408+
if (signal?.aborted) {
409+
throw new ModelError("Azure API request aborted during retry backoff", { status: 429 });
410+
}
398411
continue;
399412
}
400413

@@ -405,9 +418,9 @@ export class AzureResponsesModel implements Model {
405418
return response;
406419
}
407420

408-
throw new ModelError(
409-
`Max retries exceeded for Azure API request${lastError instanceof Error ? `: ${lastError.message}` : ""}`,
410-
);
421+
// Unreachable: the final iteration always returns or throws above.
422+
// Kept for TypeScript's control-flow analysis.
423+
throw new ModelError("Max retries exceeded for Azure API request");
411424
}
412425

413426
private async handleErrorResponse(response: Response): Promise<never> {

‎packages/stratus-sdk/src/azure/retry.ts‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
const MAX_RETRY_DELAY_MS = 30_000;
2+
3+
/**
4+
* Computes how long to wait before retrying a 429'd request.
5+
*
6+
* Prefers the more precise `retry-after-ms` header (milliseconds) that Azure
7+
* returns alongside the standard `retry-after` (seconds). Falls back to
8+
* exponential backoff with jitter when neither header is present.
9+
*
10+
* All values are capped at {@link MAX_RETRY_DELAY_MS} to prevent a misbehaving
11+
* server from stalling the caller indefinitely.
12+
*/
13+
export function computeRetryDelay(headers: Headers, attempt: number): number {
14+
// Azure-specific: millisecond precision
15+
const retryAfterMs = headers.get("retry-after-ms");
16+
if (retryAfterMs) {
17+
const ms = Number.parseInt(retryAfterMs, 10);
18+
if (!Number.isNaN(ms) && ms > 0) return Math.min(ms, MAX_RETRY_DELAY_MS);
19+
}
20+
21+
// Standard header: seconds
22+
const retryAfter = headers.get("retry-after");
23+
if (retryAfter) {
24+
const seconds = Number.parseInt(retryAfter, 10);
25+
if (!Number.isNaN(seconds) && seconds > 0) return Math.min(seconds * 1000, MAX_RETRY_DELAY_MS);
26+
}
27+
28+
// Exponential backoff with jitter, capped at 30s
29+
return Math.min(1000 * 2 ** attempt + Math.random() * 1000, MAX_RETRY_DELAY_MS);
30+
}
31+
32+
/**
33+
* Sleeps for `ms` milliseconds, but resolves immediately if `signal` is aborted.
34+
* This ensures retry backoff doesn't block an aborted request.
35+
*/
36+
export function abortableSleep(ms: number, signal?: AbortSignal): Promise<void> {
37+
if (signal?.aborted) return Promise.resolve();
38+
return new Promise((resolve) => {
39+
const timer = setTimeout(resolve, ms);
40+
signal?.addEventListener(
41+
"abort",
42+
() => {
43+
clearTimeout(timer);
44+
resolve();
45+
},
46+
{ once: true },
47+
);
48+
});
49+
}

‎packages/stratus-sdk/tests/azure/model-errors.test.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ describe("AzureChatCompletionsModel error handling", () => {
8080
ok: false,
8181
status: 429,
8282
text: "Rate limited",
83-
headers: new Headers({ "retry-after": "0" }),
83+
headers: new Headers({ "retry-after-ms": "1" }),
8484
},
8585
{
8686
ok: true,
@@ -205,25 +205,25 @@ describe("AzureChatCompletionsModel error handling", () => {
205205
ok: false,
206206
status: 429,
207207
text: "Rate limited",
208-
headers: new Headers({ "retry-after": "0" }),
208+
headers: new Headers({ "retry-after-ms": "1" }),
209209
},
210210
{
211211
ok: false,
212212
status: 429,
213213
text: "Rate limited",
214-
headers: new Headers({ "retry-after": "0" }),
214+
headers: new Headers({ "retry-after-ms": "1" }),
215215
},
216216
{
217217
ok: false,
218218
status: 429,
219219
text: "Rate limited",
220-
headers: new Headers({ "retry-after": "0" }),
220+
headers: new Headers({ "retry-after-ms": "1" }),
221221
},
222222
{
223223
ok: false,
224224
status: 429,
225225
text: "Rate limited",
226-
headers: new Headers({ "retry-after": "0" }),
226+
headers: new Headers({ "retry-after-ms": "1" }),
227227
},
228228
]);
229229

@@ -258,7 +258,7 @@ describe("AzureResponsesModel error handling", () => {
258258
ok: false,
259259
status: 429,
260260
text: "Rate limited",
261-
headers: new Headers({ "retry-after": "0" }),
261+
headers: new Headers({ "retry-after-ms": "1" }),
262262
},
263263
{
264264
ok: true,

0 commit comments

Comments
 (0)