Skip to content

Commit 086f45f

Browse files
jpr5jpr5
committed
test: add WebSocket drift detection for Responses WS, Realtime, and Gemini Live
TLS WebSocket client (ws-providers.ts) connects to real provider WS endpoints using node:tls with RFC 6455 framing, ping/pong, and connection-scoped message cursors for multi-step protocols. 4 verified drift tests: - OpenAI Responses WS: text + tool call - OpenAI Realtime: text + tool call (gpt-4o-mini-realtime-preview) Canaries: - Realtime: checks gpt-4o-mini-realtime-preview still exists in model listing API, with hints for the GA replacement - Gemini Live: checks model listing API for text-capable bidiGenerateContent models; full drift tests skipped until Google ships a non-audio Live model Supporting changes: - sdk-shapes.ts: Realtime + Gemini Live event shapes - helpers.ts: collectMockWSMessages(), classifyGeminiMessage, GEMINI_WS_PATH - models.drift.ts: filter markdown anchor fragments from model scraper
1 parent 667206e commit 086f45f

7 files changed

Lines changed: 1392 additions & 5 deletions

File tree

‎src/__tests__/drift/helpers.ts‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,12 @@
1010
import http from "node:http";
1111
import { createServer, type ServerInstance } from "../../server.js";
1212
import type { Fixture } from "../../types.js";
13+
import type { WSTestClient } from "../ws-test-client.js";
14+
import { extractShape, type SSEEventShape } from "./schema.js";
15+
16+
import { classifyGeminiMessage } from "./ws-providers.js";
17+
18+
export { classifyGeminiMessage };
1319

1420
// ---------------------------------------------------------------------------
1521
// HTTP helpers
@@ -101,3 +107,77 @@ export async function startDriftServer(): Promise<ServerInstance> {
101107
export async function stopDriftServer(instance: ServerInstance): Promise<void> {
102108
await new Promise<void>((r) => instance.server.close(() => r()));
103109
}
110+
111+
// ---------------------------------------------------------------------------
112+
// WebSocket helpers
113+
// ---------------------------------------------------------------------------
114+
115+
export const GEMINI_WS_PATH =
116+
"/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent";
117+
118+
/**
119+
* Collect mock WS messages until a terminal predicate fires.
120+
*
121+
* Uses a polling loop on waitForMessages() since ws-test-client doesn't
122+
* support predicate-based collection. The `skip` parameter tells us how
123+
* many messages have already been consumed so we don't re-read them.
124+
*
125+
* Throws if the terminal predicate never fires before the timeout expires.
126+
*/
127+
export async function collectMockWSMessages(
128+
client: WSTestClient,
129+
terminal: (msg: unknown) => boolean,
130+
timeoutMs = 15000,
131+
skip = 0,
132+
): Promise<{ events: SSEEventShape[]; rawMessages: unknown[] }> {
133+
const rawMessages: unknown[] = [];
134+
const deadline = Date.now() + timeoutMs;
135+
let count = skip;
136+
let terminated = false;
137+
138+
while (Date.now() < deadline) {
139+
const nextCount = count + 1;
140+
let msgs: string[];
141+
try {
142+
msgs = await client.waitForMessages(nextCount, Math.min(2000, deadline - Date.now()));
143+
} catch (e: unknown) {
144+
// Only suppress waitForMessages timeout — rethrow anything else
145+
if (e instanceof Error && e.message.includes("Timeout waiting for")) {
146+
if (Date.now() >= deadline) break;
147+
continue;
148+
}
149+
throw e;
150+
}
151+
// Only increment count after successful receipt
152+
count = nextCount;
153+
const latest = msgs[count - 1];
154+
let parsed: unknown;
155+
try {
156+
parsed = typeof latest === "string" ? JSON.parse(latest) : latest;
157+
} catch {
158+
throw new Error(
159+
`collectMockWSMessages: failed to parse message ${count}: ${String(latest).slice(0, 200)}`,
160+
);
161+
}
162+
rawMessages.push(parsed);
163+
if (terminal(parsed)) {
164+
terminated = true;
165+
break;
166+
}
167+
}
168+
169+
if (!terminated) {
170+
throw new Error(
171+
`collectMockWSMessages timed out after ${timeoutMs}ms without terminal message. ` +
172+
`Collected ${rawMessages.length} messages.`,
173+
);
174+
}
175+
176+
const events: SSEEventShape[] = rawMessages.map((msg) => {
177+
const m = msg as Record<string, any>;
178+
const type = m.type ?? classifyGeminiMessage(m as Record<string, unknown>);
179+
return { type, dataShape: extractShape(msg) };
180+
});
181+
182+
return { events, rawMessages };
183+
}

‎src/__tests__/drift/models.drift.ts‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic model availability",
7272
if (referenced.length === 0) return;
7373

7474
for (const m of referenced) {
75-
const found = models.some((available) => available === m || available.startsWith(`${m}`));
75+
const found = models.some((available) => available === m || available.startsWith(m));
7676
expect(found, `Model ${m} no longer available at Anthropic`).toBe(true);
7777
}
7878
});
@@ -89,11 +89,14 @@ describe.skipIf(!process.env.GOOGLE_API_KEY)("Gemini model availability", () =>
8989

9090
if (referenced.length === 0) return;
9191

92-
// Skip experimental and live-only models — they're ephemeral
93-
const stable = referenced.filter((m) => !m.includes("-exp") && !m.endsWith("-live"));
92+
// Skip experimental models, live-only models, and anchor-link fragments
93+
// scraped from markdown (e.g., "gemini-live-bidigeneratecontent")
94+
const stable = referenced.filter(
95+
(m) => !m.includes("-exp") && !m.includes("-live") && !m.includes("bidigeneratecontent"),
96+
);
9497

9598
for (const m of stable) {
96-
const found = models.some((available) => available === m || available.startsWith(`${m}`));
99+
const found = models.some((available) => available === m || available.startsWith(m));
97100
expect(found, `Model ${m} no longer available at Gemini`).toBe(true);
98101
}
99102
});

‎src/__tests__/drift/sdk-shapes.ts‎

Lines changed: 274 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -425,7 +425,280 @@ export function anthropicToolStreamEventShapes(): SSEEventShape[] {
425425
}
426426

427427
// ---------------------------------------------------------------------------
428-
// Google Gemini
428+
// OpenAI Realtime API
429+
// ---------------------------------------------------------------------------
430+
431+
export function openaiRealtimeTextEventShapes(): SSEEventShape[] {
432+
return [
433+
{
434+
type: "session.created",
435+
dataShape: extractShape({
436+
type: "session.created",
437+
event_id: "evt_abc123",
438+
session: {
439+
id: "sess_abc123",
440+
model: "gpt-4o-mini",
441+
modalities: ["text"],
442+
instructions: "",
443+
tools: [],
444+
voice: null,
445+
input_audio_format: null,
446+
output_audio_format: null,
447+
turn_detection: null,
448+
temperature: 0.8,
449+
},
450+
}),
451+
},
452+
{
453+
type: "session.updated",
454+
dataShape: extractShape({
455+
type: "session.updated",
456+
event_id: "evt_abc123",
457+
session: {
458+
model: "gpt-4o-mini",
459+
modalities: ["text"],
460+
instructions: "",
461+
tools: [],
462+
voice: null,
463+
input_audio_format: null,
464+
output_audio_format: null,
465+
turn_detection: null,
466+
temperature: 0.8,
467+
},
468+
}),
469+
},
470+
{
471+
type: "conversation.item.created",
472+
dataShape: extractShape({
473+
type: "conversation.item.created",
474+
event_id: "evt_abc123",
475+
item: {
476+
type: "message",
477+
id: "item_abc123",
478+
role: "user",
479+
content: [{ type: "input_text", text: "Say hello" }],
480+
},
481+
}),
482+
},
483+
{
484+
type: "response.created",
485+
dataShape: extractShape({
486+
type: "response.created",
487+
event_id: "evt_abc123",
488+
response: {
489+
id: "resp_abc123",
490+
status: "in_progress",
491+
output: [],
492+
},
493+
}),
494+
},
495+
{
496+
type: "response.output_item.added",
497+
dataShape: extractShape({
498+
type: "response.output_item.added",
499+
event_id: "evt_abc123",
500+
response_id: "resp_abc123",
501+
output_index: 0,
502+
item: {
503+
id: "item_abc123",
504+
type: "message",
505+
role: "assistant",
506+
content: [],
507+
},
508+
}),
509+
},
510+
{
511+
type: "response.content_part.added",
512+
dataShape: extractShape({
513+
type: "response.content_part.added",
514+
event_id: "evt_abc123",
515+
response_id: "resp_abc123",
516+
item_id: "item_abc123",
517+
output_index: 0,
518+
content_index: 0,
519+
part: { type: "text", text: "" },
520+
}),
521+
},
522+
{
523+
type: "response.text.delta",
524+
dataShape: extractShape({
525+
type: "response.text.delta",
526+
event_id: "evt_abc123",
527+
response_id: "resp_abc123",
528+
item_id: "item_abc123",
529+
output_index: 0,
530+
content_index: 0,
531+
delta: "Hello",
532+
}),
533+
},
534+
{
535+
type: "response.text.done",
536+
dataShape: extractShape({
537+
type: "response.text.done",
538+
event_id: "evt_abc123",
539+
response_id: "resp_abc123",
540+
item_id: "item_abc123",
541+
output_index: 0,
542+
content_index: 0,
543+
text: "Hello!",
544+
}),
545+
},
546+
{
547+
type: "response.content_part.done",
548+
dataShape: extractShape({
549+
type: "response.content_part.done",
550+
event_id: "evt_abc123",
551+
response_id: "resp_abc123",
552+
item_id: "item_abc123",
553+
output_index: 0,
554+
content_index: 0,
555+
part: { type: "text", text: "Hello!" },
556+
}),
557+
},
558+
{
559+
type: "response.output_item.done",
560+
dataShape: extractShape({
561+
type: "response.output_item.done",
562+
event_id: "evt_abc123",
563+
response_id: "resp_abc123",
564+
output_index: 0,
565+
item: {
566+
id: "item_abc123",
567+
type: "message",
568+
role: "assistant",
569+
content: [{ type: "text", text: "Hello!" }],
570+
},
571+
}),
572+
},
573+
{
574+
type: "response.done",
575+
dataShape: extractShape({
576+
type: "response.done",
577+
event_id: "evt_abc123",
578+
response: {
579+
id: "resp_abc123",
580+
status: "completed",
581+
output: [
582+
{
583+
id: "item_abc123",
584+
type: "message",
585+
role: "assistant",
586+
content: [{ type: "text", text: "Hello!" }],
587+
},
588+
],
589+
},
590+
}),
591+
},
592+
];
593+
}
594+
595+
export function openaiRealtimeToolCallEventShapes(): SSEEventShape[] {
596+
return [
597+
{
598+
type: "response.output_item.added",
599+
dataShape: extractShape({
600+
type: "response.output_item.added",
601+
event_id: "evt_abc123",
602+
response_id: "resp_abc123",
603+
output_index: 0,
604+
item: {
605+
id: "item_abc123",
606+
type: "function_call",
607+
call_id: "call_abc123",
608+
name: "get_weather",
609+
arguments: "",
610+
},
611+
}),
612+
},
613+
{
614+
type: "response.function_call_arguments.delta",
615+
dataShape: extractShape({
616+
type: "response.function_call_arguments.delta",
617+
event_id: "evt_abc123",
618+
response_id: "resp_abc123",
619+
item_id: "item_abc123",
620+
output_index: 0,
621+
call_id: "call_abc123",
622+
delta: '{"city":',
623+
}),
624+
},
625+
{
626+
type: "response.function_call_arguments.done",
627+
dataShape: extractShape({
628+
type: "response.function_call_arguments.done",
629+
event_id: "evt_abc123",
630+
response_id: "resp_abc123",
631+
item_id: "item_abc123",
632+
output_index: 0,
633+
call_id: "call_abc123",
634+
arguments: '{"city":"Paris"}',
635+
}),
636+
},
637+
{
638+
type: "response.output_item.done",
639+
dataShape: extractShape({
640+
type: "response.output_item.done",
641+
event_id: "evt_abc123",
642+
response_id: "resp_abc123",
643+
output_index: 0,
644+
item: {
645+
id: "item_abc123",
646+
type: "function_call",
647+
call_id: "call_abc123",
648+
name: "get_weather",
649+
arguments: '{"city":"Paris"}',
650+
},
651+
}),
652+
},
653+
];
654+
}
655+
656+
// ---------------------------------------------------------------------------
657+
// Gemini Live BidiGenerateContent
658+
// ---------------------------------------------------------------------------
659+
660+
export function geminiLiveSetupCompleteShape(): SSEEventShape {
661+
return {
662+
type: "setupComplete",
663+
dataShape: extractShape({ setupComplete: {} }),
664+
};
665+
}
666+
667+
export function geminiLiveTextEventShapes(): SSEEventShape[] {
668+
return [
669+
{
670+
type: "serverContent",
671+
dataShape: extractShape({
672+
serverContent: {
673+
modelTurn: { parts: [{ text: "Hello!" }] },
674+
turnComplete: true,
675+
},
676+
}),
677+
},
678+
];
679+
}
680+
681+
export function geminiLiveToolCallEventShapes(): SSEEventShape[] {
682+
return [
683+
{
684+
type: "toolCall",
685+
dataShape: extractShape({
686+
toolCall: {
687+
functionCalls: [
688+
{
689+
name: "get_weather",
690+
args: { city: "Paris" },
691+
id: "call_gemini_get_weather_0",
692+
},
693+
],
694+
},
695+
}),
696+
},
697+
];
698+
}
699+
700+
// ---------------------------------------------------------------------------
701+
// Google Gemini (HTTP)
429702
// ---------------------------------------------------------------------------
430703

431704
export function geminiContentResponseShape(): ShapeNode {

0 commit comments

Comments
 (0)