Skip to content

Commit 25e13f2

Browse files
authored
feat: [ENG-3381] Render Responses API requests for AI Gateway (#5143)
* track ai gateway body mapping in clickhouse * fix * render reqs * render responses api content properly * build issues and track reasoning * fix build + jawn types * remove gateway endpoint version * fix to show request preview * better reasoning ui * remove delete migration
1 parent a4fed27 commit 25e13f2

File tree

34 files changed

+215
-80
lines changed

34 files changed

+215
-80
lines changed

bifrost/lib/clients/jawnTypes/private.ts

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1949,7 +1949,7 @@ Json: JsonObject;
19491949
cache_enabled: boolean;
19501950
updated_at?: string;
19511951
request_referrer?: string | null;
1952-
gateway_endpoint_version: string | null;
1952+
ai_gateway_body_mapping: string | null;
19531953
};
19541954
"ResultSuccess_HeliconeRequest-Array_": {
19551955
data: components["schemas"]["HeliconeRequest"][];
@@ -2352,10 +2352,9 @@ Json: JsonObject;
23522352
customer_id: string;
23532353
};
23542354
/** @enum {string} */
2355-
ResponseFormat: "ANTHROPIC" | "OPENAI";
2355+
BodyMappingType: "OPENAI" | "NO_MAPPING" | "RESPONSES";
23562356
HeliconeMeta: {
2357-
gatewayEndpointVersion?: string;
2358-
gatewayResponseFormat?: components["schemas"]["ResponseFormat"];
2357+
aiGatewayBodyMapping?: components["schemas"]["BodyMappingType"];
23592358
providerModelId?: string;
23602359
gatewayModel?: string;
23612360
gatewayProvider?: components["schemas"]["ModelProviderName"];

bifrost/lib/clients/jawnTypes/public.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1841,7 +1841,7 @@ export interface components {
18411841
cache_enabled: boolean;
18421842
updated_at?: string;
18431843
request_referrer?: string | null;
1844-
gateway_endpoint_version: string | null;
1844+
ai_gateway_body_mapping: string | null;
18451845
};
18461846
"ResultSuccess_HeliconeRequest-Array_": {
18471847
data: components["schemas"]["HeliconeRequest"][];
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
ALTER TABLE request_response_rmt
2+
ADD COLUMN ai_gateway_body_mapping String DEFAULT '' AFTER model;

docs/swagger.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4210,7 +4210,7 @@
42104210
"type": "string",
42114211
"nullable": true
42124212
},
4213-
"gateway_endpoint_version": {
4213+
"ai_gateway_body_mapping": {
42144214
"type": "string",
42154215
"nullable": true
42164216
}
@@ -4253,7 +4253,7 @@
42534253
"model",
42544254
"cache_reference_id",
42554255
"cache_enabled",
4256-
"gateway_endpoint_version"
4256+
"ai_gateway_body_mapping"
42574257
],
42584258
"type": "object",
42594259
"additionalProperties": false

packages/__tests__/llm-mapper/openai-responses.test.ts

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -317,7 +317,7 @@ describe("OpenAI Responses API Mapper", () => {
317317
{
318318
id: "call_[REDACTED]",
319319
name: "[REDACTED_FUNCTION_NAME]",
320-
arguments: '{"query": "[REDACTED_QUERY]"}',
320+
arguments: { query: "[REDACTED_QUERY]" },
321321
type: "function",
322322
},
323323
],
@@ -358,13 +358,13 @@ describe("OpenAI Responses API Mapper", () => {
358358
{
359359
id: "call_[REDACTED_1]",
360360
name: "[REDACTED_FUNCTION_1]",
361-
arguments: '{"param1": "[REDACTED_VALUE_1]"}',
361+
arguments: { param1: "[REDACTED_VALUE_1]" },
362362
type: "function",
363363
},
364364
{
365365
id: "call_[REDACTED_2]",
366366
name: "[REDACTED_FUNCTION_2]",
367-
arguments: '{"param2": "[REDACTED_VALUE_2]"}',
367+
arguments: { param2: "[REDACTED_VALUE_2]" },
368368
type: "function",
369369
},
370370
],
@@ -406,7 +406,7 @@ describe("OpenAI Responses API Mapper", () => {
406406
{
407407
id: "call_[REDACTED]",
408408
name: "[REDACTED_FUNCTION]",
409-
arguments: '{"query": "[REDACTED]"}',
409+
arguments: { query: "[REDACTED]" },
410410
type: "function",
411411
},
412412
],
@@ -692,7 +692,7 @@ describe("OpenAI Responses API Mapper", () => {
692692
{
693693
id: "call_[REDACTED]",
694694
name: "[REDACTED_SEARCH_FUNCTION]",
695-
arguments: '{"query": "[REDACTED_SEARCH_QUERY]"}',
695+
arguments: { query: "[REDACTED_SEARCH_QUERY]" },
696696
type: "function",
697697
},
698698
],

packages/llm-mapper/mappers/openai/responses.ts

Lines changed: 136 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { MapperBuilder } from "../../path-mapper/builder";
2-
import { LlmSchema, Message, Response, LLMPreview } from "../../types";
2+
import { LlmSchema, Message, LLMPreview } from "../../types";
3+
import { parseFunctionArguments } from "./chat_helpers";
34

45
const typeMap: Record<string, Message["_type"]> = {
56
input_text: "message",
@@ -111,6 +112,22 @@ export const getRequestText = (requestBody: OpenAIResponseRequest): string => {
111112
return "";
112113
}
113114

115+
// Handle reasoning items - extract summary text
116+
if ((lastItem as any)?.type === "reasoning") {
117+
const summary = (lastItem as any)?.summary;
118+
if (Array.isArray(summary)) {
119+
const textItems = summary
120+
.filter((s: any) => s?.type === "summary_text" && s?.text)
121+
.map((s: any) => s.text);
122+
if (textItems.length > 0) {
123+
return textItems.join(" ");
124+
}
125+
} else if (typeof summary === "string") {
126+
return summary;
127+
}
128+
return "";
129+
}
130+
114131
const content = (lastItem as any)?.content;
115132

116133
// Content can be a string or an array of typed items
@@ -284,7 +301,7 @@ const convertRequestInputToMessages = (
284301
const toolCall = {
285302
id: msg.id || msg.call_id || `req-tool-${msgIdx}`,
286303
name: msg.name,
287-
arguments: msg.arguments,
304+
arguments: parseFunctionArguments(msg.arguments),
288305
type: "function",
289306
};
290307

@@ -316,6 +333,34 @@ const convertRequestInputToMessages = (
316333
return;
317334
}
318335

336+
// Handle reasoning messages from reasoning models (o1, o3, etc.)
337+
if (msg.type === "reasoning") {
338+
let reasoningContent = "";
339+
if (Array.isArray(msg.summary)) {
340+
reasoningContent = msg.summary
341+
.map((s: any) => {
342+
if (s.type === "summary_text" && s.text) {
343+
return s.text;
344+
}
345+
return typeof s === "string" ? s : JSON.stringify(s);
346+
})
347+
.join(" ");
348+
} else if (typeof msg.summary === "string") {
349+
reasoningContent = msg.summary;
350+
} else if (msg.summary) {
351+
reasoningContent = JSON.stringify(msg.summary);
352+
}
353+
354+
messages.push({
355+
_type: "message",
356+
role: "assistant",
357+
reasoning: reasoningContent,
358+
content: "", // Reasoning messages don't have regular content
359+
id: msg.id || `req-msg-reasoning-${msgIdx}`,
360+
});
361+
return;
362+
}
363+
319364
// Handle regular messages with role and content (type="message" in the new format)
320365
if ((msg.type === "message" || msg.role) && msg.content !== undefined) {
321366
if (typeof msg.content === "string") {
@@ -506,11 +551,12 @@ const convertTools = (
506551
tools?: OpenAIResponseRequest["tools"]
507552
): LlmSchema["request"]["tools"] => {
508553
if (!tools) return [];
509-
return tools.map((tool) => ({
554+
return tools.map((tool: any) => ({
510555
type: "function",
511-
name: tool.function?.name,
512-
description: tool.function?.description,
513-
parameters: tool.function?.parameters,
556+
// Support both Chat API (nested) and Responses API (flat) formats
557+
name: tool.function?.name ?? tool.name,
558+
description: tool.function?.description ?? tool.description,
559+
parameters: tool.function?.parameters ?? tool.parameters,
514560
}));
515561
};
516562

@@ -673,17 +719,27 @@ const convertResponse = (responseBody: any): Message[] => {
673719
if (responseBody?.item?.content && Array.isArray(responseBody.item.content)) {
674720
const item = responseBody.item;
675721
let messageText = "";
722+
let reasoningText = "";
676723

677724
// Find the 'output_text' item in the content array
678725
const textContent = item.content.find((c: any) => c.type === "output_text");
679726
if (textContent && textContent.text) {
680727
messageText = textContent.text;
681728
}
682729

730+
// Extract reasoning if present
731+
const reasoningContent = item.content.find(
732+
(c: any) => c.type === "output_reasoning"
733+
);
734+
if (reasoningContent && reasoningContent.text) {
735+
reasoningText = reasoningContent.text;
736+
}
737+
683738
messages.push({
684739
_type: "message",
685740
role: item.role || "assistant",
686741
content: messageText,
742+
reasoning: reasoningText || undefined,
687743
id: item.id || "resp-msg-0",
688744
});
689745

@@ -703,6 +759,7 @@ const convertResponse = (responseBody: any): Message[] => {
703759
// Look for items of type 'message'
704760
if (outputItem.type === "message" && outputItem.content) {
705761
let messageText = "";
762+
let reasoningText = "";
706763
// The content is an array, find the 'output_text' item
707764
if (Array.isArray(outputItem.content)) {
708765
const textContent = outputItem.content.find(
@@ -711,15 +768,69 @@ const convertResponse = (responseBody: any): Message[] => {
711768
if (textContent && textContent.text) {
712769
messageText = textContent.text;
713770
}
771+
772+
// Extract reasoning if present
773+
const reasoningContent = outputItem.content.find(
774+
(c: any) => c.type === "output_reasoning"
775+
);
776+
if (reasoningContent && reasoningContent.text) {
777+
reasoningText = reasoningContent.text;
778+
}
714779
}
715780

716781
messages.push({
717782
_type: "message",
718783
role: outputItem.role || "assistant", // Get role from the message item
719784
content: messageText,
785+
reasoning: reasoningText || undefined,
720786
id: outputItem.id || `resp-msg-${index}`, // Use ID from the output item if available
721787
});
722788
}
789+
790+
// Handle standalone reasoning items (e.g., from reasoning models like o1, o3)
791+
if (outputItem.type === "reasoning") {
792+
let reasoningContent = "";
793+
794+
if (Array.isArray(outputItem.summary)) {
795+
reasoningContent = outputItem.summary
796+
.map((s: any) => {
797+
if (s.type === "summary_text" && s.text) {
798+
return s.text;
799+
}
800+
return typeof s === "string" ? s : JSON.stringify(s);
801+
})
802+
.join(" ");
803+
} else if (typeof outputItem.summary === "string") {
804+
reasoningContent = outputItem.summary;
805+
} else if (outputItem.summary) {
806+
reasoningContent = JSON.stringify(outputItem.summary);
807+
}
808+
809+
messages.push({
810+
_type: "message",
811+
role: "assistant",
812+
reasoning: reasoningContent,
813+
content: "",
814+
id: outputItem.id || `resp-msg-reasoning-${index}`,
815+
});
816+
}
817+
818+
// Handle function_call items (assistant tool calls in responses)
819+
if (outputItem.type === "function_call") {
820+
const toolCall = {
821+
id: outputItem.id || outputItem.call_id || `resp-tool-${index}`,
822+
name: outputItem.name,
823+
arguments: parseFunctionArguments(outputItem.arguments),
824+
};
825+
826+
messages.push({
827+
_type: "functionCall",
828+
role: "assistant",
829+
tool_calls: [toolCall],
830+
content: "",
831+
id: outputItem.id || `resp-msg-${index}`,
832+
});
833+
}
723834
}
724835
);
725836

@@ -767,6 +878,25 @@ export const mapOpenAIResponse = ({
767878
// Look for the last user message or any message with content
768879
for (let i = requestMessages.length - 1; i >= 0; i--) {
769880
const message = requestMessages[i];
881+
882+
// Handle contentArray type messages (messages with array content)
883+
if (
884+
message?._type === "contentArray" &&
885+
Array.isArray(message.contentArray)
886+
) {
887+
for (const item of message.contentArray) {
888+
if (
889+
item?._type === "message" &&
890+
item.content &&
891+
typeof item.content === "string" &&
892+
item.content.trim().length > 0
893+
) {
894+
return item.content;
895+
}
896+
}
897+
}
898+
899+
// Handle regular messages with string content
770900
if (
771901
message?.content &&
772902
typeof message.content === "string" &&

packages/llm-mapper/types.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ import { ModelProviderName } from "../cost/models/providers";
44
export const DEFAULT_UUID = "00000000-0000-0000-0000-000000000000";
55

66
export type MapperType =
7-
| "ai-gateway"
7+
| "ai-gateway-chat"
8+
| "ai-gateway-responses"
89
| "openai-chat"
910
| "openai-response"
1011
| "anthropic-chat"
@@ -388,5 +389,5 @@ export interface HeliconeRequest {
388389
cache_enabled: boolean;
389390
updated_at?: string;
390391
request_referrer?: string | null;
391-
gateway_endpoint_version: string | null;
392+
ai_gateway_body_mapping: string | null;
392393
}

packages/llm-mapper/utils/getMappedContent.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@ const MAX_PREVIEW_LENGTH = 1_000;
2929
export const MAPPERS: Record<MapperType, MapperFn<any, any>> = {
3030
// the request-response will be converted to openai format if necessary
3131
// thus uses the same mapper.
32-
"ai-gateway": mapOpenAIRequest,
32+
"ai-gateway-chat": mapOpenAIRequest,
33+
"ai-gateway-responses": mapOpenAIResponse,
3334
"openai-chat": mapOpenAIRequest,
3435
"openai-response": mapOpenAIResponse,
3536
"anthropic-chat": mapAnthropicRequest,

packages/llm-mapper/utils/getMapperType.ts

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,12 @@ export const getMapperTypeFromHeliconeRequest = (
3838
}
3939

4040
if (heliconeRequest.request_referrer === "ai-gateway") {
41-
return "ai-gateway";
41+
const bodyMapping = heliconeRequest.ai_gateway_body_mapping;
42+
if (bodyMapping === "RESPONSES") {
43+
return "ai-gateway-responses";
44+
} else if (bodyMapping === "OPENAI") {
45+
return "ai-gateway-chat";
46+
}
4247
}
4348

4449
// Check for OpenAI Assistant responses

packages/tsconfig.tsbuildinfo

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)