feat: [ENG-3365] implicit caching for anthropic endpoints on AI Gateway (#5127)

connortbot · web-flow · commit a4fed274bf05 · 2025-10-28T11:36:57.000-07:00
diff --git a/packages/cost/models/providers/bedrock.ts b/packages/cost/models/providers/bedrock.ts
@@ -113,7 +113,11 @@ export class BedrockProvider extends BaseProvider {
     if (endpoint.providerModelId.includes("claude-")) {
       const anthropicBody =
         context.bodyMapping === "OPENAI"
-          ? context.toAnthropic(context.parsedBody, endpoint.providerModelId)
+          ? context.toAnthropic(
+              context.parsedBody,
+              endpoint.providerModelId,
+              { includeCacheBreakpoints: false }
+            )
           : context.parsedBody;
 
       const updatedBody = {
diff --git a/packages/cost/models/providers/vertex.ts b/packages/cost/models/providers/vertex.ts
@@ -97,7 +97,11 @@ export class VertexProvider extends BaseProvider {
     if (endpoint.providerModelId.includes("claude-")) {
       const anthropicBody =
         context.bodyMapping === "OPENAI"
-          ? context.toAnthropic(context.parsedBody, endpoint.providerModelId)
+          ? context.toAnthropic(
+              context.parsedBody,
+              endpoint.providerModelId,
+              { includeCacheBreakpoints: false }
+            )
           : context.parsedBody;
       const updatedBody = {
         ...anthropicBody,
diff --git a/packages/cost/models/types.ts b/packages/cost/models/types.ts
@@ -265,5 +265,9 @@ export interface AuthResult {
 export interface RequestBodyContext {
   parsedBody: any;
   bodyMapping: BodyMappingType;
-  toAnthropic: (body: any, providerModelId?: string) => any;
+  toAnthropic: (
+    body: any,
+    providerModelId?: string,
+    options?: { includeCacheBreakpoints?: boolean }
+  ) => any;
 }
diff --git a/packages/llm-mapper/transform/providers/openai/request/toAnthropic.ts b/packages/llm-mapper/transform/providers/openai/request/toAnthropic.ts
@@ -14,8 +14,12 @@ import { Plugin } from "@helicone-package/cost/models/types";
 export function toAnthropic(
   openAIBody: HeliconeChatCreateParams,
   providerModelId?: string,
-  plugins?: Plugin[]
+  plugins?: Plugin[],
+  options?: {
+    includeCacheBreakpoints?: boolean;
+  }
 ): AnthropicRequestBody {
+  const includeCache = options?.includeCacheBreakpoints !== false;
   const antBody: AnthropicRequestBody = {
     model: providerModelId || openAIBody.model,
     messages: [],
@@ -35,8 +39,11 @@ export function toAnthropic(
         : [];
   }
 
-  const { messages, system } = extractSystemMessage(openAIBody.messages);
-  antBody.messages = mapMessages(messages);
+  const { messages, system } = extractSystemMessage(
+    openAIBody.messages,
+    includeCache
+  );
+  antBody.messages = mapMessages(messages, includeCache);
   if (system) {
     antBody.system = system;
   }
@@ -87,11 +94,18 @@ export function toAnthropic(
     throw new Error("Logit bias is not supported");
   }
 
+  // if and only if the user did not provide any cache control breakpoints,
+  // add a default ephemeral 5m breakpoint at the end of the content.
+  if (includeCache && !hasAnyCacheControl(openAIBody)) {
+    appendDefaultCacheBreakpoint(antBody);
+  }
+
   return antBody;
 }
 
 function openAIContentToAnthropicContent(
-  content: string | HeliconeChatCompletionContentPart[] | null
+  content: string | HeliconeChatCompletionContentPart[] | null,
+  includeCache: boolean
 ): string | AnthropicContentBlock[] {
   if (content === null) {
     return "";
@@ -107,7 +121,9 @@ function openAIContentToAnthropicContent(
         return {
           type: "text",
           text: part.text,
-          cache_control: part.cache_control,
+          ...(includeCache && part.cache_control
+            ? { cache_control: part.cache_control }
+            : {}),
         };
       case "image_url":
         // expected format: { type: "image_url", image_url: { url: string } }
@@ -132,7 +148,9 @@ function openAIContentToAnthropicContent(
               media_type: mediaType,
               data: base64Data,
             },
-            cache_control: part.cache_control,
+            ...(includeCache && part.cache_control
+              ? { cache_control: part.cache_control }
+              : {}),
           };
         } else {
           return {
@@ -141,7 +159,9 @@ function openAIContentToAnthropicContent(
               type: "url",
               url: url,
             },
-            cache_control: part.cache_control,
+            ...(includeCache && part.cache_control
+              ? { cache_control: part.cache_control }
+              : {}),
           };
         }
       case "input_audio":
@@ -168,7 +188,67 @@ function openAIContentToAnthropicContent(
   });
 }
 
-function extractSystemMessage(messages: HeliconeChatCreateParams["messages"]): {
+function hasAnyCacheControl(body: HeliconeChatCreateParams): boolean {
+  const msgs = body.messages || [];
+  for (const m of msgs) {
+    if ((m as any).cache_control) {
+      return true;
+    }
+    const content = (m as any).content;
+    if (Array.isArray(content)) {
+      for (const part of content) {
+        if ((part as any)?.cache_control) {
+          return true;
+        }
+      }
+    }
+  }
+  return false;
+}
+
+function appendDefaultCacheBreakpoint(antBody: AnthropicRequestBody) {
+  const defaultCache = { type: "ephemeral" as const, ttl: "5m" as const };
+
+  if (antBody.messages.length > 0) {
+    const last = antBody.messages[antBody.messages.length - 1];
+    if (typeof last.content === "string") {
+      last.content = [
+        {
+          type: "text",
+          text: last.content,
+          cache_control: defaultCache,
+        },
+      ];
+      return;
+    }
+    if (Array.isArray(last.content) && last.content.length > 0) {
+      const idx = last.content.length - 1;
+      const block = last.content[idx];
+      last.content[idx] = {
+        ...block,
+        cache_control: defaultCache,
+      } as AnthropicContentBlock;
+      return;
+    }
+  }
+
+  if (antBody.system && Array.isArray(antBody.system) && antBody.system.length > 0) {
+    const idx = antBody.system.length - 1;
+    const block = antBody.system[idx];
+    antBody.system[idx] = {
+      ...block,
+      cache_control: defaultCache,
+    } as AnthropicContentBlock;
+    return;
+  }
+
+  // If system is a string and there are no messages, it's okay not to add it
+}
+
+function extractSystemMessage(
+  messages: HeliconeChatCreateParams["messages"],
+  includeCache: boolean
+): {
   messages: HeliconeChatCreateParams["messages"];
   system?: string | AnthropicContentBlock[];
 } {
@@ -183,7 +263,7 @@ function extractSystemMessage(messages: HeliconeChatCreateParams["messages"]): {
     typeof systemMessages[0].content === "string"
   ) {
     const content = systemMessages[0].content;
-    if (!systemMessages[0].cache_control) {
+    if (!includeCache || !systemMessages[0].cache_control) {
       return {
         messages: otherMessages,
         system: content,
@@ -195,20 +275,27 @@ function extractSystemMessage(messages: HeliconeChatCreateParams["messages"]): {
         {
           type: "text",
           text: systemMessages[0].content,
-          cache_control: systemMessages[0].cache_control,
+          ...(includeCache && systemMessages[0].cache_control
+            ? { cache_control: systemMessages[0].cache_control }
+            : {}),
         },
       ],
     };
   }
 
   const systemMessageBlocks: AnthropicContentBlock[] = [];
   for (const msg of systemMessages) {
-    const convertedBlock = openAIContentToAnthropicContent(msg.content);
+    const convertedBlock = openAIContentToAnthropicContent(
+      msg.content,
+      includeCache
+    );
     if (typeof convertedBlock === "string") {
       systemMessageBlocks.push({
         type: "text",
         text: convertedBlock,
-        cache_control: msg.cache_control,
+        ...(includeCache && (msg as any).cache_control
+          ? { cache_control: (msg as any).cache_control }
+          : {}),
       });
     } else {
       systemMessageBlocks.push(...convertedBlock);
@@ -222,7 +309,8 @@ function extractSystemMessage(messages: HeliconeChatCreateParams["messages"]): {
 }
 
 function mapMessages(
-  messages: HeliconeChatCreateParams["messages"]
+  messages: HeliconeChatCreateParams["messages"],
+  includeCache: boolean
 ): AnthropicRequestBody["messages"] {
   if (!messages) {
     return [];
@@ -241,7 +329,9 @@ function mapMessages(
             type: "tool_result",
             tool_use_id: message.tool_call_id,
             content: typeof message.content === "string" ? message.content : "",
-            cache_control: message.cache_control,
+            ...(includeCache && (message as any).cache_control
+              ? { cache_control: (message as any).cache_control }
+              : {}),
           },
         ],
       };
@@ -272,18 +362,19 @@ function mapMessages(
       let processedContent: string | AnthropicContentBlock[] = [];
       if (message.content) {
         const convertedContent = openAIContentToAnthropicContent(
-          message.content
+          message.content,
+          includeCache
         );
         if (typeof convertedContent === "string") {
           // if the message requires forming a content array
-          if (
-            message.cache_control ||
-            processedToolCallContent.length > 0
-          ) {
+          const hasMsgCache = includeCache && !!(message as any).cache_control;
+          if (hasMsgCache || processedToolCallContent.length > 0) {
             processedContent.push({
               type: "text",
               text: convertedContent,
-              cache_control: message.cache_control,
+              ...(includeCache && (message as any).cache_control
+                ? { cache_control: (message as any).cache_control }
+                : {}),
             });
           } else {
             // there was no cache control breakpoint, the content was just string,
@@ -300,8 +391,10 @@ function mapMessages(
       antMessage.content = processedContent;
       return antMessage;
     }
-
-    antMessage.content = openAIContentToAnthropicContent(message.content);
+    antMessage.content = openAIContentToAnthropicContent(
+      message.content,
+      includeCache
+    );
     return antMessage;
   });
 }
diff --git a/worker/src/lib/ai-gateway/AttemptExecutor.ts b/worker/src/lib/ai-gateway/AttemptExecutor.ts
@@ -135,7 +135,7 @@ export class AttemptExecutor {
       const bodyResult = await buildRequestBody(endpoint, {
         parsedBody,
         bodyMapping: requestWrapper.heliconeHeaders.gatewayConfig.bodyMapping,
-        toAnthropic: (body, modelId) => toAnthropic(body, modelId, plugins),
+        toAnthropic: (body, modelId, options) => toAnthropic(body, modelId, plugins, options)
       });
 
       if (isErr(bodyResult) || !bodyResult.data) {
diff --git a/worker/test/ai-gateway/map-anthropic.spec.ts b/worker/test/ai-gateway/map-anthropic.spec.ts
@@ -233,10 +233,10 @@ describe("Anthropic to OpenAI Response Mapper", () => {
 
       expect(result.model).toBe("claude-3.5-haiku");
       expect(result.messages).toHaveLength(1);
-      expect(result.messages[0]).toEqual({
-        role: "user",
-        content: "Hello, how are you?",
-      });
+      expect(result.messages[0].role).toBe("user");
+      expect(result.messages[0].content).toEqual([
+        { type: "text", text: "Hello, how are you?", cache_control: { type: "ephemeral", ttl: "5m" } },
+      ]);
       expect(result.temperature).toBe(0.7);
       expect(result.max_tokens).toBe(1000);
       expect(result.stream).toBe(false);
@@ -321,6 +321,7 @@ describe("Anthropic to OpenAI Response Mapper", () => {
         type: "tool_result",
         tool_use_id: "call_DoWwQL7W9AJBpwCtn9tRC2gt",
         content: "100",
+        cache_control: { type: "ephemeral", ttl: "5m" },
       });
 
       expect(result.tools).toHaveLength(1);
@@ -375,6 +376,7 @@ describe("Anthropic to OpenAI Response Mapper", () => {
         id: "call_123",
         name: "calculate",
         input: { expression: "2+2" },
+        cache_control: { type: "ephemeral", ttl: "5m" },
       });
     });
 
@@ -397,10 +399,10 @@ describe("Anthropic to OpenAI Response Mapper", () => {
 
       expect(result.system).toBe("You are a helpful assistant.");
       expect(result.messages).toHaveLength(1);
-      expect(result.messages[0]).toEqual({
-        role: "user",
-        content: "Hello",
-      });
+      expect(result.messages[0].role).toBe("user");
+      expect(result.messages[0].content).toEqual([
+        { type: "text", text: "Hello", cache_control: { type: "ephemeral", ttl: "5m" } },
+      ]);
     });
 
     it("should handle cache control on system message", () => {
