feat: prompts reorg to maximize cache hits

Author: theuargb

src/app/api/chat/openai-realtime/route.ts

@@ -7,8 +7,9 @@ import {
   mergeSystemPrompt,
 } from "../shared.chat";
 import {
+  buildCurrentDateTimeSystemPrompt,
   buildMcpServerCustomizationsSystemPrompt,
-  buildSpeechSystemPrompt,
+  buildSpeechSystemStaticPrompt,
 } from "lib/ai/prompts";
 
 import { safe } from "ts-safe";
@@ -84,14 +85,16 @@ export async function POST(request: NextRequest) {
     );
 
     const systemPrompt = mergeSystemPrompt(
-      buildSpeechSystemPrompt(
+      buildSpeechSystemStaticPrompt(
         session.user,
         userPreferences ?? undefined,
         agent,
       ),
       buildMcpServerCustomizationsSystemPrompt(mcpServerCustomizations),
     );
 
+    const dynamicSystemPrompt = buildCurrentDateTimeSystemPrompt();
+
     const bindingTools = [...openAITools, ...DEFAULT_VOICE_TOOLS];
 
     const r = await fetch("https://api.openai.com/v1/realtime/sessions", {
@@ -107,7 +110,7 @@ export async function POST(request: NextRequest) {
         input_audio_transcription: {
           model: "whisper-1",
         },
-        instructions: systemPrompt,
+        instructions: systemPrompt + "\n\n" + dynamicSystemPrompt,
         tools: bindingTools,
       }),
     });

src/app/api/chat/route.ts

@@ -14,8 +14,9 @@ import { customModelProvider, isToolCallUnsupportedModel } from "lib/ai/models";
 import { agentRepository, chatRepository } from "lib/db/repository";
 import globalLogger from "logger";
 import {
+  buildCurrentDateTimeSystemPrompt,
   buildMcpServerCustomizationsSystemPrompt,
-  buildUserSystemPrompt,
+  buildUserSystemStaticPrompt,
   buildToolCallUnsupportedModelSystemPrompt,
 } from "lib/ai/prompts";
 import {
@@ -263,11 +264,13 @@ export async function POST(request: Request) {
           .orElse({});
 
         const systemPrompt = mergeSystemPrompt(
-          buildUserSystemPrompt(session.user, userPreferences, agent),
+          buildUserSystemStaticPrompt(session.user, userPreferences, agent),
           buildMcpServerCustomizationsSystemPrompt(mcpServerCustomizations),
           !supportToolCall && buildToolCallUnsupportedModelSystemPrompt,
         );
 
+        const dynamicSystemPrompt = buildCurrentDateTimeSystemPrompt();
+
         const IMAGE_TOOL: Record<string, Tool> = useImageTool
           ? {
               [ImageToolName]:
@@ -318,7 +321,10 @@ export async function POST(request: Request) {
         const result = streamText({
           model,
           system: systemPrompt,
-          messages: convertToModelMessages(messages),
+          messages: [
+            { role: "system", content: dynamicSystemPrompt },
+            ...convertToModelMessages(messages),
+          ],
           experimental_transform: smoothStream({ chunking: "word" }),
           maxRetries: 2,
           tools: vercelAITooles,

src/app/api/chat/temporary/route.ts

@@ -7,7 +7,10 @@ import {
 } from "ai";
 import { customModelProvider } from "lib/ai/models";
 import globalLogger from "logger";
-import { buildUserSystemPrompt } from "lib/ai/prompts";
+import {
+  buildCurrentDateTimeSystemPrompt,
+  buildUserSystemStaticPrompt,
+} from "lib/ai/prompts";
 import { getUserPreferences } from "lib/user/server";
 
 import { colorize } from "consola/utils";
@@ -40,10 +43,16 @@ export async function POST(request: Request) {
 
     return streamText({
       model,
-      system: `${buildUserSystemPrompt(session.user, userPreferences)} ${
-        instructions ? `\n\n${instructions}` : ""
-      }`.trim(),
-      messages: convertToModelMessages(messages),
+      system: [
+        buildUserSystemStaticPrompt(session.user, userPreferences),
+        instructions,
+      ]
+        .filter(Boolean)
+        .join("\n\n"),
+      messages: [
+        { role: "system", content: buildCurrentDateTimeSystemPrompt() },
+        ...convertToModelMessages(messages),
+      ],
       experimental_transform: smoothStream({ chunking: "word" }),
     }).toUIMessageStreamResponse();
   } catch (error: any) {

src/lib/ai/prompts.ts

@@ -6,6 +6,8 @@ import { createMCPToolId } from "./mcp/mcp-tool-id";
 import { format } from "date-fns";
 import { Agent } from "app-types/agent";
 
+const CURRENT_TIME_FORMAT = "EEEE, MMMM d, yyyy 'at' h:mm:ss a";
+
 export const CREATE_THREAD_TITLE_PROMPT = `
 You are a chat title generation expert.
 
@@ -52,19 +54,34 @@ export const buildUserSystemPrompt = (
   user?: User,
   userPreferences?: UserPreferences,
   agent?: Agent,
+) => {
+  return [
+    buildUserSystemStaticPrompt(user, userPreferences, agent),
+    buildCurrentDateTimeSystemPrompt(),
+  ]
+    .filter(Boolean)
+    .join("\n\n");
+};
+
+export const buildCurrentDateTimeSystemPrompt = () => {
+  const currentTime = format(new Date(), CURRENT_TIME_FORMAT);
+  return `The current date and time is ${currentTime}.`;
+};
+
+export const buildUserSystemStaticPrompt = (
+  user?: User,
+  userPreferences?: UserPreferences,
+  agent?: Agent,
 ) => {
   const assistantName =
     agent?.name || userPreferences?.botName || "better-chatbot";
-  const currentTime = format(new Date(), "EEEE, MMMM d, yyyy 'at' h:mm:ss a");
 
-  let prompt = `You are ${assistantName}`;
+  let prompt = `You are ${assistantName}.`;
 
   if (agent?.instructions?.role) {
-    prompt += `. You are an expert in ${agent.instructions.role}`;
+    prompt += ` You are an expert in ${agent.instructions.role}.`;
   }
 
-  prompt += `. The current date and time is ${currentTime}.`;
-
   // Agent-specific instructions as primary core
   if (agent?.instructions?.systemPrompt) {
     prompt += `
@@ -136,17 +153,29 @@ export const buildSpeechSystemPrompt = (
   user: User,
   userPreferences?: UserPreferences,
   agent?: Agent,
+) => {
+  return [
+    buildSpeechSystemStaticPrompt(user, userPreferences, agent),
+    buildCurrentDateTimeSystemPrompt(),
+  ]
+    .filter(Boolean)
+    .join("\n\n");
+};
+
+export const buildSpeechSystemStaticPrompt = (
+  user: User,
+  userPreferences?: UserPreferences,
+  agent?: Agent,
 ) => {
   const assistantName = agent?.name || userPreferences?.botName || "Assistant";
-  const currentTime = format(new Date(), "EEEE, MMMM d, yyyy 'at' h:mm:ss a");
 
   let prompt = `You are ${assistantName}`;
 
   if (agent?.instructions?.role) {
     prompt += `. You are an expert in ${agent.instructions.role}`;
   }
 
-  prompt += `. The current date and time is ${currentTime}.`;
+  prompt += ".";
 
   // Agent-specific instructions as primary core
   if (agent?.instructions?.systemPrompt) {