fix: [AI chat] add auto-batching for vectorizing to avoid max token limits (#2693)

Author: dubwub

packages/fern-docs/bundle/package.json

@@ -93,7 +93,7 @@
     "@vercel/kv": "^2.0.0",
     "@vercel/otel": "^1.10.1",
     "@workos-inc/node": "^7.31.0",
-    "ai": "^4.2.10",
+    "ai": "^4.3.13",
     "algoliasearch": "^5.20.3",
     "bezier-easing": "^2.1.0",
     "braintrust": "^0.0.184",

packages/fern-docs/bundle/src/app/[host]/[domain]/api/fern-docs/search/v2/reindex/turbopuffer/route.ts

@@ -1,7 +1,7 @@
 import { NextRequest, NextResponse } from "next/server";
 
 import { createOpenAI } from "@ai-sdk/openai";
-import { embedMany } from "ai";
+import { Embedding, embedMany } from "ai";
 
 import { getAuthEdgeConfig, getEdgeFlags } from "@fern-docs/edge-config";
 import { turbopufferUpsertTask } from "@fern-docs/search-server/turbopuffer";
@@ -79,11 +79,24 @@ export async function GET(req: NextRequest): Promise<NextResponse> {
         ...edgeFlags,
       },
       vectorizer: async (chunks) => {
-        const embeddings = await embedMany({
-          model: embeddingModel,
-          values: chunks,
-        });
-        return embeddings.embeddings;
+        // max 300k tokens per request, handle this manually
+        let payload = [];
+        let payloadLength = 0;
+        let embeddings: Embedding[] = [];
+        for (const chunk of chunks) {
+          payloadLength += chunk.length;
+          payload.push(chunk);
+          if (payloadLength >= 100000) {
+            const embeddingOutput = await embedMany({
+              model: embeddingModel,
+              values: payload,
+            });
+            embeddings = embeddings.concat(embeddingOutput.embeddings);
+            payload = [];
+            payloadLength = 0;
+          }
+        }
+        return embeddings;
       },
       authed: (node) => {
         if (authEdgeConfig == null) {