fixed vercel sdk

Author: CoderOMaster

examples/cookbook/vercel-voice-agent/.env.example

@@ -2,8 +2,6 @@ MOSS_PROJECT_ID=your-project-id
 MOSS_PROJECT_KEY=your-project-key
 MOSS_INDEX_NAME=your-index-name
 
-# Vercel AI Gateway (routes to openai/gpt-4.1-mini)
+# Vercel AI Gateway — generates WebSocket tokens + routes to gpt-realtime-2
+# Get key: https://vercel.com/dashboard/ai-gateway
 AI_GATEWAY_API_KEY=your-vercel-ai-gateway-key
-
-# Deepgram (STT + TTS)
-DEEPGRAM_API_KEY=your-deepgram-api-key

examples/cookbook/vercel-voice-agent/app/api/pipeline/route.ts

@@ -0,0 +1,58 @@
+import { createGateway } from '@ai-sdk/gateway';
+import { MossClient } from '@moss-dev/moss';
+import { mossSearchTool } from '@moss-tools/vercel-sdk';
+
+export const runtime = 'nodejs';
+
+const gateway = createGateway({ apiKey: process.env.AI_GATEWAY_API_KEY });
+
+const client = new MossClient(
+  process.env.MOSS_PROJECT_ID!,
+  process.env.MOSS_PROJECT_KEY!,
+);
+
+const searchTool = mossSearchTool({
+  client,
+  indexName: process.env.MOSS_INDEX_NAME!,
+});
+
+// Load the index into local memory at startup.
+// Cloud query is broken (503) for this project — local queries work fine.
+void client.loadIndex(process.env.MOSS_INDEX_NAME!)
+  .then(() => console.log('[MOSS] index loaded locally'))
+  .catch((err: unknown) => console.error('[MOSS] loadIndex failed:', err));
+
+const MOSS_TOOL = {
+  type: 'function' as const,
+  name: 'search_knowledge_base',
+  description: searchTool.description,
+  parameters: {
+    type: 'object',
+    properties: {
+      query: { type: 'string', description: 'Concise search query' },
+      topK: { type: 'number', description: 'Number of results (default 5)' },
+    },
+    required: ['query'],
+  },
+};
+
+// POST (empty body)  → mint a short-lived WebSocket token via Vercel AI Gateway
+// POST ({ query })   → execute MOSS search on behalf of the realtime model's tool call
+export async function POST(req: Request) {
+  const body = await req.json().catch(() => ({})) as Record<string, unknown>;
+
+  if (typeof body.query === 'string') {
+    const topK = typeof body.topK === 'number' ? body.topK : 5;
+    const result = await searchTool.execute!(
+      { query: body.query, topK },
+      { toolCallId: 'realtime', messages: [], abortSignal: req.signal },
+    );
+    const docs = (result as { docs: Array<{ text: string }> }).docs ?? [];
+    return Response.json(docs.map((d) => d.text).join('\n\n'));
+  }
+
+  const { token, url } = await gateway.experimental_realtime.getToken({
+    model: 'openai/gpt-realtime-2',
+  });
+  return Response.json({ token, url, tools: [MOSS_TOOL] });
+}

examples/cookbook/vercel-voice-agent/app/api/token/route.ts

@@ -1,140 +1,108 @@
 'use client';
 
-import { useRef, useState } from 'react';
-
-type Status = 'idle' | 'recording' | 'processing' | 'speaking';
+import { experimental_useRealtime as useRealtime } from '@ai-sdk/react';
+import { gateway } from '@ai-sdk/gateway';
+import { useMemo, useState } from 'react';
 
 export default function Page() {
-  const [status, setStatus] = useState<Status>('idle');
-  const [transcript, setTranscript] = useState('');
-  const [reply, setReply] = useState('');
+  const [log, setLog] = useState<string[]>([]);
   const [error, setError] = useState<string | null>(null);
 
-  const recorderRef = useRef<MediaRecorder | null>(null);
-  const chunksRef = useRef<Blob[]>([]);
-
-  const startRecording = async () => {
-    if (status !== 'idle') return;
-    setError(null);
+  const model = useMemo(
+    () => gateway.experimental_realtime('openai/gpt-realtime-2'),
+    [],
+  );
 
-    const stream = await navigator.mediaDevices.getUserMedia({
-      audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
+  const { status, connect, disconnect, startAudioCapture, stopAudioCapture, isCapturing } =
+    useRealtime({
+      model,
+      api: { token: '/api/token' },
+      sessionConfig: {
+        voice: 'alloy',
+        turnDetection: { type: 'server-vad' },
+        instructions:
+          'You are a helpful voice assistant with access to a knowledge base. ' +
+          'Always call search_knowledge_base before answering factual questions. ' +
+          'Keep answers brief — two or three sentences.',
+      },
+      onError: (err) => {
+        console.error('[realtime error]', err);
+        setError(err instanceof Error ? err.message : String(err));
+      },
+      onToolCall: async ({ toolCall }) => {
+        if (toolCall.toolName !== 'search_knowledge_base') return;
+        const { query, topK } = toolCall.args as { query: string; topK?: number };
+        setLog((l) => [`🔍 "${query}"`, ...l.slice(0, 9)]);
+
+        const res = await fetch('/api/token', {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({ query, topK: topK ?? 5 }),
+        });
+        const text = await res.json() as string;
+        setLog((l) => [`✓ returned context`, ...l.slice(0, 9)]);
+        return text;
+      },
     });
 
-    const recorder = new MediaRecorder(stream, { mimeType: 'audio/webm' });
-    chunksRef.current = [];
-
-    recorder.ondataavailable = (e) => { if (e.data.size > 0) chunksRef.current.push(e.data); };
-    recorder.onstop = () => {
-      stream.getTracks().forEach((t) => t.stop());
-      runPipeline(new Blob(chunksRef.current, { type: 'audio/webm' }));
-    };
-
-    recorder.start();
-    recorderRef.current = recorder;
-    setStatus('recording');
-  };
-
-  const stopRecording = () => {
-    if (recorderRef.current?.state === 'recording') {
-      recorderRef.current.stop();
-      recorderRef.current = null;
-      setStatus('processing');
-    }
-  };
-
-  const runPipeline = async (blob: Blob) => {
-    try {
-      const res = await fetch('/api/pipeline', {
-        method: 'POST',
-        headers: { 'Content-Type': 'audio/webm' },
-        body: blob,
-      });
-
-      if (res.status === 204) {
-        setError('No speech detected');
-        setStatus('idle');
-        return;
+  const handleButton = async () => {
+    setError(null);
+    if (status === 'connected') {
+      stopAudioCapture();
+      disconnect();
+    } else {
+      try {
+        const stream = await navigator.mediaDevices.getUserMedia({
+          audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
+        });
+        await connect();
+        startAudioCapture(stream);
+      } catch (err) {
+        setError(err instanceof Error ? err.message : String(err));
       }
-      if (!res.ok) throw new Error(`Pipeline failed: ${res.status}`);
-
-      const raw = res.headers.get('X-Transcript');
-      if (raw) setTranscript(decodeURIComponent(raw));
-
-      const rawReply = res.headers.get('X-Reply');
-      if (rawReply) setReply(decodeURIComponent(rawReply));
-
-      const audioBlob = await res.blob();
-      const url = URL.createObjectURL(audioBlob);
-      const audio = new Audio(url);
-      setStatus('speaking');
-      audio.onended = () => { URL.revokeObjectURL(url); setStatus('idle'); };
-      audio.onerror = () => { URL.revokeObjectURL(url); setStatus('idle'); };
-      audio.play();
-    } catch (err) {
-      setError(err instanceof Error ? err.message : String(err));
-      setStatus('idle');
     }
   };
 
-  const busy = status !== 'idle';
-
   return (
-    <main style={{ maxWidth: 520, margin: '4rem auto', fontFamily: 'sans-serif', padding: '0 1rem' }}>
-      <h1 style={{ fontSize: '1.6rem', marginBottom: '0.25rem' }}>MOSS Voice Agent</h1>
+    <main style={{ maxWidth: 480, margin: '4rem auto', fontFamily: 'sans-serif', padding: '0 1rem' }}>
+      <h1 style={{ fontSize: '1.5rem', marginBottom: '0.25rem' }}>MOSS Voice Agent</h1>
       <p style={{ color: '#6b7280', fontSize: '0.85rem', marginBottom: '2rem' }}>
-        Deepgram STT · GPT-4.1 Mini · Deepgram TTS · MOSS retrieval
+        Vercel AI Gateway · gpt-realtime-2 · MOSS retrieval
       </p>
 
       <button
-        onMouseDown={startRecording}
-        onMouseUp={stopRecording}
-        onTouchStart={(e) => { e.preventDefault(); startRecording(); }}
-        onTouchEnd={(e) => { e.preventDefault(); stopRecording(); }}
-        disabled={status === 'processing' || status === 'speaking'}
+        onClick={handleButton}
+        disabled={status === 'connecting'}
         style={{
-          padding: '1rem 2.5rem',
+          padding: '0.75rem 1.75rem',
           fontSize: '1rem',
           fontWeight: 600,
-          borderRadius: 12,
+          borderRadius: 10,
           border: 'none',
-          cursor: busy && status !== 'recording' ? 'not-allowed' : 'pointer',
-          background:
-            status === 'recording'  ? '#dc2626' :
-            status === 'processing' ? '#6b7280' :
-            status === 'speaking'   ? '#059669' : '#2563eb',
+          cursor: status === 'connecting' ? 'not-allowed' : 'pointer',
+          background: status === 'connected' ? '#dc2626' : '#2563eb',
           color: '#fff',
-          userSelect: 'none',
-          WebkitUserSelect: 'none',
           transition: 'background 0.15s',
         }}
       >
-        {status === 'recording'  ? '🎙 Release to send'
-          : status === 'processing' ? 'Transcribing…'
-          : status === 'speaking'   ? '🔊 Speaking…'
-          : '● Hold to speak'}
+        {status === 'connecting' ? 'Connecting…'
+          : status === 'connected' ? 'Stop'
+          : 'Start talking'}
       </button>
 
-      <p style={{ marginTop: '0.75rem', fontSize: '0.8rem', color: '#9ca3af' }}>
-        {status === 'idle' ? 'Press and hold, speak, then release.' : `Status: ${status}`}
+      <p style={{ marginTop: '0.6rem', fontSize: '0.8rem', color: '#9ca3af' }}>
+        Status: <strong>{status}</strong>
+        {isCapturing && <span style={{ color: '#059669' }}> · mic active</span>}
       </p>
 
       {error && (
-        <p style={{ marginTop: '0.75rem', color: '#dc2626', fontSize: '0.82rem' }}>⚠ {error}</p>
-      )}
-
-      {transcript && (
-        <div style={{ marginTop: '1.5rem', padding: '0.75rem 1rem', background: '#f3f4f6', borderRadius: 8 }}>
-          <p style={{ fontSize: '0.72rem', color: '#9ca3af', margin: '0 0 0.25rem' }}>YOU</p>
-          <p style={{ margin: 0, fontSize: '0.9rem' }}>{transcript}</p>
-        </div>
+        <p style={{ marginTop: '0.5rem', color: '#dc2626', fontSize: '0.8rem' }}>⚠ {error}</p>
       )}
 
-      {reply && (
-        <div style={{ marginTop: '0.75rem', padding: '0.75rem 1rem', background: '#eff6ff', borderRadius: 8 }}>
-          <p style={{ fontSize: '0.72rem', color: '#93c5fd', margin: '0 0 0.25rem' }}>AGENT</p>
-          <p style={{ margin: 0, fontSize: '0.9rem' }}>{reply}</p>
-        </div>
+      {log.length > 0 && (
+        <ul style={{ marginTop: '1.5rem', fontSize: '0.8rem', color: '#374151', paddingLeft: '1.2rem' }}>
+          {log.map((entry, i) => <li key={i}>{entry}</li>)}
+        </ul>
       )}
     </main>
   );