feat: add basic audio support with voice recording and TTS

.eslintrc.json

@@ -8,7 +8,10 @@
   "rules": {
     "no-relative-import-paths/no-relative-import-paths": [
       "warn",
-      { "allowSameFolder": true, "prefix": "@" }
+      {
+        "allowSameFolder": true,
+        "prefix": "@"
+      }
     ]
   }
 }

app/(with-header)/share/page.tsx

@@ -27,14 +27,20 @@ export async function generateMetadata({ searchParams }: Props) {
   const snapshot = await getSnapshot(snapshot_id);
 
   if (snapshot.party_ids.length > 1) {
-    return;
+    return {};
   }
 
   const partyId = snapshot.party_ids[0];
 
+  const imageUrl = await generateOgImageUrl(partyId);
+
+  if (!imageUrl) {
+    return {};
+  }
+
   return {
     openGraph: {
-      images: [await generateOgImageUrl(partyId)],
+      images: [imageUrl],
     },
   };
 }

app/session/page.tsx

@@ -8,6 +8,7 @@ type Props = {
     session_id?: string;
     party_id: string[] | string | undefined;
     q?: string;
+    voice?: string;
   }>;
 };
 
@@ -25,20 +26,26 @@ export async function generateMetadata({
     !party_id ||
     (Array.isArray(party_id) && (party_id.length === 0 || party_id.length > 1))
   ) {
-    return;
+    return {};
   }
 
   const partyId = Array.isArray(party_id) ? party_id[0] : party_id;
 
+  const imageUrl = await generateOgImageUrl(partyId);
+
+  if (!imageUrl) {
+    return {};
+  }
+
   return {
     openGraph: {
-      images: [await generateOgImageUrl(partyId)],
+      images: [imageUrl],
     },
   };
 }
 
 async function Page({ searchParams }: Props) {
-  const { party_id, q, session_id } = await searchParams;
+  const { party_id, q, session_id, voice } = await searchParams;
   const parties = await getParties();
 
   if (session_id) {
@@ -55,7 +62,13 @@ async function Page({ searchParams }: Props) {
     parties.some((p) => p.party_id === id),
   );
 
-  return <ChatView partyIds={normalizedPartyIds} initialQuestion={q} />;
+  return (
+    <ChatView
+      partyIds={normalizedPartyIds}
+      initialQuestion={q}
+      hasPendingVoiceMessage={voice === '1'}
+    />
+  );
 }
 
 export default Page;

components/chat/chat-grouped-messages.tsx

@@ -41,6 +41,7 @@ function ChatGroupedMessages({ message, isLastMessage, parties }: Props) {
           (p) => p.party_id === message.messages[0].party_id,
         )}
         isLastMessage={isLastMessage}
+        voiceTranscriptionStatus={message.voice_transcription}
       />
     );
   }
@@ -66,14 +67,14 @@ function ChatGroupedMessages({ message, isLastMessage, parties }: Props) {
       plugins={[AutoHeight()]}
     >
       <CarouselContent>
-        {messagePartiesDict?.map(({ message, party }) => {
+        {messagePartiesDict?.map(({ message: innerMessage, party }) => {
           return (
-            <CarouselItem key={message.id}>
+            <CarouselItem key={innerMessage.id}>
               <div className="p-4">
                 <ChatSingleMessage
-                  message={message}
+                  message={innerMessage}
                   party={party}
-                  partyId={message.party_id}
+                  partyId={innerMessage.party_id}
                   showAssistantIcon={true}
                   isGroupChat
                 />

components/chat/chat-input.tsx

@@ -8,12 +8,18 @@ import { ArrowUp } from 'lucide-react';
 import { useCallback } from 'react';
 import ChatInputAddPartiesButton from './chat-input-add-parties-button';
 import MessageLoadingBorderTrail from './message-loading-border-trail';
+import {
+  VoiceRecordButton,
+  VoiceRecordingIndicator,
+  useVoiceRecordButton,
+} from './voice-record-button';
 
 function ChatInput() {
   const { user } = useAnonymousAuth();
   const input = useChatStore((state) => state.input);
   const setInput = useChatStore((state) => state.setInput);
   const addUserMessage = useChatStore((state) => state.addUserMessage);
+  const sendVoiceMessage = useChatStore((state) => state.sendVoiceMessage);
   const quickReplies = useChatStore((state) => state.currentQuickReplies);
   const loading = useChatStore((state) => {
     const loading = state.loading;
@@ -25,6 +31,12 @@ function ChatInput() {
     );
   });
 
+  const { isRecording, handleStartRecording, handleStopRecording } =
+    useVoiceRecordButton(sendVoiceMessage);
+
+  const showMicButton = input.length === 0 && !isRecording;
+  const showSendButton = input.length > 0 && !isRecording;
+
   const handleSubmit = useCallback(
     async (e: React.FormEvent<HTMLFormElement> | string) => {
       let effectiveInput = input;
@@ -58,7 +70,7 @@ function ChatInput() {
         quickReplies?.length > 0 && 'rounded-[20px]',
       )}
     >
-      {quickReplies.length > 0 && (
+      {quickReplies.length > 0 && !isRecording && (
         <>
           <ChatInputAddPartiesButton disabled={loading} />
           <div
@@ -84,24 +96,42 @@ function ChatInput() {
 
       {loading && <MessageLoadingBorderTrail />}
 
-      <input
-        className="w-full bg-chat-input py-3 pl-4 pr-11 text-[16px] placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-0 disabled:cursor-not-allowed"
-        placeholder="Schreibe eine Nachricht..."
-        onChange={handleChange}
-        value={input}
-        disabled={loading}
-        maxLength={500}
-      />
-      <Button
-        type="submit"
-        disabled={!input.length || loading}
-        className={cn(
-          'absolute right-2 top-1/2 flex size-8 -translate-y-1/2 items-center justify-center rounded-full bg-foreground text-background transition-colors hover:bg-foreground/80 disabled:bg-foreground/20 disabled:text-muted',
-          quickReplies.length > 0 && 'bottom-0 translate-y-0',
-        )}
-      >
-        <ArrowUp className="size-4 font-bold" />
-      </Button>
+      {isRecording ? (
+        <VoiceRecordingIndicator onStop={handleStopRecording} />
+      ) : (
+        <>
+          <input
+            className="w-full bg-chat-input py-3 pl-4 pr-11 text-[16px] placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-0 disabled:cursor-not-allowed"
+            placeholder="Schreibe eine Nachricht..."
+            onChange={handleChange}
+            value={input}
+            disabled={loading}
+            maxLength={500}
+          />
+          {showSendButton && (
+            <Button
+              type="submit"
+              disabled={!input.length || loading}
+              className={cn(
+                'absolute right-2 top-1/2 flex size-8 -translate-y-1/2 items-center justify-center rounded-full bg-foreground text-background transition-colors hover:bg-foreground/80 disabled:bg-foreground/20 disabled:text-muted',
+                quickReplies.length > 0 && 'bottom-0 translate-y-0',
+              )}
+            >
+              <ArrowUp className="size-4 font-bold" />
+            </Button>
+          )}
+          {showMicButton && (
+            <VoiceRecordButton
+              onClick={handleStartRecording}
+              disabled={loading}
+              className={cn(
+                'absolute right-2 top-1/2 -translate-y-1/2',
+                quickReplies.length > 0 && 'bottom-0 translate-y-0',
+              )}
+            />
+          )}
+        </>
+      )}
     </form>
   );
 }

components/chat/chat-messages-view.tsx

@@ -1,6 +1,7 @@
 'use client';
 
 import { useAnonymousAuth } from '@/components/anonymous-auth';
+import { PENDING_VOICE_MESSAGE_KEY } from '@/components/home/home-input';
 import { useChatStore } from '@/components/providers/chat-store-provider';
 import { useTenant } from '@/components/providers/tenant-provider';
 import type {
@@ -24,6 +25,7 @@ type Props = {
   allParties?: PartyDetails[];
   proposedQuestions?: ProposedQuestion[];
   initialQuestion?: string;
+  hasPendingVoiceMessage?: boolean;
 };
 
 function ChatMessagesView({
@@ -34,16 +36,20 @@ function ChatMessagesView({
   allParties,
   proposedQuestions,
   initialQuestion,
+  hasPendingVoiceMessage,
 }: Props) {
   const hasFetched = useRef(false);
+  const hasProcessedVoiceMessage = useRef(false);
   const storeMessages = useChatStore((state) => state.messages);
   const hydrateChatSession = useChatStore((state) => state.hydrateChatSession);
+  const sendVoiceMessage = useChatStore((state) => state.sendVoiceMessage);
   const { user } = useAnonymousAuth();
   const tenant = useTenant();
 
   const hasCurrentStreamingMessages = useChatStore(
     (state) => state.currentStreamingMessages !== undefined,
   );
+  const isSocketConnected = useChatStore((state) => state.socket.connected);
 
   useEffect(() => {
     if (!user?.uid) return;
@@ -70,6 +76,31 @@ function ChatMessagesView({
     tenant,
   ]);
 
+  // Handle pending voice message from home page
+  useEffect(() => {
+    if (
+      !hasPendingVoiceMessage ||
+      hasProcessedVoiceMessage.current ||
+      !isSocketConnected
+    )
+      return;
+
+    const pendingAudioBase64 = sessionStorage.getItem(
+      PENDING_VOICE_MESSAGE_KEY,
+    );
+    if (pendingAudioBase64) {
+      sessionStorage.removeItem(PENDING_VOICE_MESSAGE_KEY);
+      hasProcessedVoiceMessage.current = true;
+      // Convert base64 back to Uint8Array
+      const binaryString = atob(pendingAudioBase64);
+      const audioBytes = new Uint8Array(binaryString.length);
+      for (let i = 0; i < binaryString.length; i++) {
+        audioBytes[i] = binaryString.charCodeAt(i);
+      }
+      sendVoiceMessage(audioBytes);
+    }
+  }, [hasPendingVoiceMessage, sendVoiceMessage, isSocketConnected]);
+
   const normalizedMessages = useMemo(() => {
     if (messages && !storeMessages.length) {
       return messages;

components/chat/chat-single-message-actions.tsx

@@ -5,6 +5,7 @@ import type { StreamingMessage } from '@/lib/socket.types';
 import type { MessageItem } from '@/lib/stores/chat-store.types';
 import ChatMessageLikeDislikeButtons from './chat-message-like-dislike-buttons';
 import ChatProConButton from './chat-pro-con-button';
+import ChatTtsButton from './chat-tts-button';
 import ChatVotingBehaviorSummaryButton from './chat-voting-behavior-summary-button';
 import CopyButton from './copy-button';
 import SourcesButton from './sources-button';
@@ -84,6 +85,7 @@ function ChatSingleMessageActions({
           size="icon"
           className="size-8"
         />
+        {partyId && <ChatTtsButton partyId={partyId} messageId={message.id} />}
         <ChatMessageLikeDislikeButtons message={message} />
       </div>
     </div>

components/chat/chat-single-message.tsx

@@ -1,6 +1,9 @@
 import { useChatStore } from '@/components/providers/chat-store-provider';
 import type { PartyDetails } from '@/lib/party-details';
-import type { MessageItem } from '@/lib/stores/chat-store.types';
+import type {
+  MessageItem,
+  VoiceTranscriptionStatus,
+} from '@/lib/stores/chat-store.types';
 import { cn } from '@/lib/utils';
 import ChatMarkdown from './chat-markdown';
 import { ChatMessageIcon } from './chat-message-icon';
@@ -19,6 +22,7 @@ type Props = {
   showAssistantIcon?: boolean;
   showMessageActions?: boolean;
   isGroupChat?: boolean;
+  voiceTranscriptionStatus?: VoiceTranscriptionStatus;
 };
 
 function ChatSingleMessage({
@@ -29,6 +33,7 @@ function ChatSingleMessage({
   showAssistantIcon = true,
   showMessageActions = true,
   isGroupChat = false,
+  voiceTranscriptionStatus,
 }: Props) {
   const isLoadingAnyAction = useChatStore(
     (state) =>
@@ -52,6 +57,7 @@ function ChatSingleMessage({
       <ChatSingleUserMessage
         message={message}
         isLastMessage={isLastMessage ?? false}
+        voiceTranscriptionStatus={voiceTranscriptionStatus}
       />
     );
   }