feat(chat): improve media handling and caching for user messages (#81)

Author: hazeone

electron/gateway/manager.ts

@@ -656,7 +656,7 @@ export class GatewayManager extends EventEmitter {
   /**
    * Wait for Gateway to be ready by checking if the port is accepting connections
    */
-  private async waitForReady(retries = 120, interval = 1000): Promise<void> {
+  private async waitForReady(retries = 600, interval = 1000): Promise<void> {
     for (let i = 0; i < retries; i++) {
       // Early exit if the gateway process has already exited
       if (this.process && (this.process.exitCode !== null || this.process.signalCode !== null)) {

electron/main/ipc-handlers.ts

@@ -441,27 +441,38 @@ function registerGatewayHandlers(
   }) => {
     try {
       let message = params.message;
-      const imageAttachments: Array<{ type: string; mimeType: string; fileName: string; content: string }> = [];
+      // The Gateway processes image attachments through TWO parallel paths:
+      // Path A: `attachments` param → parsed via `parseMessageWithAttachments` →
+      //   injected as inline vision content when the model supports images.
+      //   Format: { content: base64, mimeType: string, fileName?: string }
+      // Path B: `[media attached: ...]` in message text → Gateway's native image
+      //   detection (`detectAndLoadPromptImages`) reads the file from disk and
+      //   injects it as inline vision content. Also works for history messages.
+      // We use BOTH paths for maximum reliability.
+      const imageAttachments: Array<Record<string, unknown>> = [];
       const fileReferences: string[] = [];
 
       if (params.media && params.media.length > 0) {
         for (const m of params.media) {
           logger.info(`[chat:sendWithMedia] Processing file: ${m.fileName} (${m.mimeType}), path: ${m.filePath}, exists: ${existsSync(m.filePath)}, isVision: ${VISION_MIME_TYPES.has(m.mimeType)}`);
+
+          // Always add file path reference so the model can access it via tools
+          fileReferences.push(
+            `[media attached: ${m.filePath} (${m.mimeType}) | ${m.filePath}]`,
+          );
+
           if (VISION_MIME_TYPES.has(m.mimeType)) {
-            // Raster image — inline as base64 vision attachment
+            // Send as base64 attachment in the format the Gateway expects:
+            // { content: base64String, mimeType: string, fileName?: string }
+            // The Gateway normalizer looks for `a.content` (NOT `a.source.data`).
             const fileBuffer = readFileSync(m.filePath);
-            logger.info(`[chat:sendWithMedia] Read ${fileBuffer.length} bytes, base64 length: ${fileBuffer.toString('base64').length}`);
+            const base64Data = fileBuffer.toString('base64');
+            logger.info(`[chat:sendWithMedia] Read ${fileBuffer.length} bytes, base64 length: ${base64Data.length}`);
             imageAttachments.push({
-              type: 'image',
+              content: base64Data,
               mimeType: m.mimeType,
               fileName: m.fileName,
-              content: fileBuffer.toString('base64'),
             });
-          } else {
-            // Non-vision file — reference by path (same format as channel inbound media)
-            fileReferences.push(
-              `[media attached: ${m.filePath} (${m.mimeType}) | ${m.filePath}]`,
-            );
           }
         }
       }
@@ -483,9 +494,9 @@ function registerGatewayHandlers(
         rpcParams.attachments = imageAttachments;
       }
 
-      logger.info(`[chat:sendWithMedia] Sending: message="${message.substring(0, 100)}", imageAttachments=${imageAttachments.length}, fileRefs=${fileReferences.length}`);
+      logger.info(`[chat:sendWithMedia] Sending: message="${message.substring(0, 100)}", attachments=${imageAttachments.length}, fileRefs=${fileReferences.length}`);
 
-      // Use a longer timeout when attachments are present (120s vs default 30s)
+      // Use a longer timeout when images are present (120s vs default 30s)
       const timeoutMs = imageAttachments.length > 0 ? 120000 : 30000;
       const result = await gatewayManager.rpc('chat.send', rpcParams, timeoutMs);
       logger.info(`[chat:sendWithMedia] RPC result: ${JSON.stringify(result)}`);
@@ -1557,4 +1568,26 @@ function registerFileHandlers(): void {
 
     return { id, fileName: payload.fileName, mimeType, fileSize, stagedPath, preview };
   });
+
+  // Load thumbnails for file paths on disk (used to restore previews in history)
+  ipcMain.handle('media:getThumbnails', async (_, paths: Array<{ filePath: string; mimeType: string }>) => {
+    const results: Record<string, { preview: string | null; fileSize: number }> = {};
+    for (const { filePath, mimeType } of paths) {
+      try {
+        if (!existsSync(filePath)) {
+          results[filePath] = { preview: null, fileSize: 0 };
+          continue;
+        }
+        const stat = statSync(filePath);
+        let preview: string | null = null;
+        if (mimeType.startsWith('image/')) {
+          preview = generateImagePreview(filePath, mimeType);
+        }
+        results[filePath] = { preview, fileSize: stat.size };
+      } catch {
+        results[filePath] = { preview: null, fileSize: 0 };
+      }
+    }
+    return results;
+  });
 }

electron/preload/index.ts

@@ -115,9 +115,10 @@ const electronAPI = {
         'log:getFilePath',
         'log:getDir',
         'log:listFiles',
-        // File staging
+        // File staging & media
         'file:stage',
         'file:stageBuffer',
+        'media:getThumbnails',
         // Chat send with media (reads staged files in main process)
         'chat:sendWithMedia',
         // OpenClaw extras

src/pages/Chat/ChatMessage.tsx

@@ -105,35 +105,41 @@ export const ChatMessage = memo(function ChatMessage({
           />
         )}
 
-        {/* Images (from assistant/channel content blocks) */}
+        {/* Images from content blocks (Gateway session data — persists across history reloads) */}
         {images.length > 0 && (
           <div className="flex flex-wrap gap-2">
             {images.map((img, i) => (
               <img
-                key={i}
+                key={`content-${i}`}
                 src={`data:${img.mimeType};base64,${img.data}`}
                 alt="attachment"
-                className="max-w-xs rounded-lg border"
+                className={cn(
+                  'rounded-lg border',
+                  isUser ? 'max-w-[200px] max-h-48' : 'max-w-xs',
+                )}
               />
             ))}
           </div>
         )}
 
-        {/* File attachments (user-uploaded files) */}
+        {/* File attachments (local preview — shown before history reload) */}
+        {/* Only show _attachedFiles images if no content-block images (avoid duplicates) */}
         {attachedFiles.length > 0 && (
           <div className="flex flex-wrap gap-2">
-            {attachedFiles.map((file, i) => (
-              file.mimeType.startsWith('image/') && file.preview ? (
+            {attachedFiles.map((file, i) => {
+              // Skip image attachments if we already have images from content blocks
+              if (file.mimeType.startsWith('image/') && file.preview && images.length > 0) return null;
+              return file.mimeType.startsWith('image/') && file.preview ? (
                 <img
-                  key={i}
+                  key={`local-${i}`}
                   src={file.preview}
                   alt={file.fileName}
-                  className="max-w-xs max-h-48 rounded-lg border"
+                  className="max-w-[200px] max-h-48 rounded-lg border"
                 />
               ) : (
-                <FileCard key={i} file={file} />
-              )
-            ))}
+                <FileCard key={`local-${i}`} file={file} />
+              );
+            })}
           </div>
         )}
       </div>

src/pages/Chat/message-utils.ts

@@ -5,20 +5,38 @@
  */
 import type { RawMessage, ContentBlock } from '@/stores/chat';
 
+/**
+ * Clean Gateway metadata from user message text for display.
+ * Strips: [media attached: ... | ...], [message_id: ...],
+ * and the timestamp prefix [Day Date Time Timezone].
+ */
+function cleanUserText(text: string): string {
+  return text
+    // Remove [media attached: path (mime) | path] references
+    .replace(/\s*\[media attached:[^\]]*\]/g, '')
+    // Remove [message_id: uuid]
+    .replace(/\s*\[message_id:\s*[^\]]+\]/g, '')
+    // Remove Gateway timestamp prefix like [Fri 2026-02-13 22:39 GMT+8]
+    .replace(/^\[(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+[^\]]+\]\s*/i, '')
+    .trim();
+}
+
 /**
  * Extract displayable text from a message's content field.
  * Handles both string content and array-of-blocks content.
+ * For user messages, strips Gateway-injected metadata.
  */
 export function extractText(message: RawMessage | unknown): string {
   if (!message || typeof message !== 'object') return '';
   const msg = message as Record<string, unknown>;
   const content = msg.content;
+  const isUser = msg.role === 'user';
 
-  if (typeof content === 'string') {
-    return content.trim().length > 0 ? content : '';
-  }
+  let result = '';
 
-  if (Array.isArray(content)) {
+  if (typeof content === 'string') {
+    result = content.trim().length > 0 ? content : '';
+  } else if (Array.isArray(content)) {
     const parts: string[] = [];
     for (const block of content as ContentBlock[]) {
       if (block.type === 'text' && block.text) {
@@ -28,15 +46,18 @@ export function extractText(message: RawMessage | unknown): string {
       }
     }
     const combined = parts.join('\n\n');
-    return combined.trim().length > 0 ? combined : '';
+    result = combined.trim().length > 0 ? combined : '';
+  } else if (typeof msg.text === 'string') {
+    // Fallback: try .text field
+    result = msg.text.trim().length > 0 ? msg.text : '';
   }
 
-  // Fallback: try .text field
-  if (typeof msg.text === 'string') {
-    return msg.text.trim().length > 0 ? msg.text : '';
+  // Strip Gateway metadata from user messages for clean display
+  if (isUser && result) {
+    result = cleanUserText(result);
   }
 
-  return '';
+  return result;
 }
 
 /**
@@ -64,6 +85,35 @@ export function extractThinking(message: RawMessage | unknown): string | null {
   return combined.length > 0 ? combined : null;
 }
 
+/**
+ * Extract media file references from Gateway-formatted user message text.
+ * Returns array of { filePath, mimeType } from [media attached: path (mime) | path] patterns.
+ */
+export function extractMediaRefs(message: RawMessage | unknown): Array<{ filePath: string; mimeType: string }> {
+  if (!message || typeof message !== 'object') return [];
+  const msg = message as Record<string, unknown>;
+  if (msg.role !== 'user') return [];
+  const content = msg.content;
+
+  let text = '';
+  if (typeof content === 'string') {
+    text = content;
+  } else if (Array.isArray(content)) {
+    text = (content as ContentBlock[])
+      .filter(b => b.type === 'text' && b.text)
+      .map(b => b.text!)
+      .join('\n');
+  }
+
+  const refs: Array<{ filePath: string; mimeType: string }> = [];
+  const regex = /\[media attached:\s*([^\s(]+)\s*\(([^)]+)\)\s*\|[^\]]*\]/g;
+  let match;
+  while ((match = regex.exec(text)) !== null) {
+    refs.push({ filePath: match[1], mimeType: match[2] });
+  }
+  return refs;
+}
+
 /**
  * Extract image attachments from a message.
  * Returns array of { mimeType, data } for base64 images.

src/pages/Setup/index.tsx

@@ -519,7 +519,7 @@ function RuntimeContent({ onStatusChange }: RuntimeContentProps) {
         }
         return prev;
       });
-    }, 120 * 1000); // 120 seconds — enough for gateway to fully initialize
+    }, 600 * 1000); // 600 seconds — enough for gateway to fully initialize
 
     return () => {
       if (gatewayTimeoutRef.current) {