Show browser screenshots in the chat transcript

src/main/json-claude-attachments.ts

@@ -13,7 +13,7 @@
 import { writeFileSync, mkdirSync, existsSync, readFileSync, statSync } from 'fs'
 import { tmpdir } from 'os'
 import { join } from 'path'
-import { randomUUID } from 'crypto'
+import { createHash, randomUUID } from 'crypto'
 
 const ATTACHMENT_DIR = join(tmpdir(), 'harness-attachments')
 
@@ -59,3 +59,21 @@ export function writeAttachmentImage(
   writeFileSync(path, Buffer.from(base64Data, 'base64'), { mode: 0o600 })
   return path
 }
+
+/** Same as writeAttachmentImage but keyed by content hash, so extracting
+ *  the same image twice yields the same path and writes once. Tool-result
+ *  images (browser screenshots) need this: resuming a session replays the
+ *  whole transcript through the extractor, and uuid names would leak a
+ *  fresh copy of every screenshot on every resume. */
+export function writeResultImage(base64Data: string, mediaType: string): string {
+  if (!existsSync(ATTACHMENT_DIR)) {
+    mkdirSync(ATTACHMENT_DIR, { recursive: true, mode: 0o700 })
+  }
+  const ext = EXT_BY_MEDIA_TYPE[mediaType.toLowerCase()] || 'bin'
+  const hash = createHash('sha256').update(base64Data).digest('hex').slice(0, 32)
+  const path = join(ATTACHMENT_DIR, `result-${hash}.${ext}`)
+  if (!existsSync(path)) {
+    writeFileSync(path, Buffer.from(base64Data, 'base64'), { mode: 0o600 })
+  }
+  return path
+}

src/main/json-claude-manager-fork.test.ts

@@ -39,6 +39,7 @@ vi.mock('child_process', () => ({
 
 import { Store } from './store'
 import { JsonClaudeManager } from './json-claude-manager'
+import type { JsonClaudeMessageBlock } from '../shared/state/json-claude'
 
 function transcriptDir(worktreePath: string): string {
   return join(tmpHome, '.claude', 'projects', worktreePath.replace(/[^a-zA-Z0-9]/g, '-'))
@@ -366,3 +367,113 @@ describe('JsonClaudeManager.seedFromTranscript — mid-turn messages', () => {
     expect(entries[0].text).toBe('hello')
   })
 })
+
+// Browser-screenshot tool results arrive as Anthropic-shaped image
+// blocks. The extractor spills them to disk and keeps a path, so the
+// renderer can show a thumbnail without megabytes of base64 riding
+// through every state event.
+describe('JsonClaudeManager.seedFromTranscript — tool_result images', () => {
+  // 1x1 red JPEG.
+  const JPEG_B64 =
+    '/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAABAAEBAREA/8QAFAABAAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8AKp//2Q=='
+  const written: string[] = []
+
+  beforeEach(() => {
+    tmpHome = mkdtempSync(join(tmpdir(), 'harness-seed-img-'))
+  })
+
+  afterEach(() => {
+    rmSync(tmpHome, { recursive: true, force: true })
+    for (const p of written) rmSync(p, { force: true })
+    written.length = 0
+    vi.clearAllMocks()
+  })
+
+  function seedWithScreenshot(sessionId: string, worktree: string): JsonClaudeMessageBlock {
+    const dir = transcriptDir(worktree)
+    mkdirSync(dir, { recursive: true })
+    writeFileSync(
+      join(dir, `${sessionId}.jsonl`),
+      [
+        { type: 'user', sessionId, message: { content: 'shot it' } },
+        {
+          type: 'assistant',
+          sessionId,
+          message: {
+            id: 'msg_a',
+            content: [
+              {
+                type: 'tool_use',
+                id: 'tu-shot',
+                name: 'mcp__ness-control__screenshot_tab'
+              }
+            ]
+          }
+        },
+        {
+          type: 'user',
+          sessionId,
+          message: {
+            content: [
+              {
+                type: 'tool_result',
+                tool_use_id: 'tu-shot',
+                content: [
+                  { type: 'text', text: 'took a shot' },
+                  {
+                    type: 'image',
+                    source: { type: 'base64', media_type: 'image/jpeg', data: JPEG_B64 }
+                  }
+                ]
+              }
+            ]
+          }
+        }
+      ]
+        .map((l) => JSON.stringify(l))
+        .join('\n') + '\n',
+      'utf8'
+    )
+    const store = new Store()
+    store.dispatch({
+      type: 'jsonClaude/sessionStarted',
+      payload: { sessionId, worktreePath: worktree }
+    })
+    makeManager(store).seedFromTranscript(sessionId, worktree)
+    const entries = store.getSnapshot().state.jsonClaude.sessions[sessionId].entries
+    const resultEntry = entries.find((e) => e.kind === 'tool_result')!
+    const block = resultEntry.blocks![0]
+    for (const img of block.images ?? []) written.push(img.path)
+    return block
+  }
+
+  it('spills the image to disk and keeps the text separate', () => {
+    const block = seedWithScreenshot(
+      '77777777-7777-7777-7777-777777777777',
+      '/tmp/wt-seed-shot'
+    )
+
+    expect(block.images).toHaveLength(1)
+    expect(block.images![0].mediaType).toBe('image/jpeg')
+    expect(existsSync(block.images![0].path)).toBe(true)
+    // The base64 never lands in state — only the path.
+    expect(block.content).toBe('took a shot')
+    expect(JSON.stringify(block)).not.toContain(JPEG_B64)
+    // The bytes on disk round-trip.
+    expect(readFileSync(block.images![0].path).toString('base64')).toBe(JPEG_B64)
+  })
+
+  it('reuses one file when the same image is extracted twice', () => {
+    // Resuming a session replays the whole transcript through the
+    // extractor; uuid-named files would leak a copy on every resume.
+    const first = seedWithScreenshot(
+      '88888888-8888-8888-8888-888888888888',
+      '/tmp/wt-seed-shot-a'
+    )
+    const second = seedWithScreenshot(
+      '99999999-9999-9999-9999-999999999999',
+      '/tmp/wt-seed-shot-b'
+    )
+    expect(second.images![0].path).toBe(first.images![0].path)
+  })
+})

src/main/json-claude-manager.ts

@@ -26,11 +26,13 @@ import { isPackaged, resolveBundledMcpScript } from './paths'
 import type { Store } from './store'
 import type {
   JsonClaudeChatEntry,
+  JsonClaudeImageRef,
   JsonClaudeMessageBlock,
   JsonClaudePermissionMode,
   JsonClaudeSessionState
 } from '../shared/state/json-claude'
 import { parseAutomatedMessage } from '../shared/state/json-claude'
+import { writeResultImage } from './json-claude-attachments'
 import type { ClaudeLaunchSettings } from './claude-launch'
 import { log } from './debug'
 import { shellQuote } from './shell-quote'
@@ -475,7 +477,8 @@ export class JsonClaudeManager {
                   type: 'tool_result',
                   toolUseId: r.toolUseId,
                   content: r.content,
-                  isError: r.isError
+                  isError: r.isError,
+                  ...(r.images ? { images: r.images } : {})
                 }
               ]
             })
@@ -1714,7 +1717,8 @@ export class JsonClaudeManager {
             sessionId: instance.sessionId,
             toolUseId: r.toolUseId,
             content: r.content,
-            isError: r.isError
+            isError: r.isError,
+            ...(r.images ? { images: r.images } : {})
           }
         })
       }
@@ -2277,43 +2281,75 @@ function extractAssistantBlocks(ev: Record<string, unknown>): JsonClaudeMessageB
   return out
 }
 
-function extractToolResults(
-  ev: Record<string, unknown>
-): Array<{ toolUseId: string; content: string; isError: boolean }> {
+interface ExtractedToolResult {
+  toolUseId: string
+  content: string
+  isError: boolean
+  images?: JsonClaudeImageRef[]
+}
+
+function extractToolResults(ev: Record<string, unknown>): ExtractedToolResult[] {
   const message = ev['message'] as { content?: unknown } | undefined
   const content = message?.content
   if (!Array.isArray(content)) return []
   return extractToolResultsFromArray(content)
 }
 
-function extractToolResultsFromArray(
-  content: unknown[]
-): Array<{ toolUseId: string; content: string; isError: boolean }> {
-  const out: Array<{ toolUseId: string; content: string; isError: boolean }> = []
+/** MCP tools that return images (browser screenshots) surface them as
+ *  Anthropic-shaped blocks: {type:'image', source:{type:'base64',
+ *  media_type, data}}. Spill the bytes to a temp file and keep only the
+ *  path — a PNG screenshot is megabytes of base64, and everything in a
+ *  state event gets re-broadcast to every connected client. */
+function extractResultImage(part: Record<string, unknown>): JsonClaudeImageRef | null {
+  if (part['type'] !== 'image') return null
+  const source = part['source']
+  if (!source || typeof source !== 'object') return null
+  const s = source as Record<string, unknown>
+  const data = s['data']
+  const mediaType = s['media_type']
+  if (typeof data !== 'string' || !data) return null
+  if (typeof mediaType !== 'string' || !mediaType.startsWith('image/')) return null
+  try {
+    return { path: writeResultImage(data, mediaType), mediaType }
+  } catch {
+    return null
+  }
+}
+
+function extractToolResultsFromArray(content: unknown[]): ExtractedToolResult[] {
+  const out: ExtractedToolResult[] = []
   for (const raw of content) {
     if (!raw || typeof raw !== 'object') continue
     const b = raw as Record<string, unknown>
     if (b['type'] !== 'tool_result') continue
     const id = typeof b['tool_use_id'] === 'string' ? (b['tool_use_id'] as string) : ''
     if (!id) continue
     const rawContent = b['content']
+    const images: JsonClaudeImageRef[] = []
     const text =
       typeof rawContent === 'string'
         ? rawContent
         : Array.isArray(rawContent)
           ? rawContent
               .map((p) => {
-                if (typeof p === 'object' && p && 'text' in (p as Record<string, unknown>)) {
-                  return String((p as Record<string, unknown>)['text'])
+                if (!p || typeof p !== 'object') return ''
+                const part = p as Record<string, unknown>
+                const img = extractResultImage(part)
+                if (img) {
+                  images.push(img)
+                  return ''
                 }
+                if ('text' in part) return String(part['text'])
                 return ''
               })
+              .filter((s) => s !== '')
               .join('\n')
           : JSON.stringify(rawContent)
     out.push({
       toolUseId: id,
       content: text,
-      isError: Boolean(b['is_error'])
+      isError: Boolean(b['is_error']),
+      ...(images.length > 0 ? { images } : {})
     })
   }
   return out

src/renderer/components/JsonModeChat.tsx

@@ -40,7 +40,7 @@ import { useJsonClaudeApprovals } from '../hooks/useJsonClaudeApprovals'
 import { JsonClaudeApprovalCard } from './JsonClaudeApprovalCard'
 import { JsonClaudeQuestionCard } from './JsonClaudeQuestionCard'
 import { Tooltip } from './Tooltip'
-import { dispatchToolCard, ToolCardChrome } from './json-mode-cards'
+import { dispatchToolCard, ToolCardChrome, type ToolResultView } from './json-mode-cards'
 import { NessIcon } from './json-mode-cards/tool-icons'
 import { ToolGroup } from './json-mode-cards/ToolGroup'
 import { TaskCard } from './json-mode-cards/TaskCard'
@@ -292,6 +292,10 @@ interface RenderedRow {
   toolName?: string
   hasError?: boolean
   hasPendingApproval?: boolean
+  /** This row's tool returned an image (a browser screenshot). Bubbles
+   *  up to ToolGroup so the group opens far enough to show it — a
+   *  screenshot behind two collapsed chevrons may as well not be there. */
+  hasImages?: boolean
   /** Marks this row as a thinking card. Lives in the 'tool' bucket so
    *  it groups with adjacent tool_use rows (thinking + tools are both
    *  agent work between user-facing replies), but ToolGroup counts it
@@ -855,7 +859,7 @@ function AutomatedTurnCard({
 }
 
 interface RenderContext {
-  resultsByToolUseId: Map<string, { content: string; isError: boolean }>
+  resultsByToolUseId: Map<string, ToolResultView>
   childrenByParentToolUseId: Map<string, JsonClaudeChatEntry[]>
   approvalCard: (toolUseId: string | undefined) => ReactNode
   pendingToolUseIds: Set<string>
@@ -1129,6 +1133,7 @@ function renderEntries(
             type: 'tool',
             toolName: block.name,
             hasError: !!result?.isError,
+            hasImages: !!result?.images && result.images.length > 0,
             hasPendingApproval:
               (!!block.id && ctx.pendingToolUseIds.has(block.id)) ||
               subAgentDescendantHasPendingApproval,
@@ -1681,17 +1686,15 @@ export function JsonModeChat({ sessionId, worktreePath, mode = 'awake' }: JsonMo
     // tool_use_id → tool_result lookup built once over the full
     // entries array (results live in top-level tool_result entries
     // even when their corresponding tool_use was a sub-agent's call).
-    const resultsByToolUseId = new Map<
-      string,
-      { content: string; isError: boolean }
-    >()
+    const resultsByToolUseId = new Map<string, ToolResultView>()
     for (const entry of deferredEntries) {
       if (entry.kind !== 'tool_result' || !entry.blocks) continue
       for (const b of entry.blocks) {
         if (b.type === 'tool_result' && b.toolUseId) {
           resultsByToolUseId.set(b.toolUseId, {
             content: b.content || '',
-            isError: !!b.isError
+            isError: !!b.isError,
+            ...(b.images && b.images.length > 0 ? { images: b.images } : {})
           })
         }
       }

src/renderer/components/JsonModeChatImageThumb.tsx

@@ -32,9 +32,18 @@ function fetchImage(path: string, mediaType: string): Promise<string | null> {
 interface Props {
   path: string
   mediaType: string
+  /** 'square' crops to a 64px tile — right for pasted attachments, where
+   *  the thumbnail is an affordance rather than something to read.
+   *  'wide' keeps the aspect ratio at 128px tall, for browser
+   *  screenshots where a centre-crop would throw away the page. */
+  shape?: 'square' | 'wide'
 }
 
-export function JsonModeChatImageThumb({ path, mediaType }: Props): JSX.Element {
+export function JsonModeChatImageThumb({
+  path,
+  mediaType,
+  shape = 'square'
+}: Props): JSX.Element {
   const [dataUrl, setDataUrl] = useState<string | null>(
     CACHE.has(path) ? CACHE.get(path)! : null
   )
@@ -69,19 +78,24 @@ export function JsonModeChatImageThumb({ path, mediaType }: Props): JSX.Element
   }, [showFull])
 
   const name = path.split('/').pop() || path
+  const boxClass = shape === 'wide' ? 'h-32 w-48' : 'h-16 w-16'
+  const imgClass =
+    shape === 'wide'
+      ? 'h-32 w-auto max-w-full object-contain bg-app'
+      : 'h-16 w-16 object-cover'
 
   if (pending) {
     return (
       <div
-        className="h-16 w-16 rounded bg-panel border border-border animate-pulse"
+        className={`${boxClass} rounded bg-panel border border-border animate-pulse`}
         title={path}
       />
     )
   }
   if (!dataUrl) {
     return (
       <div
-        className="h-16 w-16 rounded bg-panel border border-border flex items-center justify-center text-faint"
+        className={`${boxClass} rounded bg-panel border border-border flex items-center justify-center text-faint`}
         title={`${path} (no longer on disk)`}
       >
         <ImageOff className="icon-base" />
@@ -99,7 +113,7 @@ export function JsonModeChatImageThumb({ path, mediaType }: Props): JSX.Element
         <img
           src={dataUrl}
           alt={name}
-          className="h-16 w-16 object-cover rounded border border-border hover:border-accent transition-colors"
+          className={`${imgClass} rounded border border-border hover:border-accent transition-colors`}
         />
       </button>
       {showFull && (