From bf82d9bf82389c711eaf44b453107b67d7e2065a Mon Sep 17 00:00:00 2001 From: quanru Date: Thu, 20 Aug 2026 22:36:28 +0800 Subject: [PATCH 1/4] feat(report): make screenshot consumers format-aware Co-authored-by: quanruzhuoxiu --- .../extension/recorder/screenshot-export.ts | 96 +++++++++++++++++++ .../src/extension/recorder/utils.ts | 89 ++++------------- .../recorder-export-image-format.test.ts | 57 +++++++++++ apps/report/src/App.tsx | 13 ++- .../src/components/playground/index.tsx | 10 +- .../build-timeline-screenshots.test.ts | 15 +++ .../timeline/build-timeline-screenshots.ts | 11 ++- apps/report/src/utils/markdown-export.test.ts | 18 ++++ apps/report/src/utils/markdown-export.ts | 2 +- .../src/utils/screenshot-source.test.ts | 32 +++++++ apps/report/src/utils/screenshot-source.ts | 31 ++++++ packages/core/src/dump/report-action-dump.ts | 23 +++-- packages/core/src/dump/screenshot-store.ts | 16 +++- packages/core/src/report-cli.ts | 2 +- packages/core/src/report-markdown.ts | 76 +++++++++------ packages/core/src/report.ts | 18 +++- packages/core/src/screenshot-item.ts | 61 ++++++++---- packages/core/src/types.ts | 6 +- .../unit-test/report-action-dump-webp.test.ts | 75 +++++++++++++++ .../core/tests/unit-test/report-cli.test.ts | 42 ++++++++ .../tests/unit-test/report-markdown.test.ts | 47 +++++++++ .../core/tests/unit-test/report-split.test.ts | 45 +++++++++ .../tests/unit-test/screenshot-item.test.ts | 20 ++++ .../tests/unit-test/screenshot-store.test.ts | 24 +++++ .../playground/src/mjpeg-stream-handler.ts | 19 +++- packages/playground/src/server.ts | 41 ++++++-- .../tests/unit/server-interact.test.ts | 13 ++- .../tests/unit/server-mjpeg-stream.test.ts | 29 ++++++ .../src/agent-tools/observation-record.ts | 35 +++++-- packages/shared/src/agent-tools/types.ts | 5 +- packages/shared/src/cli/screenshot-file.ts | 17 ++-- packages/shared/src/cli/verbose-screenshot.ts | 4 +- .../unit-test/observation-record.test.ts | 36 ++++++- .../tests/unit-test/screenshot-file.test.ts | 62 ++++++++++++ .../src/hooks/usePlaygroundExecution.ts | 10 +- .../tests/playground-execution-stop.test.ts | 8 +- 36 files changed, 921 insertions(+), 187 deletions(-) create mode 100644 apps/chrome-extension/src/extension/recorder/screenshot-export.ts create mode 100644 apps/chrome-extension/tests/recorder-export-image-format.test.ts create mode 100644 apps/report/src/utils/screenshot-source.test.ts create mode 100644 apps/report/src/utils/screenshot-source.ts create mode 100644 packages/core/tests/unit-test/report-action-dump-webp.test.ts create mode 100644 packages/shared/tests/unit-test/screenshot-file.test.ts diff --git a/apps/chrome-extension/src/extension/recorder/screenshot-export.ts b/apps/chrome-extension/src/extension/recorder/screenshot-export.ts new file mode 100644 index 0000000000..e3bfd0ded4 --- /dev/null +++ b/apps/chrome-extension/src/extension/recorder/screenshot-export.ts @@ -0,0 +1,96 @@ +import { + type ScreenshotImageFormat, + type ScreenshotImageMimeType, + inferScreenshotImageFormatFromBase64, + screenshotImageExtension, + screenshotImageMimeType, +} from '@midscene/shared/img'; +import type { RecordingSession } from '../../store'; + +export interface RecorderScreenshotAsset { + body: string; + extension: ScreenshotImageFormat; + mimeType: ScreenshotImageMimeType; +} + +export const recorderScreenshotAsset = ( + screenshotBase64: string, +): RecorderScreenshotAsset => { + const separator = ';base64,'; + const separatorIndex = screenshotBase64.indexOf(separator); + const body = ( + separatorIndex === -1 + ? screenshotBase64 + : screenshotBase64.slice(separatorIndex + separator.length) + ).replace(/\s/g, ''); + const format = inferScreenshotImageFormatFromBase64(body); + if (!format) { + throw new Error('Unsupported recorder screenshot image format'); + } + + return { + body, + extension: screenshotImageExtension(format), + mimeType: screenshotImageMimeType(format), + }; +}; + +export const generateEventsMarkdownTable = ( + sessions: RecordingSession[], +): string => { + let markdown = '# Test Events Report\n\n'; + + sessions.forEach((session, sessionIndex) => { + if (session.events.length === 0) return; + + markdown += `## ${session.name}\n\n`; + if (session.description) { + markdown += `**Description:** ${session.description}\n\n`; + } + markdown += `**Created:** ${new Date(session.createdAt).toLocaleString()}\n\n`; + markdown += '| Page | Screenshot Before | Screenshot After | Action |\n'; + markdown += '|------|------------|------------|--------|\n'; + + session.events.forEach((event, eventIndex) => { + const page = event.title || event.url || ''; + const screenshotBefore = event.screenshotBefore + ? `![](./images/screenshot_${sessionIndex}_${eventIndex}_before.${recorderScreenshotAsset(event.screenshotBefore).extension})` + : 'N/A'; + const screenshotAfter = event.screenshotAfter + ? `![](./images/screenshot_${sessionIndex}_${eventIndex}_after.${recorderScreenshotAsset(event.screenshotAfter).extension})` + : 'N/A'; + let action = ''; + switch (event.type) { + case 'click': + action = `Click on ${event.elementDescription || 'element'}`; + break; + case 'input': + action = `Input "${event.value}" into ${event.elementDescription || 'field'}`; + break; + case 'navigation': + action = `Navigate to ${event.url}`; + break; + default: + action = `${event.type} on ${event.elementDescription || 'element'}`; + } + + markdown += `| ${page} | ${screenshotBefore} | ${screenshotAfter} | ${action} |\n`; + }); + + if (session.generatedCode?.yaml || session.generatedCode?.playwright) { + markdown += '## Generated Code\n\n'; + if (session.generatedCode?.yaml) { + markdown += '### YAML\n\n'; + markdown += `\`\`\`yaml\n${session.generatedCode.yaml}\n\`\`\`\n\n`; + } + if (session.generatedCode?.playwright) { + markdown += '### Playwright\n\n'; + markdown += `\`\`\`playwright\n${session.generatedCode.playwright}\n\`\`\`\n\n`; + } + } + + markdown += '\n\n\n'; + }); + + return markdown; +}; diff --git a/apps/chrome-extension/src/extension/recorder/utils.ts b/apps/chrome-extension/src/extension/recorder/utils.ts index ec9badc74f..c40919da63 100644 --- a/apps/chrome-extension/src/extension/recorder/utils.ts +++ b/apps/chrome-extension/src/extension/recorder/utils.ts @@ -13,6 +13,11 @@ import type { ChatCompletionContentPart } from 'openai/resources/index'; import type { RecordingSession } from '../../store'; import { recordLogger } from './logger'; import { withRecorderMessageTimeout } from './messageTimeout'; +import { + type RecorderScreenshotAsset, + generateEventsMarkdownTable, + recorderScreenshotAsset, +} from './screenshot-export'; import { isChromeExtension, safeChromeAPI } from './types'; const CONTENT_SCRIPT_MESSAGE_TIMEOUT_MS = 5_000; @@ -778,74 +783,11 @@ const generateDetailedSequentialMindmap = ( return mermaid; }; -// Generate markdown table for event details -const generateEventsMarkdownTable = (sessions: RecordingSession[]): string => { - let markdown = '# Test Events Report\n\n'; - - sessions.forEach((session, sessionIndex) => { - if (session.events.length === 0) return; - - markdown += `## ${session.name}\n\n`; - if (session.description) { - markdown += `**Description:** ${session.description}\n\n`; - } - markdown += `**Created:** ${new Date(session.createdAt).toLocaleString()}\n\n`; - - markdown += '| Page | Screenshot Before | Screenshot After | Action |\n'; - markdown += '|------|------------|------------|--------|\n'; - - session.events.forEach((event, eventIndex) => { - let expected = 'N/A'; - const page = event.title || event.url || ''; - const screenshotBefore = event.screenshotBefore - ? `![](./images/screenshot_${sessionIndex}_${eventIndex}_before.png)` - : 'N/A'; - const screenshotAfter = event.screenshotAfter - ? `![](./images/screenshot_${sessionIndex}_${eventIndex}_after.png)` - : 'N/A'; - if (event.type === 'navigation') { - expected = `Navigate to ${event.url}`; - } - - let action = ''; - switch (event.type) { - case 'click': - action = `Click on ${event.elementDescription || 'element'}`; - break; - case 'input': - action = `Input "${event.value}" into ${event.elementDescription || 'field'}`; - break; - case 'navigation': - action = `Navigate to ${event.url}`; - break; - default: - action = `${event.type} on ${event.elementDescription || 'element'}`; - } - - markdown += `| ${page} | ${screenshotBefore} | ${screenshotAfter} | ${action} |\n`; - }); - - if (session.generatedCode?.yaml || session.generatedCode?.playwright) { - markdown += '## Generated Code\n\n'; - if (session.generatedCode?.yaml) { - markdown += '### YAML\n\n'; - markdown += `\`\`\`yaml\n${session.generatedCode.yaml}\n\`\`\`\n\n`; - } - if (session.generatedCode?.playwright) { - markdown += '### Playwright\n\n'; - markdown += `\`\`\`playwright\n${session.generatedCode.playwright}\n\`\`\`\n\n`; - } - } - - markdown += '\n\n\n'; - }); - - return markdown; -}; - -// Convert base64 to blob -const base64ToBlob = (base64: string, mimeType: string): Blob => { - const byteCharacters = atob(base64.split(',')[1]); +const base64BodyToBlob = ( + base64Body: string, + mimeType: RecorderScreenshotAsset['mimeType'], +): Blob => { + const byteCharacters = atob(base64Body); const byteNumbers = new Array(byteCharacters.length); for (let i = 0; i < byteCharacters.length; i++) { byteNumbers[i] = byteCharacters.charCodeAt(i); @@ -875,15 +817,16 @@ export const exportAllEventsToZip = async (sessions: RecordingSession[]) => { // Process each session and extract images sessionsWithEvents.forEach((session, sessionIndex) => { session.events.forEach((event, eventIndex) => { - const ext = 'png'; if (event.screenshotBefore) { - const fileName = `screenshot_${sessionIndex}_${eventIndex}_before.${ext}`; - const blob = base64ToBlob(event.screenshotBefore, `image/${ext}`); + const asset = recorderScreenshotAsset(event.screenshotBefore); + const fileName = `screenshot_${sessionIndex}_${eventIndex}_before.${asset.extension}`; + const blob = base64BodyToBlob(asset.body, asset.mimeType); imagesFolder?.file(fileName, blob); } if (event.screenshotAfter) { - const fileName = `screenshot_${sessionIndex}_${eventIndex}_after.${ext}`; - const blob = base64ToBlob(event.screenshotAfter, `image/${ext}`); + const asset = recorderScreenshotAsset(event.screenshotAfter); + const fileName = `screenshot_${sessionIndex}_${eventIndex}_after.${asset.extension}`; + const blob = base64BodyToBlob(asset.body, asset.mimeType); imagesFolder?.file(fileName, blob); } }); diff --git a/apps/chrome-extension/tests/recorder-export-image-format.test.ts b/apps/chrome-extension/tests/recorder-export-image-format.test.ts new file mode 100644 index 0000000000..0b561a507f --- /dev/null +++ b/apps/chrome-extension/tests/recorder-export-image-format.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from '@rstest/core'; +import { + generateEventsMarkdownTable, + recorderScreenshotAsset, +} from '../src/extension/recorder/screenshot-export'; +import type { RecordingSession } from '../src/store'; + +const webpBody = + 'UklGRioAAABXRUJQVlA4IB4AAAAwAQCdASoBAAEAAUAmJQBOgCHwAP7+hNQAAAA='; +const pngBody = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='; + +describe('Chrome recorder screenshot export', () => { + it('derives the exported extension and MIME type from the image bytes', () => { + expect( + recorderScreenshotAsset(`data:image/png;base64,${webpBody}`), + ).toEqual({ + body: webpBody, + extension: 'webp', + mimeType: 'image/webp', + }); + expect(recorderScreenshotAsset(`data:image/png;base64,${pngBody}`)).toEqual( + { + body: pngBody, + extension: 'png', + mimeType: 'image/png', + }, + ); + }); + + it('uses each screenshot actual extension in the Markdown table', () => { + const sessions = [ + { + id: 'session-1', + name: 'WebP export', + createdAt: 1, + updatedAt: 1, + status: 'completed', + events: [ + { + type: 'click', + timestamp: 1, + hashId: 'event-1', + screenshotBefore: `data:image/webp;base64,${webpBody}`, + screenshotAfter: `data:image/png;base64,${pngBody}`, + }, + ], + }, + ] as RecordingSession[]; + + const markdown = generateEventsMarkdownTable(sessions); + + expect(markdown).toContain('![](./images/screenshot_0_0_before.webp)'); + expect(markdown).toContain('![](./images/screenshot_0_0_after.png)'); + expect(markdown).not.toContain('screenshot_0_0_before.png'); + }); +}); diff --git a/apps/report/src/App.tsx b/apps/report/src/App.tsx index 15b56bd0b1..a1f6474e95 100644 --- a/apps/report/src/App.tsx +++ b/apps/report/src/App.tsx @@ -54,12 +54,16 @@ import { getEmptyDumpDescription, parseDumpAttributes, } from './utils/report-dump'; +import { + type ReportScreenshotSourceRef, + resolveScreenshotFallbackPath, +} from './utils/screenshot-source'; // Shared image cache across all test cases — resolved images are cached by id const imageCache = new Map(); function resolveImageFromDom( - refOrId: string | { id: string; storage?: 'inline' | 'file'; path?: string }, + refOrId: string | ReportScreenshotSourceRef, ): string { const id = typeof refOrId === 'string' ? refOrId : refOrId.id; const cached = imageCache.get(id); @@ -74,12 +78,7 @@ function resolveImageFromDom( return data; } - if (typeof refOrId === 'object' && refOrId?.storage === 'file') { - return refOrId.path || `./screenshots/${id}.png`; - } - - // Fallback to directory path - return `./screenshots/${id}.png`; + return resolveScreenshotFallbackPath(refOrId); } let globalRenderCount = 1; diff --git a/apps/report/src/components/playground/index.tsx b/apps/report/src/components/playground/index.tsx index 3b23220cef..1c5822ce0b 100644 --- a/apps/report/src/components/playground/index.tsx +++ b/apps/report/src/components/playground/index.tsx @@ -14,6 +14,10 @@ import { } from '@midscene/core/dump'; import { type PlaygroundSDK, noReplayAPIs } from '@midscene/playground'; import type { ServerResponse } from '@midscene/playground'; +import { + screenshotImageExtension, + screenshotImageFormatFromMimeType, +} from '@midscene/shared/img'; import { ContextPreview, Logo, @@ -55,7 +59,11 @@ async function loadReferencedReplay(result: PlaygroundResult) { } const dump = (await response.json()) as IReportActionDump; result.dump = restoreImageReferences(dump, (ref) => { - const extension = ref.mimeType === 'image/jpeg' ? 'jpeg' : 'png'; + const format = screenshotImageFormatFromMimeType(ref.mimeType); + if (!format) { + throw new Error(`Unsupported screenshot mime type: ${ref.mimeType}`); + } + const extension = screenshotImageExtension(format); return new URL( `screenshots/${encodeURIComponent(ref.id)}.${extension}`, result.report!.url, diff --git a/apps/report/src/components/timeline/build-timeline-screenshots.test.ts b/apps/report/src/components/timeline/build-timeline-screenshots.test.ts index a3483c8d99..c153a6733d 100644 --- a/apps/report/src/components/timeline/build-timeline-screenshots.test.ts +++ b/apps/report/src/components/timeline/build-timeline-screenshots.test.ts @@ -4,6 +4,8 @@ import { buildTimelineScreenshots } from './build-timeline-screenshots'; const onePixelPngBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAFgwJ/lz8yrwAAAABJRU5ErkJggg=='; +const webpBase64 = + 'UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; interface TaskFixtureOptions { id: string; @@ -103,6 +105,19 @@ describe('buildTimelineScreenshots', () => { ); }); + it('infers WebP for recorder screenshots stored as raw base64 strings', () => { + const task = makeTask({ + id: 'raw-webp-recorder', + startTs: 1000, + recorder: [{ ts: 1200, screenshot: webpBase64 }], + }); + + const { allScreenshots } = buildTimelineScreenshots([task]); + + expect(allScreenshots).toHaveLength(1); + expect(allScreenshots[0].img).toBe(`data:image/webp;base64,${webpBase64}`); + }); + it('keeps already-prefixed data URLs unchanged', () => { const dataUrl = `data:image/png;base64,${onePixelPngBase64}`; const task = makeTask({ diff --git a/apps/report/src/components/timeline/build-timeline-screenshots.ts b/apps/report/src/components/timeline/build-timeline-screenshots.ts index 093a58df47..e9a71c3462 100644 --- a/apps/report/src/components/timeline/build-timeline-screenshots.ts +++ b/apps/report/src/components/timeline/build-timeline-screenshots.ts @@ -1,4 +1,8 @@ import type { ExecutionTask } from '@midscene/core'; +import { + inferScreenshotImageFormatFromBase64, + screenshotImageMimeType, +} from '@midscene/shared/img'; export interface TimelineScreenshot { id: string; @@ -30,11 +34,8 @@ const imageSrcFromString = (value: string): string => { } const body = trimmed.replace(/\s/g, ''); - const mimeType = body.startsWith('/9j/') - ? 'image/jpeg' - : body.startsWith('UklGR') - ? 'image/webp' - : 'image/png'; + const format = inferScreenshotImageFormatFromBase64(body) ?? 'png'; + const mimeType = screenshotImageMimeType(format); return `data:${mimeType};base64,${body}`; }; diff --git a/apps/report/src/utils/markdown-export.test.ts b/apps/report/src/utils/markdown-export.test.ts index d4a6d7858c..9fcae83254 100644 --- a/apps/report/src/utils/markdown-export.test.ts +++ b/apps/report/src/utils/markdown-export.test.ts @@ -105,6 +105,24 @@ describe('markdown-export helpers', () => { ); }); + it('packages an in-memory WebP attachment without changing its bytes', () => { + const webpBytes = 'webp-image-bytes'; + const files = buildMarkdownArchiveFiles('# report', [ + { + id: 'inline-webp', + suggestedFileName: 'inline.webp', + mimeType: 'image/webp', + executionIndex: 0, + taskIndex: 0, + base64Data: `data:image/webp;base64,${btoa(webpBytes)}`, + }, + ]); + + expect(new TextDecoder().decode(files['screenshots/inline.webp'])).toBe( + webpBytes, + ); + }); + it('builds display items from markdown attachment names and paths', () => { const items = getMarkdownAttachmentDisplayItems([ { diff --git a/apps/report/src/utils/markdown-export.ts b/apps/report/src/utils/markdown-export.ts index 53cfb7c632..179a3e2f86 100644 --- a/apps/report/src/utils/markdown-export.ts +++ b/apps/report/src/utils/markdown-export.ts @@ -1,7 +1,7 @@ import type { MarkdownAttachment } from '@midscene/core'; const defaultScreenshotBaseDir = './screenshots'; -const dataUrlBase64Pattern = /^data:image\/(?:png|jpeg|jpg);base64,/i; +const dataUrlBase64Pattern = /^data:image\/(?:png|jpeg|jpg|webp);base64,/i; const rawBase64Pattern = /^[a-zA-Z0-9+/=\s]+$/; type MarkdownExport = { diff --git a/apps/report/src/utils/screenshot-source.test.ts b/apps/report/src/utils/screenshot-source.test.ts new file mode 100644 index 0000000000..6dd91a6f45 --- /dev/null +++ b/apps/report/src/utils/screenshot-source.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from '@rstest/core'; +import { resolveScreenshotFallbackPath } from './screenshot-source'; + +describe('resolveScreenshotFallbackPath', () => { + it('uses MIME-specific extensions for screenshot references', () => { + expect( + resolveScreenshotFallbackPath({ + id: 'webp-shot', + mimeType: 'image/webp', + storage: 'inline', + }), + ).toBe('./screenshots/webp-shot.webp'); + expect( + resolveScreenshotFallbackPath({ + id: 'jpeg-shot', + mimeType: 'image/jpeg', + storage: 'inline', + }), + ).toBe('./screenshots/jpeg-shot.jpeg'); + }); + + it('prefers an explicit file-backed path', () => { + expect( + resolveScreenshotFallbackPath({ + id: 'webp-shot', + mimeType: 'image/webp', + storage: 'file', + path: './assets/custom.webp', + }), + ).toBe('./assets/custom.webp'); + }); +}); diff --git a/apps/report/src/utils/screenshot-source.ts b/apps/report/src/utils/screenshot-source.ts new file mode 100644 index 0000000000..7cec1d16d3 --- /dev/null +++ b/apps/report/src/utils/screenshot-source.ts @@ -0,0 +1,31 @@ +import { + screenshotImageExtension, + screenshotImageFormatFromMimeType, +} from '@midscene/shared/img'; + +export interface ReportScreenshotSourceRef { + id: string; + mimeType?: unknown; + storage?: 'inline' | 'file'; + path?: string; +} + +export function resolveScreenshotFallbackPath( + refOrId: string | ReportScreenshotSourceRef, +): string { + if ( + typeof refOrId === 'object' && + refOrId.storage === 'file' && + refOrId.path + ) { + return refOrId.path; + } + + const id = typeof refOrId === 'string' ? refOrId : refOrId.id; + const format = + typeof refOrId === 'object' + ? screenshotImageFormatFromMimeType(refOrId.mimeType) + : undefined; + const extension = format ? screenshotImageExtension(format) : 'png'; + return `./screenshots/${id}.${extension}`; +} diff --git a/packages/core/src/dump/report-action-dump.ts b/packages/core/src/dump/report-action-dump.ts index cb7b5cca2d..deeed45f82 100644 --- a/packages/core/src/dump/report-action-dump.ts +++ b/packages/core/src/dump/report-action-dump.ts @@ -6,6 +6,10 @@ import { writeFileSync, } from 'node:fs'; import { join } from 'node:path'; +import { + screenshotImageExtension, + screenshotImageFormatFromMimeType, +} from '@midscene/shared/img'; import { ScreenshotItem } from '../screenshot-item'; import type { ExecutionTask, @@ -13,7 +17,7 @@ import type { IReportActionDump, } from '../types'; import { restoreImageReferences } from './screenshot-restoration'; -import { ScreenshotStore } from './screenshot-store'; +import { type ScreenshotRef, ScreenshotStore } from './screenshot-store'; /** * Replacer function for JSON serialization that handles Page, Browser objects and ScreenshotItem @@ -253,10 +257,10 @@ export class ReportActionDump implements IReportActionDump { } /** - * Serialize the dump to files with screenshots as separate PNG files. + * Serialize the dump to files with screenshots as separate image files. * Creates: * - {basePath} - dump JSON with { $screenshot: id } references - * - {basePath}.screenshots/ - PNG files + * - {basePath}.screenshots/ - screenshot image files * * @param basePath - Base path for the dump file */ @@ -296,14 +300,21 @@ export class ReportActionDump implements IReportActionDump { const dumpString = readFileSync(basePath, 'utf-8'); const screenshotsDir = `${basePath}.screenshots`; - const loadFromExecutionScreenshotDir = (id: string, mimeType: string) => { - const ext = mimeType === 'image/jpeg' ? 'jpeg' : 'png'; + const loadFromExecutionScreenshotDir = ( + id: string, + mimeType: ScreenshotRef['mimeType'], + ) => { + const format = screenshotImageFormatFromMimeType(mimeType); + if (!format) { + throw new Error(`Unsupported screenshot mime type: ${mimeType}`); + } + const ext = screenshotImageExtension(format); const filePath = join(screenshotsDir, `${id}.${ext}`); if (!existsSync(filePath)) { return ''; } const data = readFileSync(filePath); - return `data:image/${ext};base64,${data.toString('base64')}`; + return `data:${mimeType};base64,${data.toString('base64')}`; }; // Restore image references diff --git a/packages/core/src/dump/screenshot-store.ts b/packages/core/src/dump/screenshot-store.ts index 0e7c8ae78a..0661c8fe04 100644 --- a/packages/core/src/dump/screenshot-store.ts +++ b/packages/core/src/dump/screenshot-store.ts @@ -1,6 +1,12 @@ import { existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs'; import { writeFile as writeFileAsync } from 'node:fs/promises'; import { dirname, isAbsolute, join } from 'node:path'; +import { + type ScreenshotImageMimeType, + isScreenshotImageMimeType, + screenshotImageExtension, + screenshotImageFormatFromMimeType, +} from '@midscene/shared/img'; import type { ScreenshotItem } from '../screenshot-item'; import { extractImageByIdSync } from './html-utils'; @@ -8,7 +14,7 @@ export interface ScreenshotRef { type: 'midscene_screenshot_ref'; id: string; capturedAt: number; - mimeType: 'image/png' | 'image/jpeg'; + mimeType: ScreenshotImageMimeType; storage: 'inline' | 'file'; path?: string; } @@ -22,7 +28,7 @@ export function normalizeScreenshotRef(value: unknown): ScreenshotRef | null { typeof record.id === 'string' && typeof record.capturedAt === 'number' && (record.storage === 'inline' || record.storage === 'file') && - (record.mimeType === 'image/png' || record.mimeType === 'image/jpeg') + isScreenshotImageMimeType(record.mimeType) ) { if (record.storage === 'file' && typeof record.path !== 'string') { return null; @@ -48,7 +54,11 @@ type ResolvedScreenshotSource = }; function extensionByMimeType(mimeType: ScreenshotRef['mimeType']): string { - return mimeType === 'image/jpeg' ? 'jpeg' : 'png'; + const format = screenshotImageFormatFromMimeType(mimeType); + if (!format) { + throw new Error(`Unsupported screenshot mime type: ${mimeType}`); + } + return screenshotImageExtension(format); } export function resolveScreenshotSource( diff --git a/packages/core/src/report-cli.ts b/packages/core/src/report-cli.ts index 2e12dd5100..3c8b627aee 100644 --- a/packages/core/src/report-cli.ts +++ b/packages/core/src/report-cli.ts @@ -79,7 +79,7 @@ function writeAttachmentFromReport( const resolved = resolveScreenshotSource(attachment.sourceRef ?? null, { reportPath: opts.htmlPath, fallbackId: id, - fallbackMimeType: (mimeType || 'image/png') as 'image/png' | 'image/jpeg', + fallbackMimeType: mimeType || 'image/png', }); if (resolved.type === 'data-uri') { diff --git a/packages/core/src/report-markdown.ts b/packages/core/src/report-markdown.ts index 86701c77f4..533163ac3e 100644 --- a/packages/core/src/report-markdown.ts +++ b/packages/core/src/report-markdown.ts @@ -9,13 +9,20 @@ import type { ModelBrief, ReportActionDump, } from '@/types'; +import { + type ScreenshotImageFormat, + type ScreenshotImageMimeType, + normalizeScreenshotBase64, + parseBase64, + screenshotImageExtension, + screenshotImageFormatFromMimeType, + screenshotImageMimeType, +} from '@midscene/shared/img'; import type { ScreenshotRef } from './dump/screenshot-store'; import { normalizeScreenshotRef } from './dump/screenshot-store'; import { collectReportSummary } from './report-stats'; -const screenshotDataUrlPattern = - /^data:image\/(png|jpeg|jpg);base64,([\s\S]*)$/i; -const rawBase64BodyPattern = /^[a-zA-Z0-9+/=\s]+$/; +const screenshotDataUrlPattern = /^data:image\/(?:png|jpe?g|webp);base64,/i; const jsonContextMaxStringLength = 12_000; type ExecutionTaskWithExtraUsage = ExecutionTask & { @@ -31,7 +38,7 @@ export interface MarkdownAttachment { * write the screenshot under this name to keep links in sync. See #2392. */ suggestedFileName: string; - mimeType?: string; + mimeType?: ScreenshotImageMimeType; /** * Reference to the screenshot in the source report, used to locate the * original bytes when copying them to the exported name. Absent for in-memory @@ -398,36 +405,35 @@ function extractLocateCenter( function tryExtractBase64(screenshot: unknown): string | undefined { if (typeof screenshot === 'string') { - const trimmedScreenshot = screenshot.trim(); - const dataUrlMatch = trimmedScreenshot.match(screenshotDataUrlPattern); - if (dataUrlMatch) { - const format = dataUrlMatch[1].toLowerCase() === 'jpg' ? 'jpeg' : 'png'; - const base64Body = dataUrlMatch[2].replace(/\s/g, ''); - if (!base64Body) { - return undefined; - } - return `data:image/${format};base64,${base64Body}`; - } - - if ( - trimmedScreenshot.startsWith('data:') || - !rawBase64BodyPattern.test(trimmedScreenshot) - ) { + try { + return normalizeScreenshotBase64(screenshot); + } catch { return undefined; } - - const base64Body = trimmedScreenshot.replace(/\s/g, ''); - return base64Body ? `data:image/png;base64,${base64Body}` : undefined; } if (!screenshot || typeof screenshot !== 'object') return undefined; const s = screenshot as Record; if (typeof s.base64 === 'string' && s.base64.length > 0) { - return s.base64; + return tryExtractBase64(s.base64); } return undefined; } +function screenshotMetadataFromMimeType(mimeType: unknown): { + extension: ScreenshotImageFormat; + mimeType: ScreenshotImageMimeType; +} { + const format = screenshotImageFormatFromMimeType(mimeType); + if (!format) { + throw new Error(`Unsupported screenshot mime type: ${String(mimeType)}`); + } + return { + extension: screenshotImageExtension(format), + mimeType: screenshotImageMimeType(format), + }; +} + function restoredSourceRef(screenshot: unknown): ScreenshotRef | undefined { if (!screenshot || typeof screenshot !== 'object') { return undefined; @@ -437,6 +443,14 @@ function restoredSourceRef(screenshot: unknown): ScreenshotRef | undefined { return normalizeScreenshotRef(sourceRef) ?? undefined; } +function restoredScreenshotSource(screenshot: unknown): string | undefined { + if (!screenshot || typeof screenshot !== 'object') { + return undefined; + } + const base64 = (screenshot as Record).base64; + return typeof base64 === 'string' && base64.length > 0 ? base64 : undefined; +} + function screenshotAttachment( screenshot: unknown, screenshotBaseDir: string, @@ -457,7 +471,7 @@ function screenshotAttachment( attachment: { id: screenshot.id, suggestedFileName, - mimeType: `image/${ext === 'jpeg' ? 'jpeg' : 'png'}`, + mimeType: screenshot.mimeType, executionIndex, taskIndex, base64Data: tryExtractBase64(screenshot), @@ -467,7 +481,7 @@ function screenshotAttachment( const ref = normalizeScreenshotRef(screenshot); if (ref) { - const ext = ref.mimeType === 'image/jpeg' ? 'jpeg' : 'png'; + const { extension: ext } = screenshotMetadataFromMimeType(ref.mimeType); const suggestedFileName = `execution-${executionIndex + 1}-task-${taskIndex + 1}-${ref.id}.${ext}`; return { markdown: `\n![${markdownLabel}](${screenshotBaseDir}/${suggestedFileName})`, @@ -485,7 +499,9 @@ function screenshotAttachment( const sourceRef = restoredSourceRef(screenshot); if (sourceRef) { - const ext = sourceRef.mimeType === 'image/jpeg' ? 'jpeg' : 'png'; + const { extension: ext } = screenshotMetadataFromMimeType( + sourceRef.mimeType, + ); const suggestedFileName = `execution-${executionIndex + 1}-task-${taskIndex + 1}-${sourceRef.id}.${ext}`; return { markdown: `\n![${markdownLabel}](${screenshotBaseDir}/${suggestedFileName})`, @@ -496,14 +512,16 @@ function screenshotAttachment( mimeType: sourceRef.mimeType, executionIndex, taskIndex, - base64Data: tryExtractBase64(screenshot), + base64Data: restoredScreenshotSource(screenshot), }, }; } const base64 = tryExtractBase64(screenshot); if (base64) { - const ext = base64.startsWith('data:image/jpeg') ? 'jpeg' : 'png'; + const { extension: ext, mimeType } = screenshotMetadataFromMimeType( + parseBase64(base64).mimeType, + ); const idSuffix = options?.fallbackIdSuffix ? `-${options.fallbackIdSuffix}` : ''; @@ -514,7 +532,7 @@ function screenshotAttachment( attachment: { id, suggestedFileName, - mimeType: `image/${ext}`, + mimeType, executionIndex, taskIndex, base64Data: base64, diff --git a/packages/core/src/report.ts b/packages/core/src/report.ts index a6f4a182f9..9bd4413bce 100644 --- a/packages/core/src/report.ts +++ b/packages/core/src/report.ts @@ -10,6 +10,11 @@ import { } from 'node:fs'; import * as path from 'node:path'; import { getMidsceneRunSubDir } from '@midscene/shared/common'; +import { + type ScreenshotImageFormat, + screenshotImageExtension, + screenshotImageFormatFromMimeType, +} from '@midscene/shared/img'; import { antiEscapeScriptTag, logMsg } from '@midscene/shared/utils'; import { getReportFileName } from './agent'; import { @@ -22,6 +27,7 @@ import { streamImageScriptsToFile, } from './dump/html-utils'; import { + type ScreenshotRef, normalizeScreenshotRef, resolveScreenshotSource, } from './dump/screenshot-store'; @@ -554,10 +560,14 @@ export function collectDedupedExecutions( }; } -function extensionByMimeType(mimeType: string): 'png' | 'jpeg' { - if (mimeType === 'image/png') return 'png'; - if (mimeType === 'image/jpeg') return 'jpeg'; - throw new Error(`Unsupported screenshot mime type: ${mimeType}`); +function extensionByMimeType( + mimeType: ScreenshotRef['mimeType'], +): ScreenshotImageFormat { + const format = screenshotImageFormatFromMimeType(mimeType); + if (!format) { + throw new Error(`Unsupported screenshot mime type: ${mimeType}`); + } + return screenshotImageExtension(format); } function externalizeScreenshotsInExecution( diff --git a/packages/core/src/screenshot-item.ts b/packages/core/src/screenshot-item.ts index e7bc4b6169..a55b52fe27 100644 --- a/packages/core/src/screenshot-item.ts +++ b/packages/core/src/screenshot-item.ts @@ -1,4 +1,12 @@ import { readFileSync } from 'node:fs'; +import { + type ScreenshotImageFormat, + type ScreenshotImageMimeType, + inferScreenshotImageFormatFromBase64, + screenshotImageExtension, + screenshotImageFormatFromMimeType, + screenshotImageMimeType, +} from '@midscene/shared/img'; import { uuid } from '@midscene/shared/utils'; import { extractImageByIdSync } from './dump/html-utils'; import { @@ -16,10 +24,26 @@ export type ScreenshotSerializeFormat = ScreenshotRef; /** * Detect image format from base64 data URI prefix. */ -function detectFormat(base64: string): 'png' | 'jpeg' { - if (base64.startsWith('data:image/jpeg')) return 'jpeg'; - if (base64.startsWith('data:image/jpg')) return 'jpeg'; - return 'png'; +const BASE64_SEPARATOR = ';base64,'; + +function detectFormat(base64: string): ScreenshotImageFormat { + const separatorIndex = base64.indexOf(BASE64_SEPARATOR); + const mimeType = + separatorIndex === -1 ? undefined : base64.slice(5, separatorIndex); + const detectedFormat = + separatorIndex === -1 + ? inferScreenshotImageFormatFromBase64(base64) + : screenshotImageFormatFromMimeType(mimeType); + return detectedFormat ?? 'png'; +} + +function rawBase64Body(base64: string): string { + const separatorIndex = base64.indexOf(BASE64_SEPARATOR); + const body = + separatorIndex === -1 + ? base64 + : base64.slice(separatorIndex + BASE64_SEPARATOR.length); + return body.replace(/\s/g, ''); } /** @@ -35,7 +59,7 @@ function detectFormat(base64: string): 'png' | 'jpeg' { export class ScreenshotItem { private _id: string; private _base64: string | null; - private _format: 'png' | 'jpeg'; + private _format: ScreenshotImageFormat; private _capturedAt: number; private _serializedRef: ScreenshotRef | null = null; private _persistedPath: string | null = null; @@ -45,7 +69,7 @@ export class ScreenshotItem { id: string, base64: string | null, capturedAt: number, - format: 'png' | 'jpeg', + format: ScreenshotImageFormat, ) { this._id = id; this._base64 = base64; @@ -61,14 +85,14 @@ export class ScreenshotItem { /** Create a lazily loaded ScreenshotItem backed by an image file. */ static fromFile( filePath: string, - mimeType: 'image/png' | 'image/jpeg', + mimeType: ScreenshotImageMimeType, capturedAt: number, ): ScreenshotItem { const item = new ScreenshotItem( uuid(), null, capturedAt, - mimeType === 'image/jpeg' ? 'jpeg' : 'png', + screenshotImageFormatFromMimeType(mimeType)!, ); item._persistedPath = filePath; return item; @@ -78,14 +102,19 @@ export class ScreenshotItem { return this._id; } - /** Get the image format (png or jpeg) */ - get format(): 'png' | 'jpeg' { + /** Get the image format (PNG, JPEG, or WebP). */ + get format(): ScreenshotImageFormat { return this._format; } /** Get the file extension for this screenshot */ - get extension(): string { - return this._format === 'jpeg' ? 'jpeg' : 'png'; + get extension(): ScreenshotImageFormat { + return screenshotImageExtension(this._format); + } + + /** Get the MIME type for this screenshot. */ + get mimeType(): ScreenshotImageMimeType { + return screenshotImageMimeType(this._format); } /** Get screenshot capture timestamp in milliseconds */ @@ -104,7 +133,7 @@ export class ScreenshotItem { throw new Error(`Screenshot ${this._id}: file recovery path missing`); } const buffer = readFileSync(this._persistedPath); - return `data:image/${this._format};base64,${buffer.toString('base64')}`; + return `data:${this.mimeType};base64,${buffer.toString('base64')}`; }; const loadFromInline = (): string => { @@ -197,7 +226,7 @@ export class ScreenshotItem { type: 'midscene_screenshot_ref', id: this._id, capturedAt: this._capturedAt, - mimeType: this._format === 'jpeg' ? 'image/jpeg' : 'image/png', + mimeType: this.mimeType, storage: 'inline', } ); @@ -216,7 +245,7 @@ export class ScreenshotItem { type: 'midscene_screenshot_ref', id: this._id, capturedAt: this._capturedAt, - mimeType: this._format === 'jpeg' ? 'image/jpeg' : 'image/png', + mimeType: this.mimeType, storage, }; if (storage === 'file') { @@ -234,6 +263,6 @@ export class ScreenshotItem { * Useful for writing raw binary data to files. */ get rawBase64(): string { - return this.base64.replace(/^data:image\/(png|jpeg|jpg);base64,/, ''); + return rawBase64Body(this.base64); } } diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 7f9a2419a6..ab9801592e 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -545,7 +545,7 @@ export interface ExecutionRecorderItem { export interface RecordToReportScreenshot { /** - * PNG/JPEG data URI, or raw PNG base64 body. + * PNG/JPEG/WebP data URI, or raw PNG/JPEG/WebP base64 body. */ base64: string; description?: string; @@ -797,7 +797,7 @@ export type ExecutionTaskPlanningLocate = /* How a report file stores screenshots: - `inline`: base64 image script tags embedded in the single HTML file -- `directory`: external PNG files under a sibling `screenshots/` dir +- `directory`: external image files under a sibling `screenshots/` dir */ export type ScreenshotMode = 'inline' | 'directory'; @@ -973,7 +973,7 @@ export interface AgentOpt { * Use directory-based report format with separate image files. * * When enabled: - * - Screenshots are saved as PNG files in a `screenshots/` subdirectory + * - Screenshots retain their image format in a `screenshots/` subdirectory * - Report is generated as `index.html` with relative image paths * - Reduces memory usage and report file size * diff --git a/packages/core/tests/unit-test/report-action-dump-webp.test.ts b/packages/core/tests/unit-test/report-action-dump-webp.test.ts new file mode 100644 index 0000000000..2552a24459 --- /dev/null +++ b/packages/core/tests/unit-test/report-action-dump-webp.test.ts @@ -0,0 +1,75 @@ +import { existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { ScreenshotItem } from '../../src/screenshot-item'; +import { ExecutionDump, ReportActionDump } from '../../src/types'; + +const webpBody = + 'UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; +const webpBase64 = `data:image/webp;base64,${webpBody}`; + +describe('ReportActionDump WebP file serialization', () => { + let temporaryDirectory: string; + + beforeEach(() => { + temporaryDirectory = join( + tmpdir(), + `midscene-dump-webp-${Date.now()}-${Math.random()}`, + ); + mkdirSync(temporaryDirectory, { recursive: true }); + }); + + afterEach(() => { + rmSync(temporaryDirectory, { recursive: true, force: true }); + }); + + it('writes and restores WebP bytes with their MIME type', () => { + const screenshot = ScreenshotItem.create(webpBase64, 100); + const dump = new ReportActionDump({ + sdkVersion: '1.0.0-test', + groupName: 'webp-dump', + modelBriefs: [], + executions: [ + new ExecutionDump({ + logTime: 100, + name: 'webp-execution', + tasks: [ + { + taskId: 'webp-task', + type: 'Insight', + subType: 'Locate', + param: { prompt: 'target' }, + uiContext: { + screenshot, + shotSize: { width: 2, height: 3 }, + shrunkShotToLogicalRatio: 1, + }, + executor: async () => undefined, + recorder: [], + status: 'finished', + }, + ], + }), + ], + }); + const dumpPath = join(temporaryDirectory, 'dump.json'); + + dump.serializeToFiles(dumpPath); + + const screenshotPath = join( + `${dumpPath}.screenshots`, + `${screenshot.id}.webp`, + ); + expect(existsSync(screenshotPath)).toBe(true); + expect(readFileSync(screenshotPath).toString('base64')).toBe(webpBody); + + const restored = JSON.parse( + ReportActionDump.fromFilesAsInlineJson(dumpPath), + ); + expect(restored.executions[0].tasks[0].uiContext.screenshot).toMatchObject({ + base64: webpBase64, + capturedAt: 100, + }); + }); +}); diff --git a/packages/core/tests/unit-test/report-cli.test.ts b/packages/core/tests/unit-test/report-cli.test.ts index f80b55149b..2497db54cb 100644 --- a/packages/core/tests/unit-test/report-cli.test.ts +++ b/packages/core/tests/unit-test/report-cli.test.ts @@ -24,6 +24,9 @@ function fakeBase64(sizeBytes: number): string { return `data:image/png;base64,${'A'.repeat(sizeBytes)}`; } +const webpBase64 = + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; + function createExecution( id: string, screenshot: ScreenshotItem | ScreenshotRef, @@ -223,6 +226,45 @@ describe('createReportCliCommands', () => { expect(existsSync(join(outputDir, 'report.md'))).toBe(true); }); + it('exports WebP screenshots and markdown links with the .webp suffix', async () => { + const reportPath = join(tmpDir, 'input-report-sdk-webp', 'index.html'); + mkdirSync(join(tmpDir, 'input-report-sdk-webp'), { recursive: true }); + const screenshot = ScreenshotItem.create(webpBase64, Date.now()); + const dump = new ReportActionDump({ + groupName: 'sdk-webp-test', + sdkVersion: '1.0.0-test', + modelBriefs: [], + executions: [createExecution('exec-sdk-webp', screenshot)], + }); + writeFileSync( + reportPath, + [ + generateImageScriptTag(screenshot.id, screenshot.base64), + generateDumpScriptTag(dump.serialize(), { + 'data-group-id': 'webp-group', + }), + ].join('\n'), + 'utf-8', + ); + + const outputDir = join(tmpDir, 'output-sdk-webp'); + const result = await reportFileToMarkdown({ + htmlPath: reportPath, + outputDir, + }); + const expectedFileName = `execution-1-task-1-${screenshot.id}.webp`; + + expect(result.screenshotFiles).toEqual([ + join(outputDir, 'screenshots', expectedFileName), + ]); + expect(readFileSync(result.screenshotFiles[0]).toString('base64')).toBe( + webpBase64.split(',')[1], + ); + expect(readFileSync(join(outputDir, 'report.md'), 'utf-8')).toContain( + `./screenshots/${expectedFileName}`, + ); + }); + it('throws SDK-friendly validation errors for reportFileToMarkdown', async () => { await expect( reportFileToMarkdown({ diff --git a/packages/core/tests/unit-test/report-markdown.test.ts b/packages/core/tests/unit-test/report-markdown.test.ts index 7cfa5c9c04..8c60509882 100644 --- a/packages/core/tests/unit-test/report-markdown.test.ts +++ b/packages/core/tests/unit-test/report-markdown.test.ts @@ -28,6 +28,53 @@ function createTask(overrides: Record = {}) { } describe('report-markdown', () => { + it('exports WebP screenshots with matching MIME types and file extensions', () => { + const webpDataUrl = + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; + const execution: IExecutionDump = { + logTime: 1710000000000, + name: 'webp execution', + tasks: [ + createTask({ + uiContext: { + screenshot: ScreenshotItem.create(webpDataUrl, 1710000000000), + }, + recorder: [ + { + type: 'screenshot', + ts: 1710000000050, + screenshot: { + type: 'midscene_screenshot_ref', + id: 'webp-recorder', + capturedAt: 1710000000050, + mimeType: 'image/webp', + storage: 'file', + path: './screenshots/webp-recorder.webp', + }, + }, + ], + }), + ], + }; + + const result = executionToMarkdown(execution); + + expect(result.attachments).toHaveLength(2); + expect(result.attachments[0]).toMatchObject({ + mimeType: 'image/webp', + base64Data: webpDataUrl, + }); + expect(result.attachments[1]).toMatchObject({ + mimeType: 'image/webp', + sourceRef: { path: './screenshots/webp-recorder.webp' }, + }); + expect( + result.attachments.every(({ suggestedFileName }) => + suggestedFileName.endsWith('.webp'), + ), + ).toBe(true); + }); + it('handles single execution markdown with screenshot file links', () => { const screenshot = ScreenshotItem.create( 'data:image/png;base64,Zm9v', diff --git a/packages/core/tests/unit-test/report-split.test.ts b/packages/core/tests/unit-test/report-split.test.ts index 9c9f869c41..6c56364f6a 100644 --- a/packages/core/tests/unit-test/report-split.test.ts +++ b/packages/core/tests/unit-test/report-split.test.ts @@ -18,6 +18,9 @@ function fakeBase64(sizeBytes: number): string { return `data:image/png;base64,${'A'.repeat(sizeBytes)}`; } +const webpBase64 = + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; + function createExecution( id: string, screenshot: ScreenshotItem | ScreenshotRef, @@ -129,6 +132,48 @@ describe('splitReportHtmlByExecution', () => { } }); + it('externalizes inline WebP screenshots as .webp files', () => { + const reportPath = join(tmpDir, 'webp-report', 'index.html'); + mkdirSync(join(tmpDir, 'webp-report'), { recursive: true }); + const screenshot = ScreenshotItem.create(webpBase64, Date.now()); + const dump = new ReportActionDump({ + groupName: 'webp-split-test', + sdkVersion: '1.0.0-test', + modelBriefs: [], + executions: [createExecution('webp-exec', screenshot)], + }); + writeFileSync( + reportPath, + [ + generateImageScriptTag(screenshot.id, screenshot.base64), + generateDumpScriptTag(dump.serialize(), { + 'data-group-id': 'webp-group', + }), + ].join('\n'), + 'utf-8', + ); + + const result = splitReportHtmlByExecution({ + htmlPath: reportPath, + outputDir: join(tmpDir, 'webp-output'), + }); + const outputDump = JSON.parse( + readFileSync(result.executionJsonFiles[0], 'utf-8'), + ); + const outputRef = outputDump.executions[0].tasks[0].uiContext.screenshot; + + expect(outputRef).toMatchObject({ + mimeType: 'image/webp', + path: `./screenshots/${screenshot.id}.webp`, + }); + expect(result.screenshotFiles).toEqual([ + join(tmpDir, 'webp-output', 'screenshots', `${screenshot.id}.webp`), + ]); + expect(readFileSync(result.screenshotFiles[0]).toString('base64')).toBe( + webpBase64.split(',')[1], + ); + }); + it('should process large report incrementally without accumulating all dump scripts', () => { const reportPath = join(tmpDir, 'large-report', 'index.html'); mkdirSync(join(tmpDir, 'large-report'), { recursive: true }); diff --git a/packages/core/tests/unit-test/screenshot-item.test.ts b/packages/core/tests/unit-test/screenshot-item.test.ts index d20c0246f1..c4d2c42a33 100644 --- a/packages/core/tests/unit-test/screenshot-item.test.ts +++ b/packages/core/tests/unit-test/screenshot-item.test.ts @@ -6,6 +6,8 @@ import { ScreenshotItem } from '../../src/screenshot-item'; describe('ScreenshotItem', () => { const testBase64 = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA'; + const webpBase64 = + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; describe('create', () => { it('should create a ScreenshotItem from base64 string', () => { @@ -19,6 +21,19 @@ describe('ScreenshotItem', () => { const item = ScreenshotItem.create(testBase64, capturedAt); expect(item.capturedAt).toBe(capturedAt); }); + + it('classifies WebP screenshots without changing their body', () => { + const item = ScreenshotItem.create(webpBase64, 123); + + expect(item.format).toBe('webp'); + expect(item.extension).toBe('webp'); + expect(item.mimeType).toBe('image/webp'); + expect(item.rawBase64).toBe(webpBase64.split(',')[1]); + expect(item.toSerializable()).toMatchObject({ + capturedAt: 123, + mimeType: 'image/webp', + }); + }); }); describe('base64 getter', () => { @@ -126,6 +141,11 @@ describe('ScreenshotItem', () => { expect(item.rawBase64).toBe('/9j/4AAQ'); }); + it('should strip data URI prefix from WebP', () => { + const item = ScreenshotItem.create(webpBase64, Date.now()); + expect(item.rawBase64).toBe(webpBase64.split(',')[1]); + }); + it('should return unchanged if no prefix', () => { const item = ScreenshotItem.create( 'iVBORw0KGgoAAAANSUhEUgAAAAUA', diff --git a/packages/core/tests/unit-test/screenshot-store.test.ts b/packages/core/tests/unit-test/screenshot-store.test.ts index f63f801157..0a1b3e564d 100644 --- a/packages/core/tests/unit-test/screenshot-store.test.ts +++ b/packages/core/tests/unit-test/screenshot-store.test.ts @@ -13,6 +13,9 @@ import { ScreenshotItem } from '../../src/screenshot-item'; describe('ScreenshotStore', () => { const pngBase64 = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA'; + const webpBody = + 'UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; + const webpBase64 = `data:image/webp;base64,${webpBody}`; let tmpRoot: string; beforeEach(() => { @@ -44,6 +47,27 @@ describe('ScreenshotStore', () => { expect(store.loadBase64(ref)).toContain('data:image/png;base64,'); }); + it('persists and restores WebP with its MIME type and extension', async () => { + const reportPath = join(tmpRoot, 'index.html'); + const screenshotsDir = join(tmpRoot, 'screenshots'); + const item = ScreenshotItem.create(webpBase64, 100); + const store = new ScreenshotStore({ + mode: 'directory', + reportPath, + screenshotsDir, + }); + + const ref = await store.persist(item); + const filePath = join(screenshotsDir, `${item.id}.webp`); + + expect(ref).toMatchObject({ + mimeType: 'image/webp', + path: `./screenshots/${item.id}.webp`, + }); + expect(readFileSync(filePath).toString('base64')).toBe(webpBody); + expect(store.loadBase64(ref)).toBe(webpBase64); + }); + it('deduplicates same screenshot persistence by id', async () => { const reportPath = join(tmpRoot, 'index.html'); const screenshotsDir = join(tmpRoot, 'screenshots'); diff --git a/packages/playground/src/mjpeg-stream-handler.ts b/packages/playground/src/mjpeg-stream-handler.ts index db105b4751..55a15d7c1f 100644 --- a/packages/playground/src/mjpeg-stream-handler.ts +++ b/packages/playground/src/mjpeg-stream-handler.ts @@ -1,5 +1,9 @@ import http from 'node:http'; import type { Agent as PageAgent } from '@midscene/core/agent'; +import { + inferScreenshotImageFormatFromBase64, + screenshotImageMimeType, +} from '@midscene/shared/img'; import { getDebug } from '@midscene/shared/logger'; import type { Request, Response } from 'express'; import { @@ -29,6 +33,19 @@ function toMjpegFrameDataUrl(data: string, contentType?: string) { return `data:${contentType || 'image/jpeg'};base64,${data}`; } +function screenshotContentType(data: string): string { + const body = data.replace(/^data:image\/[^;]+;base64,/i, ''); + const format = inferScreenshotImageFormatFromBase64(body); + if (format) { + return screenshotImageMimeType(format); + } + + const dataUriMimeType = data.match( + /^data:(image\/(?:png|jpe?g|webp));base64,/i, + )?.[1]; + return dataUriMimeType?.toLowerCase().replace('jpg', 'jpeg') ?? 'image/jpeg'; +} + /** * Inputs the handler reads on every request, late-bound through callbacks * so a single handler instance can survive across device reconnects without @@ -263,7 +280,7 @@ export class MjpegStreamHandler { writeMjpegFrame(res, boundary, { data: base64, - contentType: 'image/jpeg', + contentType: screenshotContentType(base64), }); } catch (err) { if (stopped) break; diff --git a/packages/playground/src/server.ts b/packages/playground/src/server.ts index 01c8afb687..e944c8b3b0 100644 --- a/packages/playground/src/server.ts +++ b/packages/playground/src/server.ts @@ -11,7 +11,7 @@ import { } from 'node:fs'; import { stat } from 'node:fs/promises'; import type { Server } from 'node:http'; -import { basename, dirname, join, resolve, sep } from 'node:path'; +import { basename, dirname, extname, join, resolve, sep } from 'node:path'; import { fileURLToPath } from 'node:url'; import type { AgentDescribeElementAtPointResult, @@ -37,7 +37,14 @@ import { overrideAIConfig, } from '@midscene/shared/env'; import { generateElementByPoint } from '@midscene/shared/extractor'; -import { annotateRects, imageInfoOfBase64 } from '@midscene/shared/img'; +import { + annotateRects, + imageInfoOfBase64, + screenshotImageExtension, + screenshotImageFormatFromExtension, + screenshotImageFormatFromMimeType, + screenshotImageMimeType, +} from '@midscene/shared/img'; import { getDebug } from '@midscene/shared/logger'; import type { MidsceneRecorderScreenshotAssetRef, @@ -350,7 +357,7 @@ function writeRecorderAiDescribeScreenshot( const { base64, mimeType } = extractBase64Payload(imageBase64); const bytes = Buffer.from(base64, 'base64'); const sha256 = createHash('sha256').update(bytes).digest('hex'); - const extension = mimeType?.includes('jpeg') ? 'jpg' : 'png'; + const extension = recorderScreenshotAssetExtension(mimeType); const pathParts = formatRecorderAiDescribeScreenshotPathParts(); const dumpRoot = resolve( getMidsceneRunSubDir('dump'), @@ -393,7 +400,11 @@ function assertRecorderScreenshotAssetId(assetId: string) { } function recorderScreenshotAssetExtension(mimeType?: string) { - return mimeType?.includes('jpeg') ? 'jpg' : 'png'; + const format = screenshotImageFormatFromMimeType(mimeType); + if (!format) { + return 'png'; + } + return format === 'jpeg' ? 'jpg' : screenshotImageExtension(format); } function recorderScreenshotAssetPath(assetId: string, mimeType?: string) { @@ -428,7 +439,7 @@ function initializeRecorderScreenshotAssetUsage() { continue; } const match = entry.name.match( - /^(.*)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.(?:png|jpg)$/i, + /^(.*)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.(?:png|jpg|webp)$/i, ); if (!match?.[1]) { continue; @@ -508,7 +519,7 @@ function readRecorderScreenshotAsset( function findRecorderScreenshotAssetPath(assetId: string) { const safeAssetId = assertRecorderScreenshotAssetId(assetId); const root = getRecorderScreenshotAssetRoot(); - for (const extension of ['png', 'jpg']) { + for (const extension of ['png', 'jpg', 'webp']) { const filePath = assertPathInsideDirectory( root, join(root, `${safeAssetId}.${extension}`), @@ -554,7 +565,7 @@ function pruneRecorderScreenshotAssetsForSession( if (!entry.isFile() || !entry.name.startsWith(prefix)) { continue; } - const assetId = entry.name.replace(/\.(?:png|jpg)$/i, ''); + const assetId = entry.name.replace(/\.(?:png|jpg|webp)$/i, ''); const filePath = join(root, entry.name); if (!retained.has(assetId)) { rmSync(filePath, { force: true }); @@ -3095,7 +3106,7 @@ class PlaygroundServer { return res.status(404).json({ error: 'Report not found or expired' }); } - const match = /^([A-Za-z0-9_-]+)\.(png|jpe?g)$/.exec( + const match = /^([A-Za-z0-9_-]+)\.(png|jpe?g|webp)$/.exec( req.params.assetName, ); if (!match) { @@ -3125,7 +3136,11 @@ class PlaygroundServer { if (commaIndex === -1) { throw new Error('Invalid screenshot data URI'); } - const mimeType = extension === 'png' ? 'image/png' : 'image/jpeg'; + const format = screenshotImageFormatFromExtension(extension); + if (!format) { + throw new Error(`Unsupported screenshot extension: ${extension}`); + } + const mimeType = screenshotImageMimeType(format); return res .type(mimeType) .send(Buffer.from(dataUri.slice(commaIndex + 1), 'base64')); @@ -3769,7 +3784,13 @@ class PlaygroundServer { .status(404) .json({ error: 'Recorder screenshot not found' }); } - res.type(filePath.endsWith('.jpg') ? 'image/jpeg' : 'image/png'); + const format = screenshotImageFormatFromExtension( + extname(filePath).slice(1), + ); + if (!format) { + throw new Error('Unsupported recorder screenshot format'); + } + res.type(screenshotImageMimeType(format)); return res.send(readFileSync(filePath)); } catch (error) { return res.status(400).json({ diff --git a/packages/playground/tests/unit/server-interact.test.ts b/packages/playground/tests/unit/server-interact.test.ts index 7c019645a2..9549493abd 100644 --- a/packages/playground/tests/unit/server-interact.test.ts +++ b/packages/playground/tests/unit/server-interact.test.ts @@ -18,12 +18,15 @@ rs.mock('@midscene/core', () => ({ const VALID_PNG_BASE64 = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAYAAACNiR0NAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAAKklEQVR4nO3MIQEAAAzDsPo3/ePhDi4CwpWxUMMXaaFH4QgLPQpHWHg6fOdROhs7ULsmAAAAAElFTkSuQmCC'; +const VALID_WEBP_BASE64 = + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; function createMockResponse() { return { statusCode: 200, body: undefined as unknown, headers: {} as Record, + contentType: undefined as string | undefined, status(code: number) { this.statusCode = code; return this; @@ -36,7 +39,8 @@ function createMockResponse() { this.body = payload; return this; }, - type() { + type(contentType: string) { + this.contentType = contentType; return this; }, setHeader(name: string, value: string) { @@ -434,7 +438,7 @@ describe('PlaygroundServer manual interaction APIs', () => { type: 'midscene_screenshot_ref', id: 'shot-1', capturedAt: 1, - mimeType: 'image/png', + mimeType: 'image/webp', storage: 'inline', }, }, @@ -443,7 +447,7 @@ describe('PlaygroundServer manual interaction APIs', () => { }, ], }; - const reportHTML = `\n\n`; + const reportHTML = `\n\n`; rs.mocked(createReadStream).mockImplementation( () => ({ @@ -497,12 +501,13 @@ describe('PlaygroundServer manual interaction APIs', () => { const screenshotResponse = createMockResponse(); await screenshotHandler( { - params: { reportId: report.id, assetName: 'shot-1.png' }, + params: { reportId: report.id, assetName: 'shot-1.webp' }, }, screenshotResponse, ); expect(Buffer.isBuffer(screenshotResponse.body)).toBe(true); expect((screenshotResponse.body as Buffer).length).toBeGreaterThan(0); + expect(screenshotResponse.contentType).toBe('image/webp'); } finally { await server.close(); } diff --git a/packages/playground/tests/unit/server-mjpeg-stream.test.ts b/packages/playground/tests/unit/server-mjpeg-stream.test.ts index b0ffddd8d1..88b715bf61 100644 --- a/packages/playground/tests/unit/server-mjpeg-stream.test.ts +++ b/packages/playground/tests/unit/server-mjpeg-stream.test.ts @@ -385,6 +385,35 @@ describe('PlaygroundServer MJPEG streaming', () => { ); }); + test('GET /mjpeg labels polling WebP bytes with image/webp', async () => { + const webpBase64 = + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; + const screenshotBase64 = rs.fn(async () => webpBase64); + const server = new PlaygroundServer({ + interface: { + interfaceType: 'web', + actionSpace: () => [], + screenshotBase64, + size: async () => ({ width: 2, height: 3 }), + }, + } as any); + + await server.launch(6128); + const mjpegHandler = getRouteHandler(server, 'get', '/mjpeg'); + const request = createMockRequest(); + const response = createMockStreamResponse(); + const streamPromise = mjpegHandler(request, response); + for (let i = 0; i < 10 && screenshotBase64.mock.calls.length === 0; i++) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + request.listeners.get('close')?.(); + await streamPromise; + + expect(response.chunks.map((chunk) => chunk.toString()).join('')).toContain( + 'Content-Type: image/webp', + ); + }); + test('GET /mjpeg falls back to screenshot polling when producer emits no initial frame', async () => { const stop = rs.fn(); const screenshotBase64 = rs.fn(async () => diff --git a/packages/shared/src/agent-tools/observation-record.ts b/packages/shared/src/agent-tools/observation-record.ts index 0326550f99..0a2d82b6d3 100644 --- a/packages/shared/src/agent-tools/observation-record.ts +++ b/packages/shared/src/agent-tools/observation-record.ts @@ -21,6 +21,11 @@ import { } from 'node:path'; import { z } from 'zod'; import { getMidsceneRunSubDir } from '../common'; +import { + screenshotImageExtension, + screenshotImageFormatFromMimeType, + screenshotImageMimeType, +} from '../img/image-format'; import type { UIObservationFrame, UIObservationRecord } from './types'; const observationRecordSchema = z @@ -34,7 +39,7 @@ const observationRecordSchema = z z .object({ path: z.string().min(1), - mimeType: z.enum(['image/png', 'image/jpeg']), + mimeType: z.enum(['image/png', 'image/jpeg', 'image/webp']), capturedAt: z.number().finite().nonnegative(), }) .strict(), @@ -172,18 +177,28 @@ export class UIObservationRecordWriter { if (this.finalized) { throw new Error('UI observation record has already been finalized'); } - const match = /^data:image\/(png|jpe?g);base64,([\s\S]+)$/i.exec(dataUrl); + const match = /^data:image\/(png|jpe?g|webp);base64,([\s\S]+)$/i.exec( + dataUrl, + ); if (!match) { - throw new Error('UI observation frame must be a PNG or JPEG data URL'); + throw new Error( + 'UI observation frame must be a PNG, JPEG, or WebP data URL', + ); } - const format = match[1].toLowerCase() === 'png' ? 'png' : 'jpeg'; - const mimeType = format === 'png' ? 'image/png' : 'image/jpeg'; + const matchedMimeType = `image/${match[1].toLowerCase()}`; + const format = screenshotImageFormatFromMimeType(matchedMimeType); + if (!format) { + throw new Error( + `UI observation frame has unsupported MIME type: ${matchedMimeType}`, + ); + } + const mimeType = screenshotImageMimeType(format); const bytes = Buffer.from(match[2], 'base64'); if (bytes.length === 0) { throw new Error('UI observation frame contains no image data'); } const digest = createHash('sha256').update(bytes).digest('hex'); - const fileName = `${digest}.${format}`; + const fileName = `${digest}.${screenshotImageExtension(format)}`; mkdirSync(this.temporaryFramesDirectory, { recursive: true }); const temporaryPath = join(this.temporaryFramesDirectory, fileName); if (!existsSync(temporaryPath)) { @@ -302,7 +317,13 @@ export function writeUIObservationRecord( let relativePath = copiedPaths.get(sourcePath); if (!relativePath) { - const imageExtension = frame.mimeType === 'image/png' ? 'png' : 'jpeg'; + const format = screenshotImageFormatFromMimeType(frame.mimeType); + if (!format) { + throw new Error( + `Invalid UI observation record at frames.${index}.mimeType: unsupported image MIME type ${frame.mimeType}`, + ); + } + const imageExtension = screenshotImageExtension(format); const fileName = `${String(copiedPaths.size).padStart(4, '0')}.${imageExtension}`; relativePath = `${basename(framesDirectory)}/${fileName}`; copyFileSync(sourcePath, join(temporaryFramesDirectory, fileName)); diff --git a/packages/shared/src/agent-tools/types.ts b/packages/shared/src/agent-tools/types.ts index 6c970f7a69..8a84fd6f0c 100644 --- a/packages/shared/src/agent-tools/types.ts +++ b/packages/shared/src/agent-tools/types.ts @@ -1,4 +1,5 @@ import type { z } from 'zod'; +import type { ScreenshotImageMimeType } from '../img/image-format'; import type { ToolDefaults } from './tool-defaults'; // Avoid circular dependency: don't import from @midscene/core @@ -102,7 +103,7 @@ export type UserPromptLike = export interface RecordToReportScreenshot { /** - * PNG/JPEG data URI, or raw PNG base64 body. + * PNG/JPEG/WebP data URI, or raw PNG/JPEG/WebP base64 body. */ base64: string; description?: string; @@ -130,7 +131,7 @@ export interface UIObservationFrame { /** Path to the captured image. Serialized manifests use relative paths. */ path: string; /** MIME type of the captured image file. */ - mimeType: 'image/png' | 'image/jpeg'; + mimeType: ScreenshotImageMimeType; /** Capture timestamp in milliseconds. */ capturedAt: number; } diff --git a/packages/shared/src/cli/screenshot-file.ts b/packages/shared/src/cli/screenshot-file.ts index 4ecb0d38eb..db843c3041 100644 --- a/packages/shared/src/cli/screenshot-file.ts +++ b/packages/shared/src/cli/screenshot-file.ts @@ -1,6 +1,11 @@ import { existsSync, mkdirSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { + type ScreenshotImageFormat, + screenshotImageFormatFromExtension, + screenshotImageFormatFromMimeType, +} from '../img/image-format'; export interface WriteCliScreenshotFileOptions { id?: unknown; @@ -20,14 +25,12 @@ function safeScreenshotFilenamePart(value: unknown): string { function extensionFromImageMetadata( mimeType: unknown, extension: unknown, -): 'png' | 'jpeg' { - if (extension === 'jpeg' || extension === 'jpg') { - return 'jpeg'; +): ScreenshotImageFormat { + const extensionFormat = screenshotImageFormatFromExtension(extension); + if (extensionFormat) { + return extensionFormat; } - if (extension === 'png') { - return 'png'; - } - return mimeType === 'image/jpeg' ? 'jpeg' : 'png'; + return screenshotImageFormatFromMimeType(mimeType) ?? 'png'; } export function writeCliScreenshotFile( diff --git a/packages/shared/src/cli/verbose-screenshot.ts b/packages/shared/src/cli/verbose-screenshot.ts index b27d62234d..f77bfc621a 100644 --- a/packages/shared/src/cli/verbose-screenshot.ts +++ b/packages/shared/src/cli/verbose-screenshot.ts @@ -71,7 +71,9 @@ function screenshotRawBase64(value: unknown): string | undefined { } const base64 = getStringProperty(value, 'base64'); - const match = base64?.match(/^data:image\/(?:png|jpeg|jpg);base64,(.+)$/); + const match = base64?.match( + /^data:image\/(?:png|jpeg|jpg|webp);base64,(.+)$/, + ); return match?.[1]; } diff --git a/packages/shared/tests/unit-test/observation-record.test.ts b/packages/shared/tests/unit-test/observation-record.test.ts index 9cbcd1c8ad..6e2ce019f4 100644 --- a/packages/shared/tests/unit-test/observation-record.test.ts +++ b/packages/shared/tests/unit-test/observation-record.test.ts @@ -25,6 +25,8 @@ function tempDirectory(): string { const dataUrl = (text: string) => `data:image/png;base64,${Buffer.from(text).toString('base64')}`; +const webpDataUrl = + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; describe('UIObservationRecordWriter', () => { afterEach(() => { @@ -104,7 +106,39 @@ describe('UIObservationRecordWriter', () => { join(tempDirectory(), 'record.json'), ); expect(() => writer.persistFrame('not-an-image', 100)).toThrow( - /PNG or JPEG data URL/, + /PNG, JPEG, or WebP data URL/, + ); + }); + + it('persists and exports WebP frames with WebP metadata', () => { + const directory = tempDirectory(); + const writer = new UIObservationRecordWriter( + join(directory, 'record.json'), + ); + const frame = writer.persistFrame(webpDataUrl, 100); + + expect(frame.mimeType).toBe('image/webp'); + expect(frame.path).toMatch(/\.webp$/); + expect(existsSync(writer.resolveFramePath(frame))).toBe(true); + + const record = writer.finalize([frame], { + startedAt: 50, + endedAt: 150, + shotSize: { width: 2, height: 3 }, + shrunkShotToLogicalRatio: 1, + }); + const outputPath = join(directory, 'exported-record.json'); + writeUIObservationRecord(record, outputPath); + + const serialized = JSON.parse(readFileSync(outputPath, 'utf8')); + expect(serialized.frames[0]).toEqual({ + path: 'exported-record.frames/0000.webp', + mimeType: 'image/webp', + capturedAt: 100, + }); + const resolved = readUIObservationRecord(outputPath); + expect(readFileSync(resolved.frames[0].path)).toEqual( + Buffer.from(webpDataUrl.split(',')[1], 'base64'), ); }); diff --git a/packages/shared/tests/unit-test/screenshot-file.test.ts b/packages/shared/tests/unit-test/screenshot-file.test.ts new file mode 100644 index 0000000000..cc04a42dab --- /dev/null +++ b/packages/shared/tests/unit-test/screenshot-file.test.ts @@ -0,0 +1,62 @@ +import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from '@rstest/core'; +import { writeCliScreenshotFile } from '../../src/cli/screenshot-file'; +import { collectScreenshotRefs } from '../../src/cli/verbose-screenshot'; + +const webpBase64 = + 'UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; + +describe('CLI WebP screenshot files', () => { + const temporaryDirectories: string[] = []; + + const makeTemporaryDirectory = () => { + const directory = mkdtempSync(join(tmpdir(), 'midscene-webp-')); + temporaryDirectories.push(directory); + return directory; + }; + + afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it('uses the .webp extension from screenshot MIME metadata', () => { + const directoryPath = makeTemporaryDirectory(); + const filePath = writeCliScreenshotFile(webpBase64, { + id: 'webp-shot', + mimeType: 'image/webp', + directoryPath, + }); + + expect(filePath).toBe(join(directoryPath, 'webp-shot.webp')); + expect(readFileSync(filePath).toString('base64')).toBe(webpBase64); + }); + + it('exports inline WebP screenshots for verbose output', () => { + const directoryPath = makeTemporaryDirectory(); + const screenshot = { + base64: `data:image/webp;base64,${webpBase64}`, + extension: 'webp', + toSerializable: () => ({ + type: 'midscene_screenshot_ref' as const, + id: 'inline-webp', + capturedAt: 1, + mimeType: 'image/webp', + storage: 'inline', + }), + }; + + const [collected] = collectScreenshotRefs(screenshot, { + exportMode: 'report', + reportFile: join(directoryPath, 'report.html'), + }); + + expect(collected.file).toBe('inline-webp.webp'); + expect( + existsSync(join(directoryPath, 'screenshots', 'inline-webp.webp')), + ).toBe(true); + }); +}); diff --git a/packages/visualizer/src/hooks/usePlaygroundExecution.ts b/packages/visualizer/src/hooks/usePlaygroundExecution.ts index a8bf30b974..d359dce74a 100644 --- a/packages/visualizer/src/hooks/usePlaygroundExecution.ts +++ b/packages/visualizer/src/hooks/usePlaygroundExecution.ts @@ -10,6 +10,10 @@ import { parseImageScripts, restoreImageReferences, } from '@midscene/core/dump'; +import { + screenshotImageExtension, + screenshotImageFormatFromMimeType, +} from '@midscene/shared/img'; import { useCallback } from 'react'; import { useEnvConfig } from '../store/store'; import type { @@ -154,7 +158,11 @@ async function loadReportReplay( } const dump = (await response.json()) as IReportActionDump; result.dump = restoreImageReferences(dump, (ref) => { - const extension = ref.mimeType === 'image/jpeg' ? 'jpeg' : 'png'; + const format = screenshotImageFormatFromMimeType(ref.mimeType); + if (!format) { + throw new Error(`Unsupported screenshot mime type: ${ref.mimeType}`); + } + const extension = screenshotImageExtension(format); return new URL( `screenshots/${encodeURIComponent(ref.id)}.${extension}`, result.report!.url, diff --git a/packages/visualizer/tests/playground-execution-stop.test.ts b/packages/visualizer/tests/playground-execution-stop.test.ts index 088f1ebe48..389efe72fe 100644 --- a/packages/visualizer/tests/playground-execution-stop.test.ts +++ b/packages/visualizer/tests/playground-execution-stop.test.ts @@ -97,7 +97,7 @@ function Harness({ return null; } -function replayDump() { +function replayDump(mimeType: 'image/png' | 'image/webp' = 'image/png') { return { sdkVersion: 'test', groupName: 'Playground run', @@ -115,7 +115,7 @@ function replayDump() { type: 'midscene_screenshot_ref', id: 'shot-1', capturedAt: 1, - mimeType: 'image/png', + mimeType, storage: 'inline', }, }, @@ -322,7 +322,7 @@ describe('usePlaygroundExecution stop handling', () => { const fetchMock = rs.fn(async () => Promise.resolve({ ok: true, - json: async () => replayDump(), + json: async () => replayDump('image/webp'), }), ); rs.stubGlobal('fetch', fetchMock); @@ -377,7 +377,7 @@ describe('usePlaygroundExecution stop handling', () => { const restoredDump = allScriptsFromDumpMock.mock.calls[0]?.[0] as any; expect( restoredDump.executions[0].tasks[0].uiContext.screenshot.base64, - ).toBe('http://localhost/reports/report-1/screenshots/shot-1.png'); + ).toBe('http://localhost/reports/report-1/screenshots/shot-1.webp'); await act(async () => root.unmount()); }); From b2bf215d1cfdb39686fd9b0dc6f0ada321e10d54 Mon Sep 17 00:00:00 2001 From: quanru Date: Thu, 20 Aug 2026 22:43:32 +0800 Subject: [PATCH 2/4] test(core): keep format-aware consumers independently green Co-authored-by: quanruzhuoxiu --- packages/core/tests/unit-test/agent-dump-update.test.ts | 2 +- .../tests/unit-test/common-context-parser-orientation.test.ts | 4 ++-- .../unit-test/common-context-parser-shrink-factor.test.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/core/tests/unit-test/agent-dump-update.test.ts b/packages/core/tests/unit-test/agent-dump-update.test.ts index 9f9bda4066..31e6379b19 100644 --- a/packages/core/tests/unit-test/agent-dump-update.test.ts +++ b/packages/core/tests/unit-test/agent-dump-update.test.ts @@ -255,7 +255,7 @@ describe('Agent dump update screenshot serialization', () => { screenshots: [{ base64: 'data:image/svg+xml;base64,custom' }], }), ).rejects.toThrow( - 'recordToReport: screenshot #1 base64 must be a PNG/JPEG data URI or raw PNG base64 string', + 'recordToReport: screenshot #1 base64 must be a PNG/JPEG/WebP data URI or raw PNG/JPEG/WebP base64 string', ); expect(screenshotBase64).not.toHaveBeenCalled(); diff --git a/packages/core/tests/unit-test/common-context-parser-orientation.test.ts b/packages/core/tests/unit-test/common-context-parser-orientation.test.ts index a7bc23f5de..6ecbbff1aa 100644 --- a/packages/core/tests/unit-test/common-context-parser-orientation.test.ts +++ b/packages/core/tests/unit-test/common-context-parser-orientation.test.ts @@ -3,8 +3,8 @@ import type { AbstractInterface } from '@/device'; import { beforeEach, describe, expect, it, vi } from 'vitest'; // Mock imageInfoOfBase64 to control screenshot dimensions -vi.mock('@midscene/shared/img', () => ({ - createImgBase64ByFormat: vi.fn(), +vi.mock('@midscene/shared/img', async (importOriginal) => ({ + ...(await importOriginal()), imageInfoOfBase64: vi.fn(), resizeBase64ImageToJpeg: vi .fn() diff --git a/packages/core/tests/unit-test/common-context-parser-shrink-factor.test.ts b/packages/core/tests/unit-test/common-context-parser-shrink-factor.test.ts index 2b9ca7f747..0cbc28fb72 100644 --- a/packages/core/tests/unit-test/common-context-parser-shrink-factor.test.ts +++ b/packages/core/tests/unit-test/common-context-parser-shrink-factor.test.ts @@ -2,8 +2,8 @@ import { commonContextParser } from '@/agent/utils'; import type { AbstractInterface } from '@/device'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -vi.mock('@midscene/shared/img', () => ({ - createImgBase64ByFormat: vi.fn(), +vi.mock('@midscene/shared/img', async (importOriginal) => ({ + ...(await importOriginal()), imageInfoOfBase64: vi.fn(), resizeBase64ImageToJpeg: vi .fn() From 63e1a3075fc3dd22991e8e25ea63394b645afdeb Mon Sep 17 00:00:00 2001 From: quanru Date: Wed, 26 Aug 2026 17:48:50 +0800 Subject: [PATCH 3/4] fix(core): enforce screenshot format consistency Validate encoded screenshot bytes at screenshot boundaries. Avoid eager reads when a screenshot file is already persisted. Co-authored-by: quanruzhuoxiu --- apps/report/src/App.tsx | 14 ++-- .../src/components/playground/index.tsx | 4 - .../timeline/build-timeline-screenshots.ts | 16 ++-- .../src/utils/screenshot-source.test.ts | 6 ++ apps/report/src/utils/screenshot-source.ts | 33 ++------ packages/core/src/agent/utils.ts | 25 +----- packages/core/src/dump/screenshot-store.ts | 12 ++- packages/core/src/screenshot-item.ts | 61 ++++++--------- .../unit-test/agent-context-retry.test.ts | 7 +- .../tests/unit-test/agent-dump-update.test.ts | 11 +-- .../tests/unit-test/agent-ui-observer.test.ts | 18 ++++- .../common-context-parser-orientation.test.ts | 4 +- ...ommon-context-parser-shrink-factor.test.ts | 9 ++- .../dump-screenshot-sequence.test.ts | 16 ++-- .../tests/unit-test/execution-dump.test.ts | 4 +- .../unit-test/llm-planning-retry.test.ts | 2 +- .../unit-test/merge-browser-parse.test.ts | 3 +- .../model-adapter/auto-glm/locate.test.ts | 6 +- .../auto-glm/planning-messages.test.ts | 2 +- .../model-adapter/auto-glm/planning.test.ts | 2 +- .../ui-tars/planning-response-parser.test.ts | 2 +- .../model-adapter/ui-tars/planning.test.ts | 2 +- .../unit-test/planning-tap-locator.test.ts | 6 +- .../unit-test/recorder-frame-sequence.test.ts | 9 ++- .../core/tests/unit-test/report-cli.test.ts | 3 +- .../tests/unit-test/report-issues.test.ts | 3 +- .../tests/unit-test/report-markdown.test.ts | 12 +-- .../unit-test/report-merge-count.test.ts | 3 +- .../unit-test/report-merge-status.test.ts | 3 +- .../core/tests/unit-test/report-split.test.ts | 3 +- .../tests/unit-test/screenshot-item.test.ts | 6 ++ .../unit-test/task-runner-log-time.test.ts | 5 +- .../tests/unit-test/task-runner/index.test.ts | 5 +- .../tests/unit-test/tasks-null-data.test.ts | 9 ++- .../test-helpers/report-generator.ts | 8 +- .../core/tests/unit-test/ui-observer.test.ts | 10 ++- .../src/agent-tools/observation-record.ts | 26 ++----- packages/shared/src/cli/screenshot-file.ts | 39 ++++++---- packages/shared/src/img/base64.ts | 78 +++++++++++++------ packages/shared/src/img/index.ts | 2 + packages/shared/src/img/transform.ts | 2 + .../shared/tests/unit-test/cli-runner.test.ts | 14 +++- .../unit-test/image/screenshot-base64.test.ts | 43 ++++++++++ .../unit-test/observation-record.test.ts | 25 +++++- .../tests/unit-test/screenshot-file.test.ts | 14 ++++ .../tests/unit-test/tool-generator.test.ts | 15 +++- .../shared/tests/unit-test/transform.test.ts | 25 +++--- 47 files changed, 381 insertions(+), 246 deletions(-) create mode 100644 packages/shared/tests/unit-test/image/screenshot-base64.test.ts diff --git a/apps/report/src/App.tsx b/apps/report/src/App.tsx index c8b0d2425c..cf8704318d 100644 --- a/apps/report/src/App.tsx +++ b/apps/report/src/App.tsx @@ -18,6 +18,7 @@ import { reportToMarkdown, restoreImageReferences, } from '@midscene/core'; +import type { StoredImageRef } from '@midscene/core/dump'; import { antiEscapeScriptTag } from '@midscene/shared/utils'; import { Logo, @@ -55,18 +56,13 @@ import { getEmptyDumpDescription, parseDumpAttributes, } from './utils/report-dump'; -import { - type ReportScreenshotSourceRef, - resolveScreenshotFallbackPath, -} from './utils/screenshot-source'; +import { resolveScreenshotFallbackPath } from './utils/screenshot-source'; // Shared image cache across all test cases — resolved images are cached by id const imageCache = new Map(); -function resolveImageFromDom( - refOrId: string | ReportScreenshotSourceRef, -): string { - const id = typeof refOrId === 'string' ? refOrId : refOrId.id; +function resolveImageFromDom(ref: StoredImageRef): string { + const id = ref.id; const cached = imageCache.get(id); if (cached) return cached; @@ -79,7 +75,7 @@ function resolveImageFromDom( return data; } - return resolveScreenshotFallbackPath(refOrId); + return resolveScreenshotFallbackPath(ref); } let globalRenderCount = 1; diff --git a/apps/report/src/components/playground/index.tsx b/apps/report/src/components/playground/index.tsx index 58e2498ae6..ee1cc6542a 100644 --- a/apps/report/src/components/playground/index.tsx +++ b/apps/report/src/components/playground/index.tsx @@ -16,10 +16,6 @@ import { } from '@midscene/core/dump'; import { type PlaygroundSDK, noReplayAPIs } from '@midscene/playground'; import type { ServerResponse } from '@midscene/playground'; -import { - screenshotImageExtension, - screenshotImageFormatFromMimeType, -} from '@midscene/shared/img'; import { ContextPreview, Logo, diff --git a/apps/report/src/components/timeline/build-timeline-screenshots.ts b/apps/report/src/components/timeline/build-timeline-screenshots.ts index e9a71c3462..b6bce9d06e 100644 --- a/apps/report/src/components/timeline/build-timeline-screenshots.ts +++ b/apps/report/src/components/timeline/build-timeline-screenshots.ts @@ -1,8 +1,5 @@ import type { ExecutionTask } from '@midscene/core'; -import { - inferScreenshotImageFormatFromBase64, - screenshotImageMimeType, -} from '@midscene/shared/img'; +import { parseScreenshotBase64 } from '@midscene/shared/img'; export interface TimelineScreenshot { id: string; @@ -27,16 +24,17 @@ const rawBase64BodyPattern = /^[a-zA-Z0-9+/=\s]+$/; const imageSrcFromString = (value: string): string => { const trimmed = value.trim(); if (trimmed.startsWith('data:')) { - return trimmed; + return parseScreenshotBase64(trimmed, { + label: 'Report timeline screenshot', + }).dataUrl; } if (trimmed.length < 32 || !rawBase64BodyPattern.test(trimmed)) { return value; } - const body = trimmed.replace(/\s/g, ''); - const format = inferScreenshotImageFormatFromBase64(body) ?? 'png'; - const mimeType = screenshotImageMimeType(format); - return `data:${mimeType};base64,${body}`; + return parseScreenshotBase64(trimmed, { + label: 'Report timeline screenshot', + }).dataUrl; }; const screenshotBase64 = (screenshot: unknown): string | undefined => { diff --git a/apps/report/src/utils/screenshot-source.test.ts b/apps/report/src/utils/screenshot-source.test.ts index 6dd91a6f45..126bf1691a 100644 --- a/apps/report/src/utils/screenshot-source.test.ts +++ b/apps/report/src/utils/screenshot-source.test.ts @@ -5,14 +5,18 @@ describe('resolveScreenshotFallbackPath', () => { it('uses MIME-specific extensions for screenshot references', () => { expect( resolveScreenshotFallbackPath({ + type: 'midscene_screenshot_ref', id: 'webp-shot', + capturedAt: 1, mimeType: 'image/webp', storage: 'inline', }), ).toBe('./screenshots/webp-shot.webp'); expect( resolveScreenshotFallbackPath({ + type: 'midscene_screenshot_ref', id: 'jpeg-shot', + capturedAt: 1, mimeType: 'image/jpeg', storage: 'inline', }), @@ -22,7 +26,9 @@ describe('resolveScreenshotFallbackPath', () => { it('prefers an explicit file-backed path', () => { expect( resolveScreenshotFallbackPath({ + type: 'midscene_screenshot_ref', id: 'webp-shot', + capturedAt: 1, mimeType: 'image/webp', storage: 'file', path: './assets/custom.webp', diff --git a/apps/report/src/utils/screenshot-source.ts b/apps/report/src/utils/screenshot-source.ts index 7cec1d16d3..6546ca09c1 100644 --- a/apps/report/src/utils/screenshot-source.ts +++ b/apps/report/src/utils/screenshot-source.ts @@ -1,31 +1,12 @@ import { - screenshotImageExtension, - screenshotImageFormatFromMimeType, -} from '@midscene/shared/img'; + type StoredImageRef, + imageFileExtensionForMimeType, +} from '@midscene/core/dump'; -export interface ReportScreenshotSourceRef { - id: string; - mimeType?: unknown; - storage?: 'inline' | 'file'; - path?: string; -} - -export function resolveScreenshotFallbackPath( - refOrId: string | ReportScreenshotSourceRef, -): string { - if ( - typeof refOrId === 'object' && - refOrId.storage === 'file' && - refOrId.path - ) { - return refOrId.path; +export function resolveScreenshotFallbackPath(ref: StoredImageRef): string { + if (ref.storage === 'file' && ref.path) { + return ref.path; } - const id = typeof refOrId === 'string' ? refOrId : refOrId.id; - const format = - typeof refOrId === 'object' - ? screenshotImageFormatFromMimeType(refOrId.mimeType) - : undefined; - const extension = format ? screenshotImageExtension(format) : 'png'; - return `./screenshots/${id}.${extension}`; + return `./screenshots/${ref.id}.${imageFileExtensionForMimeType(ref.mimeType)}`; } diff --git a/packages/core/src/agent/utils.ts b/packages/core/src/agent/utils.ts index 8c8a9a6309..4481713951 100644 --- a/packages/core/src/agent/utils.ts +++ b/packages/core/src/agent/utils.ts @@ -23,8 +23,8 @@ import { } from '@midscene/shared/env'; import { generateElementByRect } from '@midscene/shared/extractor'; import { - createImgBase64ByFormat, imageInfoOfBase64, + normalizeScreenshotBase64, } from '@midscene/shared/img'; import { getDebug } from '@midscene/shared/logger'; import { _keyDefinitions } from '@midscene/shared/us-keyboard-layout'; @@ -35,29 +35,6 @@ import type { TaskCache } from './task-cache'; import { debug as cacheDebug } from './task-cache'; const agentDebug = getDebug('agent'); -const screenshotDataUrlPattern = /^data:image\/[a-zA-Z0-9.+-]+;base64,/i; - -const inferBase64ImageFormat = (base64Body: string) => { - if (base64Body.startsWith('iVBORw0KGgo')) { - return 'png'; - } - return 'jpeg'; -}; - -const normalizeScreenshotBase64 = (screenshotBase64: string) => { - const trimmedBase64 = screenshotBase64.trim(); - if (screenshotDataUrlPattern.test(trimmedBase64)) { - return trimmedBase64; - } - - const base64Body = trimmedBase64.replace(/\s/g, ''); - assert(base64Body, 'screenshotBase64 must include image data'); - return createImgBase64ByFormat( - inferBase64ImageFormat(base64Body), - base64Body, - ); -}; - const legacyScrollTypeMap = { once: 'singleAction', untilBottom: 'scrollToBottom', diff --git a/packages/core/src/dump/screenshot-store.ts b/packages/core/src/dump/screenshot-store.ts index a4eeebc9cf..07cb407932 100644 --- a/packages/core/src/dump/screenshot-store.ts +++ b/packages/core/src/dump/screenshot-store.ts @@ -255,7 +255,11 @@ export class ReportImageStore { const shouldWriteFileCopy = this.mode === 'directory' || this.alsoWriteFileCopy; const fileLocation = shouldWriteFileCopy - ? await this.writeImageFileIfNeeded({ id, extension, rawBase64 }) + ? await this.writeImageFileIfNeeded({ + id, + extension, + readRawBase64: () => rawBase64, + }) : null; let ref: ImageUrlRef; @@ -327,7 +331,7 @@ export class ReportImageStore { const { relativePath, absolutePath } = await this.writeImageFileIfNeeded({ id: screenshot.id, extension: screenshot.extension, - rawBase64: screenshot.rawBase64, + readRawBase64: () => screenshot.rawBase64, }); if (options.markAsPersisted) { @@ -340,7 +344,7 @@ export class ReportImageStore { private async writeImageFileIfNeeded(image: { id: string; extension: string; - rawBase64: string; + readRawBase64: () => string; }): Promise<{ relativePath: string; absolutePath: string }> { const screenshotsDir = this.screenshotsDir; if (!screenshotsDir) { @@ -358,7 +362,7 @@ export class ReportImageStore { if (!this.writtenFileIds.has(image.id)) { await writeFileAsync( absolutePath, - Buffer.from(image.rawBase64, 'base64'), + Buffer.from(image.readRawBase64(), 'base64'), ); this.writtenFileIds.add(image.id); } diff --git a/packages/core/src/screenshot-item.ts b/packages/core/src/screenshot-item.ts index a1d393a3fa..32b0eed3be 100644 --- a/packages/core/src/screenshot-item.ts +++ b/packages/core/src/screenshot-item.ts @@ -2,9 +2,8 @@ import { readFileSync } from 'node:fs'; import { type ScreenshotImageFormat, type ScreenshotImageMimeType, - inferScreenshotImageFormatFromBase64, + parseScreenshotBase64, screenshotImageExtension, - screenshotImageFormatFromMimeType, screenshotImageMimeType, } from '@midscene/shared/img'; import { uuid } from '@midscene/shared/utils'; @@ -21,31 +20,6 @@ import { */ export type ScreenshotSerializeFormat = ScreenshotRef; -/** - * Detect image format from base64 data URI prefix. - */ -const BASE64_SEPARATOR = ';base64,'; - -function detectFormat(base64: string): ScreenshotImageFormat { - const separatorIndex = base64.indexOf(BASE64_SEPARATOR); - const mimeType = - separatorIndex === -1 ? undefined : base64.slice(5, separatorIndex); - const detectedFormat = - separatorIndex === -1 - ? inferScreenshotImageFormatFromBase64(base64) - : screenshotImageFormatFromMimeType(mimeType); - return detectedFormat ?? 'png'; -} - -function rawBase64Body(base64: string): string { - const separatorIndex = base64.indexOf(BASE64_SEPARATOR); - const body = - separatorIndex === -1 - ? base64 - : base64.slice(separatorIndex + BASE64_SEPARATOR.length); - return body.replace(/\s/g, ''); -} - /** * ScreenshotItem encapsulates screenshot data. * @@ -79,7 +53,15 @@ export class ScreenshotItem { /** Create a new ScreenshotItem from base64 data */ static create(base64: string, capturedAt: number): ScreenshotItem { - return new ScreenshotItem(uuid(), base64, capturedAt, detectFormat(base64)); + const parsed = parseScreenshotBase64(base64, { + label: 'ScreenshotItem base64', + }); + return new ScreenshotItem( + uuid(), + parsed.dataUrl, + capturedAt, + parsed.format, + ); } /** Create a lazily loaded ScreenshotItem backed by an image file. */ @@ -88,12 +70,12 @@ export class ScreenshotItem { mimeType: ScreenshotImageMimeType, capturedAt: number, ): ScreenshotItem { - const item = new ScreenshotItem( - uuid(), - null, - capturedAt, - screenshotImageFormatFromMimeType(mimeType)!, + const fileBytes = readFileSync(filePath); + const parsed = parseScreenshotBase64( + `data:${mimeType};base64,${fileBytes.toString('base64')}`, + { label: `Screenshot file ${filePath}` }, ); + const item = new ScreenshotItem(uuid(), null, capturedAt, parsed.format); item._persistedPath = filePath; return item; } @@ -133,7 +115,10 @@ export class ScreenshotItem { throw new Error(`Screenshot ${this._id}: file recovery path missing`); } const buffer = readFileSync(this._persistedPath); - return `data:${this.mimeType};base64,${buffer.toString('base64')}`; + return parseScreenshotBase64( + `data:${this.mimeType};base64,${buffer.toString('base64')}`, + { label: `Screenshot ${this._id} persisted file` }, + ).dataUrl; }; const loadFromInline = (): string => { @@ -142,7 +127,9 @@ export class ScreenshotItem { } const data = extractImageByIdSync(this._persistedHtmlPath, this._id); if (data) { - return data; + return parseScreenshotBase64(data, { + label: `Screenshot ${this._id} persisted HTML image`, + }).dataUrl; } throw new Error( `Screenshot ${this._id}: cannot recover from HTML (id not found in ${this._persistedHtmlPath})`, @@ -263,6 +250,8 @@ export class ScreenshotItem { * Useful for writing raw binary data to files. */ get rawBase64(): string { - return rawBase64Body(this.base64); + return parseScreenshotBase64(this.base64, { + label: `Screenshot ${this._id}`, + }).body; } } diff --git a/packages/core/tests/unit-test/agent-context-retry.test.ts b/packages/core/tests/unit-test/agent-context-retry.test.ts index 7b75c68b50..c948569754 100644 --- a/packages/core/tests/unit-test/agent-context-retry.test.ts +++ b/packages/core/tests/unit-test/agent-context-retry.test.ts @@ -29,7 +29,10 @@ const modelConfig = { }; const fakeUIContext = { - screenshot: ScreenshotItem.create('', Date.now()), + screenshot: ScreenshotItem.create( + 'data:image/png;base64,iVBORw0KGgoAAAAA', + Date.now(), + ), shotSize: { width: 1280, height: 720 }, shrunkShotToLogicalRatio: 1, } as unknown as UIContext; @@ -40,7 +43,7 @@ function createMockInterface() { actionSpace: () => [], describe: () => 'test page', size: async () => ({ width: 1280, height: 720 }), - screenshotBase64: async () => '', + screenshotBase64: async () => 'data:image/png;base64,iVBORw0KGgoAAAAA', } as any; } diff --git a/packages/core/tests/unit-test/agent-dump-update.test.ts b/packages/core/tests/unit-test/agent-dump-update.test.ts index e8e99e978c..c834643a6f 100644 --- a/packages/core/tests/unit-test/agent-dump-update.test.ts +++ b/packages/core/tests/unit-test/agent-dump-update.test.ts @@ -27,7 +27,8 @@ function createMockInterface() { } function createLargeBase64DataUri(byteSize: number): string { - const payload = 'A'.repeat(byteSize); + const signature = 'iVBORw0KGgoAAAAA'; + const payload = `${signature}${'A'.repeat(Math.max(0, byteSize - signature.length))}`; return `data:image/png;base64,${payload}`; } @@ -162,8 +163,8 @@ describe('Agent dump update screenshot serialization', () => { (agent as any).reportGenerator = reportGeneratorStub; - const beforeScreenshot = 'before'; - const afterScreenshot = 'data:image/jpg;base64,after'; + const beforeScreenshot = 'iVBORw0KGgoAAAAA'; + const afterScreenshot = 'data:image/jpg;base64,/9j/4AAQSkZJRgAB'; await agent.recordToReport('comparison', { content: 'before and after state', screenshots: [ @@ -189,8 +190,8 @@ describe('Agent dump update screenshot serialization', () => { 'After click', ]); expect(task.recorder?.map((item) => item.screenshot?.base64)).toEqual([ - 'data:image/png;base64,before', - 'data:image/jpeg;base64,after', + 'data:image/png;base64,iVBORw0KGgoAAAAA', + 'data:image/jpeg;base64,/9j/4AAQSkZJRgAB', ]); expect(task.recorder?.[0].ts ?? 0).toBeLessThan(task.recorder?.[1].ts ?? 0); diff --git a/packages/core/tests/unit-test/agent-ui-observer.test.ts b/packages/core/tests/unit-test/agent-ui-observer.test.ts index e858392468..5fc8c5c039 100644 --- a/packages/core/tests/unit-test/agent-ui-observer.test.ts +++ b/packages/core/tests/unit-test/agent-ui-observer.test.ts @@ -159,7 +159,13 @@ describe('Agent.startObserving', () => { tempDirectories.push(directory); const framePaths = ['before', 'toast', 'after'].map((name) => { const path = join(directory, `${name}.png`); - writeFileSync(path, Buffer.from(name)); + writeFileSync( + path, + Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + Buffer.from(name), + ]), + ); return path; }); @@ -196,7 +202,7 @@ describe('Agent.startObserving', () => { expect( executionOptions.uiContext.screenshotSequence.map( (frame: ScreenshotItem) => - Buffer.from(frame.rawBase64, 'base64').toString(), + Buffer.from(frame.rawBase64, 'base64').subarray(8).toString(), ), ).toEqual(['before', 'toast', 'after']); @@ -245,7 +251,13 @@ describe('Agent.startObserving', () => { const directory = mkdtempSync(join(tmpdir(), 'midscene-adapter-record-')); tempDirectories.push(directory); const framePath = join(directory, 'frame.png'); - writeFileSync(framePath, Buffer.from('frame')); + writeFileSync( + framePath, + Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + Buffer.from('frame'), + ]), + ); const record: UIObservationRecord = { type: 'midscene_ui_observation', version: 1, diff --git a/packages/core/tests/unit-test/common-context-parser-orientation.test.ts b/packages/core/tests/unit-test/common-context-parser-orientation.test.ts index 6ecbbff1aa..1c9de642c6 100644 --- a/packages/core/tests/unit-test/common-context-parser-orientation.test.ts +++ b/packages/core/tests/unit-test/common-context-parser-orientation.test.ts @@ -8,7 +8,7 @@ vi.mock('@midscene/shared/img', async (importOriginal) => ({ imageInfoOfBase64: vi.fn(), resizeBase64ImageToJpeg: vi .fn() - .mockResolvedValue('data:image/jpeg;base64,mock-resized-base64-data'), + .mockResolvedValue('data:image/jpeg;base64,/9j/4AAQ'), })); import { imageInfoOfBase64 } from '@midscene/shared/img'; @@ -22,7 +22,7 @@ function createMockInterface( return { screenshotBase64: vi .fn() - .mockResolvedValue('data:image/jpeg;base64,mock-base64-data'), + .mockResolvedValue('data:image/jpeg;base64,/9j/4AAQ'), size: vi .fn() .mockResolvedValue({ width: logicalWidth, height: logicalHeight }), diff --git a/packages/core/tests/unit-test/common-context-parser-shrink-factor.test.ts b/packages/core/tests/unit-test/common-context-parser-shrink-factor.test.ts index 0cbc28fb72..846258b2e3 100644 --- a/packages/core/tests/unit-test/common-context-parser-shrink-factor.test.ts +++ b/packages/core/tests/unit-test/common-context-parser-shrink-factor.test.ts @@ -7,7 +7,7 @@ vi.mock('@midscene/shared/img', async (importOriginal) => ({ imageInfoOfBase64: vi.fn(), resizeBase64ImageToJpeg: vi .fn() - .mockResolvedValue('data:image/jpeg;base64,mock-resized-base64-data'), + .mockResolvedValue('data:image/jpeg;base64,/9j/4AAQ'), })); import { @@ -15,7 +15,8 @@ import { resizeBase64ImageToJpeg, } from '@midscene/shared/img'; -const mockScreenshotBase64 = 'data:image/png;base64,mock-base64-data'; +const mockScreenshotBase64 = + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA'; const mockedImageInfo = vi.mocked(imageInfoOfBase64); const mockedResizeToJpeg = vi.mocked(resizeBase64ImageToJpeg); @@ -41,7 +42,7 @@ describe('commonContextParser screenshotShrinkFactor', () => { it('converts PNG screenshots to JPEG quality 90 when not shrinking', async () => { const mockInterface = createMockInterface(800, 400); mockedImageInfo.mockResolvedValue({ width: 2400, height: 1200 }); - mockedResizeToJpeg.mockResolvedValue('data:image/jpeg;base64,jpeg-image'); + mockedResizeToJpeg.mockResolvedValue('data:image/jpeg;base64,/9j/4AAQ'); const result = await commonContextParser(mockInterface, {}); @@ -50,7 +51,7 @@ describe('commonContextParser screenshotShrinkFactor', () => { targetSize: { width: 2400, height: 1200 }, jpegQuality: 90, }); - expect(result.screenshot.base64).toBe('data:image/jpeg;base64,jpeg-image'); + expect(result.screenshot.base64).toBe('data:image/jpeg;base64,/9j/4AAQ'); }); it('does not shrink when screenshotShrinkFactor is not provided', async () => { diff --git a/packages/core/tests/unit-test/dump-screenshot-sequence.test.ts b/packages/core/tests/unit-test/dump-screenshot-sequence.test.ts index 7dba14babe..94bdf21efb 100644 --- a/packages/core/tests/unit-test/dump-screenshot-sequence.test.ts +++ b/packages/core/tests/unit-test/dump-screenshot-sequence.test.ts @@ -15,9 +15,9 @@ import { * must still be serialized. */ describe('dump serialization drops screenshotSequence', () => { - const FRAME_A = 'data:image/png;base64,iVBORw0KGgoAAAA-FRAME-A'; - const FRAME_B = 'data:image/png;base64,iVBORw0KGgoAAAA-FRAME-B'; - const FRAME_C = 'data:image/png;base64,iVBORw0KGgoAAAA-FRAME-C'; + const FRAME_A = 'data:image/png;base64,iVBORw0KGgpGUkFNRS1B'; + const FRAME_B = 'data:image/png;base64,iVBORw0KGgpGUkFNRS1C'; + const FRAME_C = 'data:image/png;base64,iVBORw0KGgpGUkFNRS1D'; const buildExecutionDumpData = (): IExecutionDump => { const representative = ScreenshotItem.create(FRAME_C, 3); @@ -52,8 +52,8 @@ describe('dump serialization drops screenshotSequence', () => { expect(serialized).not.toContain('screenshotSequence'); // The early frames must not leak their base64 either. - expect(serialized).not.toContain('FRAME-A'); - expect(serialized).not.toContain('FRAME-B'); + expect(serialized).not.toContain(FRAME_A); + expect(serialized).not.toContain(FRAME_B); // The representative screenshot is still serialized (as a ref). expect(serialized).toContain('screenshot'); }); @@ -72,8 +72,8 @@ describe('dump serialization drops screenshotSequence', () => { expect(serialized).not.toContain('screenshotSequence'); // Inline mode would embed each frame's base64; the sequence frames must be // dropped so only the representative frame is inlined. - expect(serialized).not.toContain('FRAME-A'); - expect(serialized).not.toContain('FRAME-B'); - expect(serialized).toContain('FRAME-C'); + expect(serialized).not.toContain(FRAME_A); + expect(serialized).not.toContain(FRAME_B); + expect(serialized).toContain(FRAME_C); }); }); diff --git a/packages/core/tests/unit-test/execution-dump.test.ts b/packages/core/tests/unit-test/execution-dump.test.ts index 871d45a317..b3f03403c9 100644 --- a/packages/core/tests/unit-test/execution-dump.test.ts +++ b/packages/core/tests/unit-test/execution-dump.test.ts @@ -367,7 +367,7 @@ describe('ReportActionDump', () => { it('should preserve capturedAt in inline screenshot serialization', () => { const capturedAt = 1700000000123; const screenshot = ScreenshotItem.create( - 'data:image/png;base64,test-inline-screenshot', + 'data:image/png;base64,iVBORw0KGgoAAAAA', capturedAt, ); @@ -401,7 +401,7 @@ describe('ReportActionDump', () => { const screenshotData = parsed.executions[0].tasks[0].uiContext.screenshot; expect(screenshotData.base64).toBe( - 'data:image/png;base64,test-inline-screenshot', + 'data:image/png;base64,iVBORw0KGgoAAAAA', ); expect(screenshotData.capturedAt).toBe(capturedAt); }); diff --git a/packages/core/tests/unit-test/llm-planning-retry.test.ts b/packages/core/tests/unit-test/llm-planning-retry.test.ts index a03b098686..3b5ce15a6a 100644 --- a/packages/core/tests/unit-test/llm-planning-retry.test.ts +++ b/packages/core/tests/unit-test/llm-planning-retry.test.ts @@ -47,7 +47,7 @@ const mockModelConfig = ( const mockContext = (): UIContext => ({ screenshot: { - base64: 'data:image/png;base64,AA==', + base64: 'data:image/png;base64,iVBORw0KGgoAAAAA', }, shotSize: { width: 100, diff --git a/packages/core/tests/unit-test/merge-browser-parse.test.ts b/packages/core/tests/unit-test/merge-browser-parse.test.ts index 5266bf3da3..8904ac10b5 100644 --- a/packages/core/tests/unit-test/merge-browser-parse.test.ts +++ b/packages/core/tests/unit-test/merge-browser-parse.test.ts @@ -27,7 +27,8 @@ import { antiEscapeScriptTag, escapeScriptTag } from '@midscene/shared/utils'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; function fakeBase64(sizeBytes: number): string { - return `data:image/png;base64,${'A'.repeat(sizeBytes)}`; + const signature = 'iVBORw0KGgoAAAAA'; + return `data:image/png;base64,${signature}${'A'.repeat(Math.max(0, sizeBytes - signature.length))}`; } function createDump(screenshots: ScreenshotItem[]): ReportActionDump { diff --git a/packages/core/tests/unit-test/model-adapter/auto-glm/locate.test.ts b/packages/core/tests/unit-test/model-adapter/auto-glm/locate.test.ts index f89bee3d41..51a67b3516 100644 --- a/packages/core/tests/unit-test/model-adapter/auto-glm/locate.test.ts +++ b/packages/core/tests/unit-test/model-adapter/auto-glm/locate.test.ts @@ -48,7 +48,7 @@ const autoGlmAdapter = new ResolvedModelAdapter( const context: UIContext = { screenshot: { - base64: 'data:image/png;base64,AA==', + base64: 'data:image/png;base64,iVBORw0KGgoAAAAA', } as any, shotSize: { width: 1000, @@ -177,7 +177,7 @@ describe('Auto-GLM custom locate', () => { height: 200, }, image: { - imageBase64: 'data:image/png;base64,CROP==', + imageBase64: 'data:image/png;base64,iVBORw0KGgpDUk9Q', width: 300, height: 200, }, @@ -200,7 +200,7 @@ describe('Auto-GLM custom locate', () => { expect.objectContaining({ type: 'image_url', image_url: expect.objectContaining({ - url: 'data:image/png;base64,CROP==', + url: 'data:image/png;base64,iVBORw0KGgpDUk9Q', }), }), ]), diff --git a/packages/core/tests/unit-test/model-adapter/auto-glm/planning-messages.test.ts b/packages/core/tests/unit-test/model-adapter/auto-glm/planning-messages.test.ts index 677acea905..f30ad796ef 100644 --- a/packages/core/tests/unit-test/model-adapter/auto-glm/planning-messages.test.ts +++ b/packages/core/tests/unit-test/model-adapter/auto-glm/planning-messages.test.ts @@ -51,7 +51,7 @@ const autoGlmAdapter = new ResolvedModelAdapter( const context: UIContext = { screenshot: { - base64: 'data:image/png;base64,AA==', + base64: 'data:image/png;base64,iVBORw0KGgoAAAAA', } as any, shotSize: { width: 1000, diff --git a/packages/core/tests/unit-test/model-adapter/auto-glm/planning.test.ts b/packages/core/tests/unit-test/model-adapter/auto-glm/planning.test.ts index 8c50ec09b1..d89b9a543d 100644 --- a/packages/core/tests/unit-test/model-adapter/auto-glm/planning.test.ts +++ b/packages/core/tests/unit-test/model-adapter/auto-glm/planning.test.ts @@ -51,7 +51,7 @@ const autoGlmAdapter = new ResolvedModelAdapter( const context: UIContext = { screenshot: { - base64: 'data:image/png;base64,AA==', + base64: 'data:image/png;base64,iVBORw0KGgoAAAAA', } as any, shotSize: { width: 1000, diff --git a/packages/core/tests/unit-test/model-adapter/ui-tars/planning-response-parser.test.ts b/packages/core/tests/unit-test/model-adapter/ui-tars/planning-response-parser.test.ts index 28f098313e..c4a160645e 100644 --- a/packages/core/tests/unit-test/model-adapter/ui-tars/planning-response-parser.test.ts +++ b/packages/core/tests/unit-test/model-adapter/ui-tars/planning-response-parser.test.ts @@ -17,7 +17,7 @@ vi.mock('@ui-tars/action-parser', () => ({ const context: UIContext = { screenshot: { - base64: 'data:image/png;base64,AA==', + base64: 'data:image/png;base64,iVBORw0KGgoAAAAA', } as any, shotSize: { width: 1000, diff --git a/packages/core/tests/unit-test/model-adapter/ui-tars/planning.test.ts b/packages/core/tests/unit-test/model-adapter/ui-tars/planning.test.ts index 58a5c87a98..089eba818a 100644 --- a/packages/core/tests/unit-test/model-adapter/ui-tars/planning.test.ts +++ b/packages/core/tests/unit-test/model-adapter/ui-tars/planning.test.ts @@ -24,7 +24,7 @@ vi.mock('@/ai-model/service-caller/index', async (importOriginal) => { const context: UIContext = { screenshot: { - base64: 'data:image/png;base64,AA==', + base64: 'data:image/png;base64,iVBORw0KGgoAAAAA', } as any, shotSize: { width: 1000, diff --git a/packages/core/tests/unit-test/planning-tap-locator.test.ts b/packages/core/tests/unit-test/planning-tap-locator.test.ts index 4de80d40e3..cae75f2fde 100644 --- a/packages/core/tests/unit-test/planning-tap-locator.test.ts +++ b/packages/core/tests/unit-test/planning-tap-locator.test.ts @@ -32,7 +32,7 @@ function createLocateRequest() { const options = { context: { screenshot: ScreenshotItem.create( - 'data:image/png;base64,SCREENSHOT==', + 'data:image/png;base64,iVBORw0KGgoAAAAA', 123, ), shotSize: { @@ -59,7 +59,7 @@ function createLocateRequest() { return { elementDescriptionText: 'submit button', locateImage: { - imageBase64: 'data:image/png;base64,CROP==', + imageBase64: 'data:image/png;base64,iVBORw0KGgpDUk9Q', width: 320, height: 240, }, @@ -114,7 +114,7 @@ describe('resolvePlanningTapLocator', () => { const [, planOptions, locatorPlanner] = vi.mocked(runCustomPlanning).mock.calls[0]; expect(planOptions.context.screenshot.base64).toBe( - 'data:image/png;base64,CROP==', + 'data:image/png;base64,iVBORw0KGgpDUk9Q', ); expect(planOptions.context.screenshot.capturedAt).toBe(123); expect(planOptions.context.shotSize).toEqual({ width: 320, height: 240 }); diff --git a/packages/core/tests/unit-test/recorder-frame-sequence.test.ts b/packages/core/tests/unit-test/recorder-frame-sequence.test.ts index 52e7c7b753..0ada435335 100644 --- a/packages/core/tests/unit-test/recorder-frame-sequence.test.ts +++ b/packages/core/tests/unit-test/recorder-frame-sequence.test.ts @@ -5,7 +5,7 @@ import { describe, expect, it } from 'vitest'; const makeUiContext = (frameCount: number): UIContext => { const frames = Array.from({ length: frameCount }, (_, i) => - ScreenshotItem.create(`data:image/png;base64,iVBORw0KGgo-${i}`, 1000 + i), + ScreenshotItem.create('data:image/png;base64,iVBORw0KGgoAAAAA', 1000 + i), ); return { screenshot: frames[frameCount - 1], @@ -49,7 +49,7 @@ describe('recordAndReleaseScreenshotSequence', () => { type: 'screenshot' as const, ts: 5, screenshot: ScreenshotItem.create( - 'data:image/png;base64,iVBORw0KGgo-x', + 'data:image/png;base64,iVBORw0KGgoAAAAA', 5, ), timing: 'after-calling', @@ -68,7 +68,10 @@ describe('recordAndReleaseScreenshotSequence', () => { it('is a no-op when there is no screenshot sequence', () => { const task = makeTask(); const uiContext = { - screenshot: ScreenshotItem.create('data:image/png;base64,iVBORw0KGgo', 1), + screenshot: ScreenshotItem.create( + 'data:image/png;base64,iVBORw0KGgoAAAAA', + 1, + ), shotSize: { width: 10, height: 10 }, shrunkShotToLogicalRatio: 1, } as UIContext; diff --git a/packages/core/tests/unit-test/report-cli.test.ts b/packages/core/tests/unit-test/report-cli.test.ts index 2497db54cb..2505cde4c3 100644 --- a/packages/core/tests/unit-test/report-cli.test.ts +++ b/packages/core/tests/unit-test/report-cli.test.ts @@ -21,7 +21,8 @@ import { ScreenshotItem } from '../../src/screenshot-item'; import { ExecutionDump, ReportActionDump } from '../../src/types'; function fakeBase64(sizeBytes: number): string { - return `data:image/png;base64,${'A'.repeat(sizeBytes)}`; + const signature = 'iVBORw0KGgoAAAAA'; + return `data:image/png;base64,${signature}${'A'.repeat(Math.max(0, sizeBytes - signature.length))}`; } const webpBase64 = diff --git a/packages/core/tests/unit-test/report-issues.test.ts b/packages/core/tests/unit-test/report-issues.test.ts index 5921fc6a44..67b35e8497 100644 --- a/packages/core/tests/unit-test/report-issues.test.ts +++ b/packages/core/tests/unit-test/report-issues.test.ts @@ -23,7 +23,8 @@ import { // ---------- helpers ---------- function fakeBase64(sizeBytes: number): string { - return `data:image/png;base64,${'A'.repeat(sizeBytes)}`; + const signature = 'iVBORw0KGgoAAAAA'; + return `data:image/png;base64,${signature}${'A'.repeat(Math.max(0, sizeBytes - signature.length))}`; } function fakeScreenshot(size = 200): ScreenshotItem { diff --git a/packages/core/tests/unit-test/report-markdown.test.ts b/packages/core/tests/unit-test/report-markdown.test.ts index 8c60509882..07ccd3a0fd 100644 --- a/packages/core/tests/unit-test/report-markdown.test.ts +++ b/packages/core/tests/unit-test/report-markdown.test.ts @@ -77,7 +77,7 @@ describe('report-markdown', () => { it('handles single execution markdown with screenshot file links', () => { const screenshot = ScreenshotItem.create( - 'data:image/png;base64,Zm9v', + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA', 1710000000000, ); const execution: IExecutionDump = { @@ -221,7 +221,7 @@ describe('report-markdown', () => { it('uses timing fallback fields and custom screenshot directory', () => { const screenshot = ScreenshotItem.create( - 'data:image/png;base64,Zm9v', + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA', 1710000000000, ); @@ -448,7 +448,7 @@ describe('report-markdown', () => { taskId: 'task-recorder-string', uiContext: { screenshot: { - base64: 'data:image/png;base64,bWFpbg==', + base64: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA', capturedAt: 1710000000000, }, shotSize: { width: 800, height: 600 }, @@ -458,7 +458,7 @@ describe('report-markdown', () => { type: 'screenshot', ts: 1710000000050, timing: 'after action', - screenshot: 'data:image/png;base64,cmVjb3JkZXI=', + screenshot: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAYA', }, ], }), @@ -472,10 +472,10 @@ describe('report-markdown', () => { expect(result.markdown).toContain('timing=after action'); expect(result.attachments).toHaveLength(2); expect(result.attachments[0].base64Data).toBe( - 'data:image/png;base64,bWFpbg==', + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA', ); expect(result.attachments[1].base64Data).toBe( - 'data:image/png;base64,cmVjb3JkZXI=', + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAYA', ); expect(result.attachments[1].suggestedFileName).toContain('recorder-1'); expect(result.attachments[1].suggestedFileName).not.toBe( diff --git a/packages/core/tests/unit-test/report-merge-count.test.ts b/packages/core/tests/unit-test/report-merge-count.test.ts index d059cb7b1b..ad2b04095c 100644 --- a/packages/core/tests/unit-test/report-merge-count.test.ts +++ b/packages/core/tests/unit-test/report-merge-count.test.ts @@ -25,8 +25,9 @@ import { describe, expect, it } from 'vitest'; // ---------- helpers ---------- function fakeScreenshot(size = 200): ScreenshotItem { + const signature = 'iVBORw0KGgoAAAAA'; return ScreenshotItem.create( - `data:image/png;base64,${'A'.repeat(size)}`, + `data:image/png;base64,${signature}${'A'.repeat(Math.max(0, size - signature.length))}`, Date.now(), ); } diff --git a/packages/core/tests/unit-test/report-merge-status.test.ts b/packages/core/tests/unit-test/report-merge-status.test.ts index 946b87b464..b2677caf62 100644 --- a/packages/core/tests/unit-test/report-merge-status.test.ts +++ b/packages/core/tests/unit-test/report-merge-status.test.ts @@ -27,7 +27,8 @@ import { } from '../../src/types'; function fakeBase64(sizeBytes: number): string { - return `data:image/png;base64,${'A'.repeat(sizeBytes)}`; + const signature = 'iVBORw0KGgoAAAAA'; + return `data:image/png;base64,${signature}${'A'.repeat(Math.max(0, sizeBytes - signature.length))}`; } type TaskShape = { diff --git a/packages/core/tests/unit-test/report-split.test.ts b/packages/core/tests/unit-test/report-split.test.ts index d60400eef9..1960aeebdd 100644 --- a/packages/core/tests/unit-test/report-split.test.ts +++ b/packages/core/tests/unit-test/report-split.test.ts @@ -18,7 +18,8 @@ import { ScreenshotItem } from '../../src/screenshot-item'; import { ExecutionDump, ReportActionDump } from '../../src/types'; function fakeBase64(sizeBytes: number): string { - return `data:image/png;base64,${'A'.repeat(sizeBytes)}`; + const signature = 'iVBORw0KGgoAAAAA'; + return `data:image/png;base64,${signature}${'A'.repeat(Math.max(0, sizeBytes - signature.length))}`; } const webpBase64 = diff --git a/packages/core/tests/unit-test/screenshot-item.test.ts b/packages/core/tests/unit-test/screenshot-item.test.ts index c4d2c42a33..4fbb9275d6 100644 --- a/packages/core/tests/unit-test/screenshot-item.test.ts +++ b/packages/core/tests/unit-test/screenshot-item.test.ts @@ -34,6 +34,12 @@ describe('ScreenshotItem', () => { mimeType: 'image/webp', }); }); + + it('rejects a declared MIME type that disagrees with encoded bytes', () => { + expect(() => + ScreenshotItem.create(webpBase64.replace('image/webp', 'image/png'), 1), + ).toThrow('declares image/png but encoded bytes are image/webp'); + }); }); describe('base64 getter', () => { diff --git a/packages/core/tests/unit-test/task-runner-log-time.test.ts b/packages/core/tests/unit-test/task-runner-log-time.test.ts index 05f42c5ae7..8df23006f0 100644 --- a/packages/core/tests/unit-test/task-runner-log-time.test.ts +++ b/packages/core/tests/unit-test/task-runner-log-time.test.ts @@ -4,7 +4,10 @@ import type { UIContext } from '@/types'; import { describe, expect, it } from 'vitest'; const fakeUIContextBuilder = async () => { - const screenshot = ScreenshotItem.create('', Date.now()); + const screenshot = ScreenshotItem.create( + 'data:image/png;base64,iVBORw0KGgoAAAAA', + Date.now(), + ); return { screenshot, tree: { node: null, children: [] }, diff --git a/packages/core/tests/unit-test/task-runner/index.test.ts b/packages/core/tests/unit-test/task-runner/index.test.ts index 02b21c71fb..3b8005d74a 100644 --- a/packages/core/tests/unit-test/task-runner/index.test.ts +++ b/packages/core/tests/unit-test/task-runner/index.test.ts @@ -66,7 +66,10 @@ const insightFindTask = (shouldThrow?: boolean) => { }; const fakeUIContextBuilder = async () => { - const screenshot = ScreenshotItem.create('', Date.now()); + const screenshot = ScreenshotItem.create( + 'data:image/png;base64,iVBORw0KGgoAAAAA', + Date.now(), + ); return { screenshot, tree: { node: null, children: [] }, diff --git a/packages/core/tests/unit-test/tasks-null-data.test.ts b/packages/core/tests/unit-test/tasks-null-data.test.ts index 3da5906ebb..cc4c451d85 100644 --- a/packages/core/tests/unit-test/tasks-null-data.test.ts +++ b/packages/core/tests/unit-test/tasks-null-data.test.ts @@ -16,7 +16,9 @@ vi.mock('@/ai-model/workflows/planning', async (importOriginal) => { }); // Helper function to create mock UIContext with ScreenshotItem -const createMockUIContext = async (screenshotData = 'mock-screenshot') => { +const createMockUIContext = async ( + screenshotData = 'data:image/png;base64,iVBORw0KGgoAAAAA', +) => { const screenshot = ScreenshotItem.create(screenshotData, Date.now()); return { screenshot, @@ -26,7 +28,10 @@ const createMockUIContext = async (screenshotData = 'mock-screenshot') => { }; const createEmptyUIContext = async () => { - const screenshot = ScreenshotItem.create('', Date.now()); + const screenshot = ScreenshotItem.create( + 'data:image/png;base64,iVBORw0KGgoAAAAA', + Date.now(), + ); return { screenshot, shotSize: { width: 0, height: 0 }, diff --git a/packages/core/tests/unit-test/test-helpers/report-generator.ts b/packages/core/tests/unit-test/test-helpers/report-generator.ts index 208168c1a3..1af878d8bf 100644 --- a/packages/core/tests/unit-test/test-helpers/report-generator.ts +++ b/packages/core/tests/unit-test/test-helpers/report-generator.ts @@ -9,7 +9,13 @@ export function fakeBase64( sizeBytes: number, format: 'png' | 'jpeg' | 'webp' = 'png', ): string { - return `data:image/${format};base64,${'A'.repeat(sizeBytes)}`; + const signature = { + png: 'iVBORw0KGgoAAAAA', + jpeg: '/9j/4AAQSkZJRgAB', + webp: 'UklGRgAAAABXRUJQ', + }[format]; + const body = `${signature}${'A'.repeat(Math.max(0, sizeBytes - signature.length))}`; + return `data:image/${format};base64,${body}`; } export const defaultReportMeta: ReportMeta = { diff --git a/packages/core/tests/unit-test/ui-observer.test.ts b/packages/core/tests/unit-test/ui-observer.test.ts index 0a20017399..b55d833b8b 100644 --- a/packages/core/tests/unit-test/ui-observer.test.ts +++ b/packages/core/tests/unit-test/ui-observer.test.ts @@ -274,7 +274,10 @@ describe('UIObserver', () => { for (let index = 0; index < 12; index++) { const persisted = writer.persistFrame( - `data:image/png;base64,${Buffer.from(`frame-${index}`).toString('base64')}`, + `data:image/png;base64,${Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + Buffer.from(`frame-${index}`), + ]).toString('base64')}`, index, ); (observer as any).pushFrame({ @@ -299,7 +302,10 @@ describe('UIObserver', () => { const { deps } = makeDeps(fake); const observer = new UIObserverImpl(deps, options({ intervalMs: 200 })); const persisted = deps.observationRecordWriter.persistFrame( - `data:image/png;base64,${Buffer.from('frame').toString('base64')}`, + `data:image/png;base64,${Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + Buffer.from('frame'), + ]).toString('base64')}`, 100, ); (observer as any).pushFrame({ diff --git a/packages/shared/src/agent-tools/observation-record.ts b/packages/shared/src/agent-tools/observation-record.ts index 0a2d82b6d3..6241f9d6ea 100644 --- a/packages/shared/src/agent-tools/observation-record.ts +++ b/packages/shared/src/agent-tools/observation-record.ts @@ -21,10 +21,10 @@ import { } from 'node:path'; import { z } from 'zod'; import { getMidsceneRunSubDir } from '../common'; +import { parseScreenshotBase64 } from '../img/base64'; import { screenshotImageExtension, screenshotImageFormatFromMimeType, - screenshotImageMimeType, } from '../img/image-format'; import type { UIObservationFrame, UIObservationRecord } from './types'; @@ -177,26 +177,10 @@ export class UIObservationRecordWriter { if (this.finalized) { throw new Error('UI observation record has already been finalized'); } - const match = /^data:image\/(png|jpe?g|webp);base64,([\s\S]+)$/i.exec( - dataUrl, - ); - if (!match) { - throw new Error( - 'UI observation frame must be a PNG, JPEG, or WebP data URL', - ); - } - const matchedMimeType = `image/${match[1].toLowerCase()}`; - const format = screenshotImageFormatFromMimeType(matchedMimeType); - if (!format) { - throw new Error( - `UI observation frame has unsupported MIME type: ${matchedMimeType}`, - ); - } - const mimeType = screenshotImageMimeType(format); - const bytes = Buffer.from(match[2], 'base64'); - if (bytes.length === 0) { - throw new Error('UI observation frame contains no image data'); - } + const parsed = parseScreenshotBase64(dataUrl, { + label: 'UI observation frame', + }); + const { bytes, format, mimeType } = parsed; const digest = createHash('sha256').update(bytes).digest('hex'); const fileName = `${digest}.${screenshotImageExtension(format)}`; mkdirSync(this.temporaryFramesDirectory, { recursive: true }); diff --git a/packages/shared/src/cli/screenshot-file.ts b/packages/shared/src/cli/screenshot-file.ts index db843c3041..f85412795f 100644 --- a/packages/shared/src/cli/screenshot-file.ts +++ b/packages/shared/src/cli/screenshot-file.ts @@ -1,8 +1,8 @@ import { existsSync, mkdirSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { parseScreenshotBase64 } from '../img/base64'; import { - type ScreenshotImageFormat, screenshotImageFormatFromExtension, screenshotImageFormatFromMimeType, } from '../img/image-format'; @@ -22,25 +22,36 @@ function safeScreenshotFilenamePart(value: unknown): string { return text.replace(/[^a-zA-Z0-9._-]/g, '_') || 'shot'; } -function extensionFromImageMetadata( - mimeType: unknown, - extension: unknown, -): ScreenshotImageFormat { - const extensionFormat = screenshotImageFormatFromExtension(extension); - if (extensionFormat) { - return extensionFormat; - } - return screenshotImageFormatFromMimeType(mimeType) ?? 'png'; -} - export function writeCliScreenshotFile( rawBase64: string, options: WriteCliScreenshotFileOptions = {}, ): string { - const extension = extensionFromImageMetadata( + const parsed = parseScreenshotBase64(rawBase64, { + label: 'CLI screenshot', + }); + const declaredMimeFormat = screenshotImageFormatFromMimeType( options.mimeType, + ); + const declaredExtensionFormat = screenshotImageFormatFromExtension( options.extension, ); + if (options.mimeType !== undefined && !declaredMimeFormat) { + throw new Error(`Unsupported screenshot MIME type: ${options.mimeType}`); + } + if (options.extension !== undefined && !declaredExtensionFormat) { + throw new Error(`Unsupported screenshot extension: ${options.extension}`); + } + for (const [label, declaredFormat] of [ + ['MIME type', declaredMimeFormat], + ['extension', declaredExtensionFormat], + ] as const) { + if (declaredFormat && declaredFormat !== parsed.format) { + throw new Error( + `CLI screenshot ${label} describes ${declaredFormat}, but encoded bytes are ${parsed.format}`, + ); + } + } + const extension = parsed.extension; const directory = options.directoryPath ? options.directoryPath : options.directoryName @@ -57,7 +68,7 @@ export function writeCliScreenshotFile( const filePath = join(directory, filename); if (options.overwrite !== false || !existsSync(filePath)) { - writeFileSync(filePath, Buffer.from(rawBase64, 'base64')); + writeFileSync(filePath, parsed.bytes); } return filePath; diff --git a/packages/shared/src/img/base64.ts b/packages/shared/src/img/base64.ts index 31c25c71d1..7ab2810da0 100644 --- a/packages/shared/src/img/base64.ts +++ b/packages/shared/src/img/base64.ts @@ -2,15 +2,17 @@ import assert from 'node:assert'; import { Buffer } from 'node:buffer'; import { type ScreenshotImageFormat, + type ScreenshotImageMimeType, detectScreenshotImageFormatFromBuffer, inferScreenshotImageFormatFromBase64, + screenshotImageExtension, + screenshotImageFormatFromMimeType, screenshotImageMimeType, } from './image-format'; const base64ImageDataUrlPattern = /^data:image\/[a-zA-Z0-9.+-]+;base64,/i; const supportedScreenshotDataUriPattern = /^data:image\/(png|jpe?g|webp);base64,([\s\S]*)$/i; -const rawBase64BodyPattern = /^[A-Za-z0-9+/=\s]+$/; export type JpegBase64DataUrl = `data:image/jpeg;base64,${string}`; export type WebpBase64DataUrl = `data:image/webp;base64,${string}`; @@ -19,6 +21,16 @@ export interface NormalizeScreenshotBase64Options { label?: string; } +export interface ParsedScreenshotBase64 { + /** Canonical data URL whose MIME type matches the encoded bytes. */ + dataUrl: string; + body: string; + bytes: Buffer; + format: ScreenshotImageFormat; + mimeType: ScreenshotImageMimeType; + extension: ScreenshotImageFormat; +} + export const normalizeBase64Body = (body: string) => body.replace(/\s/g, ''); export function detectImageMimeTypeFromBuffer( @@ -46,45 +58,63 @@ export const createImgBase64ByFormat = (format: string, body: string) => { return `data:image/${format};base64,${normalizeBase64Body(body)}`; }; -export const normalizeScreenshotBase64 = ( +/** + * Parse a screenshot from a data URL or raw Base64 body. + * + * The encoded bytes are authoritative. A declared MIME type must describe the + * same format; unsupported, empty, malformed, or mismatched inputs throw. + */ +export function parseScreenshotBase64( base64: string, options?: NormalizeScreenshotBase64Options, -) => { +): ParsedScreenshotBase64 { const label = options?.label ?? 'screenshot base64'; - const trimmedBase64 = base64.trim(); - if (!trimmedBase64) { + if (typeof base64 !== 'string' || !base64.trim()) { throw new Error(`${label} cannot be empty`); } + const trimmedBase64 = base64.trim(); const dataUriMatch = trimmedBase64.match(supportedScreenshotDataUriPattern); - if (dataUriMatch) { - const imageFormat: ScreenshotImageFormat = - dataUriMatch[1].toLowerCase() === 'jpg' - ? 'jpeg' - : (dataUriMatch[1].toLowerCase() as ScreenshotImageFormat); - const body = dataUriMatch[2]; - if (!normalizeBase64Body(body)) { - throw new Error(`${label} cannot be empty`); - } - return createImgBase64ByFormat(imageFormat, body); - } - - if (trimmedBase64.startsWith('data:')) { + if (!dataUriMatch && trimmedBase64.startsWith('data:')) { throw new Error( `${label} must be a PNG/JPEG/WebP data URI or raw PNG/JPEG/WebP base64 string`, ); } - if (!rawBase64BodyPattern.test(trimmedBase64)) { + const declaredFormat = dataUriMatch + ? screenshotImageFormatFromMimeType(`image/${dataUriMatch[1]}`) + : undefined; + const body = normalizeBase64Body(dataUriMatch?.[2] ?? trimmedBase64); + if (!body || !/^[A-Za-z0-9+/]*={0,2}$/.test(body) || body.length % 4 === 1) { + throw new Error(`${label} contains invalid base64 image data`); + } + + const bytes = Buffer.from(body, 'base64'); + const format = detectScreenshotImageFormatFromBuffer(bytes); + if (!format) { + throw new Error(`${label} does not contain a PNG, JPEG, or WebP image`); + } + if (declaredFormat && declaredFormat !== format) { throw new Error( - `${label} must be a PNG/JPEG/WebP data URI or raw PNG/JPEG/WebP base64 string`, + `${label} declares ${screenshotImageMimeType(declaredFormat)} but encoded bytes are ${screenshotImageMimeType(format)}`, ); } - const base64Body = normalizeBase64Body(trimmedBase64); - const inferredFormat = inferScreenshotImageFormatFromBase64(base64Body); - return createImgBase64ByFormat(inferredFormat ?? 'png', base64Body); -}; + const mimeType = screenshotImageMimeType(format); + return { + dataUrl: createImgBase64ByFormat(format, body), + body, + bytes, + format, + mimeType, + extension: screenshotImageExtension(format), + }; +} + +export const normalizeScreenshotBase64 = ( + base64: string, + options?: NormalizeScreenshotBase64Options, +) => parseScreenshotBase64(base64, options).dataUrl; export const normalizeBase64Image = (base64: string) => { const trimmedBase64 = base64.trim(); diff --git a/packages/shared/src/img/index.ts b/packages/shared/src/img/index.ts index c3903d179e..926ad0ab0e 100644 --- a/packages/shared/src/img/index.ts +++ b/packages/shared/src/img/index.ts @@ -44,7 +44,9 @@ export { inferBase64ImageFormat, normalizeBase64Image, normalizeScreenshotBase64, + parseScreenshotBase64, type NormalizeScreenshotBase64Options, + type ParsedScreenshotBase64, type ConstrainBase64ImageToMaxSizeOptions, type JpegBase64DataUrl, type WebpBase64DataUrl, diff --git a/packages/shared/src/img/transform.ts b/packages/shared/src/img/transform.ts index 1e94fea3ea..5704116e57 100644 --- a/packages/shared/src/img/transform.ts +++ b/packages/shared/src/img/transform.ts @@ -44,6 +44,7 @@ export { export { type JpegBase64DataUrl, type NormalizeScreenshotBase64Options, + type ParsedScreenshotBase64, type WebpBase64DataUrl, createImgBase64ByFormat, inferBase64ImageFormat, @@ -51,6 +52,7 @@ export { normalizeBase64Image, normalizeScreenshotBase64, parseBase64, + parseScreenshotBase64, } from './base64'; export { DEFAULT_JPEG_SCREENSHOT_QUALITY, diff --git a/packages/shared/tests/unit-test/cli-runner.test.ts b/packages/shared/tests/unit-test/cli-runner.test.ts index e95484ead0..34ef8a4248 100644 --- a/packages/shared/tests/unit-test/cli-runner.test.ts +++ b/packages/shared/tests/unit-test/cli-runner.test.ts @@ -413,8 +413,18 @@ describe('runToolsCLI', () => { }); it('writes image tool results with an extension matching the mime type', async () => { + const jpegData = Buffer.concat([ + Buffer.from([0xff, 0xd8, 0xff]), + Buffer.from('hello'), + ]); const handler = rs.fn().mockResolvedValue({ - content: [{ type: 'image', data: 'aGVsbG8=', mimeType: 'image/jpeg' }], + content: [ + { + type: 'image', + data: jpegData.toString('base64'), + mimeType: 'image/jpeg', + }, + ], isError: false, }); const tools = createMockTools([{ name: 'take_screenshot', handler }]); @@ -429,7 +439,7 @@ describe('runToolsCLI', () => { const screenshotPath = message?.replace('Screenshot saved: ', ''); expect(screenshotPath).toBeDefined(); expect(existsSync(screenshotPath!)).toBe(true); - expect(readFileSync(screenshotPath!, 'utf8')).toBe('hello'); + expect(readFileSync(screenshotPath!)).toEqual(jpegData); consoleSpy.mockRestore(); }); diff --git a/packages/shared/tests/unit-test/image/screenshot-base64.test.ts b/packages/shared/tests/unit-test/image/screenshot-base64.test.ts new file mode 100644 index 0000000000..f3d69ea0e6 --- /dev/null +++ b/packages/shared/tests/unit-test/image/screenshot-base64.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from '@rstest/core'; +import { + localImg2Base64, + normalizeScreenshotBase64, + parseScreenshotBase64, +} from '../../../src/img'; +import { getFixture } from '../../utils'; + +describe('parseScreenshotBase64', () => { + it('uses encoded bytes as the canonical screenshot format', () => { + const webp = + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; + + expect(parseScreenshotBase64(webp)).toMatchObject({ + dataUrl: webp, + format: 'webp', + mimeType: 'image/webp', + extension: 'webp', + }); + }); + + it('rejects a MIME type that disagrees with the encoded bytes', () => { + const webpBody = + 'UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; + + expect(() => + parseScreenshotBase64(`data:image/png;base64,${webpBody}`), + ).toThrow('declares image/png but encoded bytes are image/webp'); + }); + + it('rejects unsupported bytes instead of defaulting to PNG', () => { + expect(() => + normalizeScreenshotBase64(Buffer.from('not an image').toString('base64')), + ).toThrow('does not contain a PNG, JPEG, or WebP image'); + }); + + it('canonicalizes a raw JPEG body without trusting a file extension', () => { + const jpeg = localImg2Base64(getFixture('heytea.jpeg')); + const body = jpeg.split(',')[1]; + + expect(normalizeScreenshotBase64(body)).toBe(jpeg); + }); +}); diff --git a/packages/shared/tests/unit-test/observation-record.test.ts b/packages/shared/tests/unit-test/observation-record.test.ts index 6e2ce019f4..65c9891e0e 100644 --- a/packages/shared/tests/unit-test/observation-record.test.ts +++ b/packages/shared/tests/unit-test/observation-record.test.ts @@ -23,8 +23,14 @@ function tempDirectory(): string { return directory; } +const pngSignature = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, +]); const dataUrl = (text: string) => - `data:image/png;base64,${Buffer.from(text).toString('base64')}`; + `data:image/png;base64,${Buffer.concat([ + pngSignature, + Buffer.from(text), + ]).toString('base64')}`; const webpDataUrl = 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; @@ -68,7 +74,9 @@ describe('UIObservationRecordWriter', () => { const resolved = readUIObservationRecord(outputPath); expect(isAbsolute(resolved.frames[0].path)).toBe(true); expect(existsSync(resolved.frames[0].path)).toBe(true); - expect(readFileSync(resolved.frames[0].path, 'utf8')).toBe('same-frame'); + expect(readFileSync(resolved.frames[0].path)).toEqual( + Buffer.from(dataUrl('same-frame').split(',')[1], 'base64'), + ); }); it('rejects missing, absolute, and escaping image paths', () => { @@ -106,7 +114,18 @@ describe('UIObservationRecordWriter', () => { join(tempDirectory(), 'record.json'), ); expect(() => writer.persistFrame('not-an-image', 100)).toThrow( - /PNG, JPEG, or WebP data URL/, + /UI observation frame/, + ); + }); + + it('rejects frame MIME metadata that disagrees with encoded bytes', () => { + const writer = new UIObservationRecordWriter( + join(tempDirectory(), 'record.json'), + ); + const mismatched = webpDataUrl.replace('image/webp', 'image/png'); + + expect(() => writer.persistFrame(mismatched, 100)).toThrow( + 'declares image/png but encoded bytes are image/webp', ); }); diff --git a/packages/shared/tests/unit-test/screenshot-file.test.ts b/packages/shared/tests/unit-test/screenshot-file.test.ts index cc04a42dab..750b917d3e 100644 --- a/packages/shared/tests/unit-test/screenshot-file.test.ts +++ b/packages/shared/tests/unit-test/screenshot-file.test.ts @@ -35,6 +35,20 @@ describe('CLI WebP screenshot files', () => { expect(readFileSync(filePath).toString('base64')).toBe(webpBase64); }); + it('rejects metadata that disagrees with screenshot bytes', () => { + const directoryPath = makeTemporaryDirectory(); + + expect(() => + writeCliScreenshotFile(webpBase64, { + id: 'mislabeled-shot', + mimeType: 'image/png', + extension: 'png', + directoryPath, + }), + ).toThrow('encoded bytes are webp'); + expect(existsSync(join(directoryPath, 'mislabeled-shot.png'))).toBe(false); + }); + it('exports inline WebP screenshots for verbose output', () => { const directoryPath = makeTemporaryDirectory(); const screenshot = { diff --git a/packages/shared/tests/unit-test/tool-generator.test.ts b/packages/shared/tests/unit-test/tool-generator.test.ts index 7aebfab0d9..fb5b73b904 100644 --- a/packages/shared/tests/unit-test/tool-generator.test.ts +++ b/packages/shared/tests/unit-test/tool-generator.test.ts @@ -1692,6 +1692,12 @@ describe('toolDefaults (deep locate / deep think)', () => { }); it('exports inline verbose dump screenshots to readable file paths', async () => { + const exportedScreenshotPath = join( + '/tmp', + 'screenshots', + 'inline-shot-1.png', + ); + rmSync(exportedScreenshotPath, { force: true }); let dumpListener: | ((dump: string, executionDump?: unknown) => void) | undefined; @@ -1699,9 +1705,13 @@ describe('toolDefaults (deep locate / deep think)', () => { | ((event: Record) => void) | undefined; const unsubscribe = rs.fn(); + const inlineScreenshotBytes = Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + Buffer.from('foo'), + ]); const inlineScreenshot = { extension: 'png', - rawBase64: 'Zm9v', + rawBase64: inlineScreenshotBytes.toString('base64'), toSerializable: () => ({ type: 'midscene_screenshot_ref', id: 'inline-shot-1', @@ -1791,10 +1801,11 @@ describe('toolDefaults (deep locate / deep think)', () => { ); expect(screenshotPath).toBeDefined(); expect(existsSync(screenshotPath!)).toBe(true); - expect(readFileSync(screenshotPath!, 'utf8')).toBe('foo'); + expect(readFileSync(screenshotPath!)).toEqual(inlineScreenshotBytes); expect(addProgressListener).toHaveBeenCalledOnce(); expect(addDumpUpdateListener).not.toHaveBeenCalled(); expect(unsubscribe).toHaveBeenCalledOnce(); + rmSync(exportedScreenshotPath, { force: true }); consoleSpy.mockRestore(); }); diff --git a/packages/shared/tests/unit-test/transform.test.ts b/packages/shared/tests/unit-test/transform.test.ts index fb780bb995..2888aae219 100644 --- a/packages/shared/tests/unit-test/transform.test.ts +++ b/packages/shared/tests/unit-test/transform.test.ts @@ -104,16 +104,21 @@ describe('normalizeBase64Image', () => { }); describe('normalizeScreenshotBase64', () => { + const webpBody = + 'UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; + it('accepts PNG, JPEG, and WebP data urls', () => { expect( - normalizeScreenshotBase64(' data:image/png;base64,aaa\r\nbbb '), - ).toBe('data:image/png;base64,aaabbb'); + normalizeScreenshotBase64( + ' data:image/png;base64,iVBORw0KGgo\r\nAAAANSUhEUgAAAAUA ', + ), + ).toBe('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA'); expect(normalizeScreenshotBase64('data:image/jpeg;base64,/9j/4AAQ')).toBe( 'data:image/jpeg;base64,/9j/4AAQ', ); expect( - normalizeScreenshotBase64('data:image/webp;base64,UklGRjQAAA=='), - ).toBe('data:image/webp;base64,UklGRjQAAA=='); + normalizeScreenshotBase64(`data:image/webp;base64,${webpBody}`), + ).toBe(`data:image/webp;base64,${webpBody}`); }); it('normalizes jpg data urls to jpeg', () => { @@ -123,8 +128,8 @@ describe('normalizeScreenshotBase64', () => { }); it('recognizes raw PNG base64', () => { - expect(normalizeScreenshotBase64(' iVBORw0KGgo aaa\r\nbbb ')).toBe( - 'data:image/png;base64,iVBORw0KGgoaaabbb', + expect(normalizeScreenshotBase64(' iVBORw0KGgo AAAANSUhEUgAAAAUA ')).toBe( + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA', ); }); @@ -135,9 +140,11 @@ describe('normalizeScreenshotBase64', () => { }); it('recognizes raw WebP base64', () => { - expect(normalizeScreenshotBase64(' UklGRjQAAA BXRUJQ ')).toBe( - 'data:image/webp;base64,UklGRjQAAABXRUJQ', - ); + expect( + normalizeScreenshotBase64( + ` ${webpBody.slice(0, 20)} ${webpBody.slice(20)} `, + ), + ).toBe(`data:image/webp;base64,${webpBody}`); }); it('uses the provided label in validation errors', () => { From 476e9f4da0d0db3a3a673e5f8c1d0bbc32609b74 Mon Sep 17 00:00:00 2001 From: Leyang Date: Thu, 27 Aug 2026 17:16:50 +0800 Subject: [PATCH 4/4] fix(web-integration): use valid placeholder screenshots --- packages/web-integration/src/agent-tools-cdp.ts | 4 ++-- packages/web-integration/src/agent-tools-puppeteer.ts | 4 ++-- packages/web-integration/src/agent-tools.ts | 7 ++----- packages/web-integration/src/common/screenshot.ts | 8 ++++++++ packages/web-integration/src/platform.ts | 4 ++-- .../tests/ai/web/static/static-page.test.ts | 5 ++++- .../tests/unit-test/playground-server.test.ts | 8 ++++++-- .../web-integration/tests/unit-test/static-page.test.ts | 3 ++- packages/web-integration/tests/unit-test/util.test.ts | 3 ++- 9 files changed, 30 insertions(+), 16 deletions(-) create mode 100644 packages/web-integration/src/common/screenshot.ts diff --git a/packages/web-integration/src/agent-tools-cdp.ts b/packages/web-integration/src/agent-tools-cdp.ts index 139463bfe3..5eac8e8d81 100644 --- a/packages/web-integration/src/agent-tools-cdp.ts +++ b/packages/web-integration/src/agent-tools-cdp.ts @@ -1,4 +1,3 @@ -import { ScreenshotItem } from '@midscene/core'; import { extractAgentBehaviorInitArgs, getAgentInitArgsSignature, @@ -24,6 +23,7 @@ import { readSavedTargetId, saveTargetId, } from './cdp-target-store'; +import { createPlaceholderScreenshot } from './common/screenshot'; import { defaultStaticPageViewportSize } from './common/viewport'; import { PuppeteerAgent } from './puppeteer'; import { StaticPage } from './static'; @@ -79,7 +79,7 @@ export class WebCdpMidsceneTools extends BaseMidsceneTools< protected createTemporaryDevice() { return new StaticPage({ - screenshot: ScreenshotItem.create('', Date.now()), + screenshot: createPlaceholderScreenshot(), shotSize: defaultStaticPageViewportSize, shrunkShotToLogicalRatio: 1, }); diff --git a/packages/web-integration/src/agent-tools-puppeteer.ts b/packages/web-integration/src/agent-tools-puppeteer.ts index 52d7c184c8..e153a4761b 100644 --- a/packages/web-integration/src/agent-tools-puppeteer.ts +++ b/packages/web-integration/src/agent-tools-puppeteer.ts @@ -3,7 +3,6 @@ import { existsSync } from 'node:fs'; import { mkdir, open, readFile, unlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { ScreenshotItem } from '@midscene/core'; import { extractAgentBehaviorInitArgs, getAgentInitArgsSignature, @@ -24,6 +23,7 @@ import { adaptWebAgentInitArgs, webAgentInitArgShape, } from './agent-init-args'; +import { createPlaceholderScreenshot } from './common/screenshot'; import { type ViewportSize, defaultPuppeteerWindowViewportSize, @@ -411,7 +411,7 @@ export class WebPuppeteerMidsceneTools extends BaseMidsceneTools< protected createTemporaryDevice() { return new StaticPage({ - screenshot: ScreenshotItem.create('', Date.now()), + screenshot: createPlaceholderScreenshot(), shotSize: this.viewport ?? defaultStaticPageViewportSize, shrunkShotToLogicalRatio: 1, }); diff --git a/packages/web-integration/src/agent-tools.ts b/packages/web-integration/src/agent-tools.ts index 65e1d54a44..cfab34ca54 100644 --- a/packages/web-integration/src/agent-tools.ts +++ b/packages/web-integration/src/agent-tools.ts @@ -1,4 +1,3 @@ -import { ScreenshotItem } from '@midscene/core'; import { extractAgentBehaviorInitArgs, getAgentInitArgsSignature, @@ -15,6 +14,7 @@ import { webAgentInitArgShape, } from './agent-init-args'; import { AgentOverChromeBridge } from './bridge-mode'; +import { createPlaceholderScreenshot } from './common/screenshot'; import { defaultStaticPageViewportSize } from './common/viewport'; import { StaticPage } from './static'; @@ -41,11 +41,8 @@ export class WebMidsceneTools extends BaseMidsceneTools< }; protected createTemporaryDevice() { - // Use require to avoid type incompatibility with DeviceAction vs ActionSpaceItem - // StaticPage.actionSpace() returns DeviceAction[] which is compatible at runtime - // Use screenshotBase64 field to avoid async ScreenshotItem.create() return new StaticPage({ - screenshot: ScreenshotItem.create('', Date.now()), + screenshot: createPlaceholderScreenshot(), shotSize: defaultStaticPageViewportSize, shrunkShotToLogicalRatio: 1, }); diff --git a/packages/web-integration/src/common/screenshot.ts b/packages/web-integration/src/common/screenshot.ts new file mode 100644 index 0000000000..fb712938e6 --- /dev/null +++ b/packages/web-integration/src/common/screenshot.ts @@ -0,0 +1,8 @@ +import { ScreenshotItem } from '@midscene/core'; + +export const transparentPixelPngBase64 = + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='; + +export function createPlaceholderScreenshot(): ScreenshotItem { + return ScreenshotItem.create(transparentPixelPngBase64, Date.now()); +} diff --git a/packages/web-integration/src/platform.ts b/packages/web-integration/src/platform.ts index d27c9186c4..0a6483f9c0 100644 --- a/packages/web-integration/src/platform.ts +++ b/packages/web-integration/src/platform.ts @@ -1,4 +1,3 @@ -import { ScreenshotItem } from '@midscene/core'; import type { Agent } from '@midscene/core/agent'; import { type AgentFactory, @@ -7,6 +6,7 @@ import { createMjpegPreviewDescriptor, definePlaygroundPlatform, } from '@midscene/playground'; +import { createPlaceholderScreenshot } from './common/screenshot'; import { StaticPage, StaticPageAgent } from './static'; export interface WebPlatformOptions { @@ -20,7 +20,7 @@ export interface WebPlatformOptions { function createDefaultWebAgent(): StaticPageAgent { const page = new StaticPage({ shotSize: { width: 800, height: 600 }, - screenshot: ScreenshotItem.create('', Date.now()), + screenshot: createPlaceholderScreenshot(), shrunkShotToLogicalRatio: 1, }); diff --git a/packages/web-integration/tests/ai/web/static/static-page.test.ts b/packages/web-integration/tests/ai/web/static/static-page.test.ts index 7cb1357c86..931b1ec1c4 100644 --- a/packages/web-integration/tests/ai/web/static/static-page.test.ts +++ b/packages/web-integration/tests/ai/web/static/static-page.test.ts @@ -41,7 +41,10 @@ describe( const page = new StaticPage({ shotSize: { width: 800, height: 600 }, shrunkShotToLogicalRatio: 1, - screenshot: ScreenshotItem.create('', Date.now()), + screenshot: ScreenshotItem.create( + contextJson.screenshotBase64, + Date.now(), + ), }); const agent = new StaticPageAgent(page); server = new PlaygroundServer(agent); diff --git a/packages/web-integration/tests/unit-test/playground-server.test.ts b/packages/web-integration/tests/unit-test/playground-server.test.ts index 8ae771cc94..8e1992ee85 100644 --- a/packages/web-integration/tests/unit-test/playground-server.test.ts +++ b/packages/web-integration/tests/unit-test/playground-server.test.ts @@ -1,6 +1,10 @@ import { ScreenshotItem } from '@midscene/core'; import { PlaygroundServer } from '@midscene/playground'; import { afterAll, beforeAll, describe, expect, it } from '@rstest/core'; +import { + createPlaceholderScreenshot, + transparentPixelPngBase64, +} from '../../src/common/screenshot'; import { StaticPage, StaticPageAgent } from '../../src/static'; describe('Playground Server', () => { @@ -10,7 +14,7 @@ describe('Playground Server', () => { const page = new StaticPage({ shotSize: { width: 800, height: 600 }, shrunkShotToLogicalRatio: 1, - screenshot: ScreenshotItem.create('', Date.now()), + screenshot: createPlaceholderScreenshot(), }); const agent = new StaticPageAgent(page); server = new PlaygroundServer(agent); @@ -46,7 +50,7 @@ describe('Playground Server', () => { }); it('updates static context with a JSON-serialized ScreenshotItem', async () => { - const screenshotBase64 = 'data:image/png;base64,abc123'; + const screenshotBase64 = transparentPixelPngBase64; const serializedScreenshot = JSON.parse( JSON.stringify(ScreenshotItem.create(screenshotBase64, Date.now())), ); diff --git a/packages/web-integration/tests/unit-test/static-page.test.ts b/packages/web-integration/tests/unit-test/static-page.test.ts index 1ec3f38b28..774adc856a 100644 --- a/packages/web-integration/tests/unit-test/static-page.test.ts +++ b/packages/web-integration/tests/unit-test/static-page.test.ts @@ -1,8 +1,9 @@ import { ScreenshotItem } from '@midscene/core'; import { describe, expect, it } from '@rstest/core'; +import { transparentPixelPngBase64 } from '../../src/common/screenshot'; import { StaticPage } from '../../src/static'; -const screenshotBase64 = 'data:image/png;base64,abc123'; +const screenshotBase64 = transparentPixelPngBase64; function createContext( screenshot: ConstructorParameters[0]['screenshot'], diff --git a/packages/web-integration/tests/unit-test/util.test.ts b/packages/web-integration/tests/unit-test/util.test.ts index 5735fc0d75..978a702d28 100644 --- a/packages/web-integration/tests/unit-test/util.test.ts +++ b/packages/web-integration/tests/unit-test/util.test.ts @@ -7,6 +7,7 @@ import { buildDetailedLocateParamAndRestParams, } from '@midscene/core/yaml'; import { describe, expect, it } from '@rstest/core'; +import { transparentPixelPngBase64 } from '../../src/common/screenshot'; describe('getKeyCommands', () => { it('should return a single key without command when no meta or control key is provided', () => { @@ -166,7 +167,7 @@ describe('buildDetailedLocateParamAndRestParams', () => { tree: { node: null, children: [] }, shotSize: { width: 800, height: 600 }, shrunkShotToLogicalRatio: 1, - screenshot: ScreenshotItem.create('mock-base64-string', Date.now()), + screenshot: ScreenshotItem.create(transparentPixelPngBase64, Date.now()), }; const options = { deepLocate: true,