diff --git a/apps/report/src/App.tsx b/apps/report/src/App.tsx index 9434cd4032..f09b140d27 100644 --- a/apps/report/src/App.tsx +++ b/apps/report/src/App.tsx @@ -54,12 +54,16 @@ import { getEmptyDumpDescription, parseDumpAttributes, } from './utils/report-dump'; +import { + type ReportScreenshotSourceRef, + resolveScreenshotFallbackPath, +} from './utils/screenshot-source'; // Shared image cache across all test cases — resolved images are cached by id const imageCache = new Map(); function resolveImageFromDom( - refOrId: string | { id: string; storage?: 'inline' | 'file'; path?: string }, + refOrId: string | ReportScreenshotSourceRef, ): string { const id = typeof refOrId === 'string' ? refOrId : refOrId.id; const cached = imageCache.get(id); @@ -74,12 +78,7 @@ function resolveImageFromDom( return data; } - if (typeof refOrId === 'object' && refOrId?.storage === 'file') { - return refOrId.path || `./screenshots/${id}.png`; - } - - // Fallback to directory path - return `./screenshots/${id}.png`; + return resolveScreenshotFallbackPath(refOrId); } let globalRenderCount = 1; diff --git a/apps/report/src/components/playground/index.tsx b/apps/report/src/components/playground/index.tsx index 3b23220cef..59a445db37 100644 --- a/apps/report/src/components/playground/index.tsx +++ b/apps/report/src/components/playground/index.tsx @@ -14,6 +14,10 @@ import { } from '@midscene/core/dump'; import { type PlaygroundSDK, noReplayAPIs } from '@midscene/playground'; import type { ServerResponse } from '@midscene/playground'; +import { + screenshotImageExtension, + screenshotImageFormatFromMimeType, +} from '@midscene/shared/img/image-format'; import { ContextPreview, Logo, @@ -55,7 +59,11 @@ async function loadReferencedReplay(result: PlaygroundResult) { } const dump = (await response.json()) as IReportActionDump; result.dump = restoreImageReferences(dump, (ref) => { - const extension = ref.mimeType === 'image/jpeg' ? 'jpeg' : 'png'; + const format = screenshotImageFormatFromMimeType(ref.mimeType); + if (!format) { + throw new Error(`Unsupported screenshot mime type: ${ref.mimeType}`); + } + const extension = screenshotImageExtension(format); return new URL( `screenshots/${encodeURIComponent(ref.id)}.${extension}`, result.report!.url, diff --git a/apps/report/src/components/timeline/build-timeline-screenshots.test.ts b/apps/report/src/components/timeline/build-timeline-screenshots.test.ts index cae0e5a947..3f85fad707 100644 --- a/apps/report/src/components/timeline/build-timeline-screenshots.test.ts +++ b/apps/report/src/components/timeline/build-timeline-screenshots.test.ts @@ -4,6 +4,8 @@ import { buildTimelineScreenshots } from './build-timeline-screenshots'; const onePixelPngBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAFgwJ/lz8yrwAAAABJRU5ErkJggg=='; +const webpBase64 = + 'UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; interface TaskFixtureOptions { id: string; @@ -103,6 +105,19 @@ describe('buildTimelineScreenshots', () => { ); }); + it('labels raw WebP recorder screenshots with the WebP MIME type', () => { + const task = makeTask({ + id: 'raw-webp-recorder', + startTs: 1000, + recorder: [{ ts: 1200, screenshot: webpBase64 }], + }); + + const { allScreenshots } = buildTimelineScreenshots([task]); + + expect(allScreenshots).toHaveLength(1); + expect(allScreenshots[0].img).toBe(`data:image/webp;base64,${webpBase64}`); + }); + it('keeps already-prefixed data URLs unchanged', () => { const dataUrl = `data:image/png;base64,${onePixelPngBase64}`; const task = makeTask({ diff --git a/apps/report/src/components/timeline/build-timeline-screenshots.ts b/apps/report/src/components/timeline/build-timeline-screenshots.ts index 093a58df47..93b5d3937b 100644 --- a/apps/report/src/components/timeline/build-timeline-screenshots.ts +++ b/apps/report/src/components/timeline/build-timeline-screenshots.ts @@ -1,4 +1,8 @@ import type { ExecutionTask } from '@midscene/core'; +import { + inferScreenshotImageFormatFromBase64, + screenshotImageMimeType, +} from '@midscene/shared/img/image-format'; export interface TimelineScreenshot { id: string; @@ -30,11 +34,8 @@ const imageSrcFromString = (value: string): string => { } const body = trimmed.replace(/\s/g, ''); - const mimeType = body.startsWith('/9j/') - ? 'image/jpeg' - : body.startsWith('UklGR') - ? 'image/webp' - : 'image/png'; + const format = inferScreenshotImageFormatFromBase64(body) ?? 'png'; + const mimeType = screenshotImageMimeType(format); return `data:${mimeType};base64,${body}`; }; diff --git a/apps/report/src/utils/markdown-export.test.ts b/apps/report/src/utils/markdown-export.test.ts index 1f9d49be17..8aee18695b 100644 --- a/apps/report/src/utils/markdown-export.test.ts +++ b/apps/report/src/utils/markdown-export.test.ts @@ -105,6 +105,27 @@ describe('markdown-export helpers', () => { ); }); + it('packages WebP data URI attachments without changing their bytes', () => { + const files = buildMarkdownArchiveFiles('# webp report', [ + { + id: 'webp-inline', + suggestedFileName: 'webp-inline.webp', + mimeType: 'image/webp', + executionIndex: 0, + taskIndex: 0, + base64Data: `data:image/webp;base64,${btoa('webp-bytes')}`, + }, + ]); + + expect(Object.keys(files).sort()).toEqual([ + 'report.md', + 'screenshots/webp-inline.webp', + ]); + expect( + new TextDecoder().decode(files['screenshots/webp-inline.webp']), + ).toBe('webp-bytes'); + }); + it('builds display items from markdown attachment names and paths', () => { const items = getMarkdownAttachmentDisplayItems([ { diff --git a/apps/report/src/utils/markdown-export.ts b/apps/report/src/utils/markdown-export.ts index 53cfb7c632..179a3e2f86 100644 --- a/apps/report/src/utils/markdown-export.ts +++ b/apps/report/src/utils/markdown-export.ts @@ -1,7 +1,7 @@ import type { MarkdownAttachment } from '@midscene/core'; const defaultScreenshotBaseDir = './screenshots'; -const dataUrlBase64Pattern = /^data:image\/(?:png|jpeg|jpg);base64,/i; +const dataUrlBase64Pattern = /^data:image\/(?:png|jpeg|jpg|webp);base64,/i; const rawBase64Pattern = /^[a-zA-Z0-9+/=\s]+$/; type MarkdownExport = { diff --git a/apps/report/src/utils/screenshot-source.test.ts b/apps/report/src/utils/screenshot-source.test.ts new file mode 100644 index 0000000000..c811d836bb --- /dev/null +++ b/apps/report/src/utils/screenshot-source.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'vitest'; +import { resolveScreenshotFallbackPath } from './screenshot-source'; + +describe('resolveScreenshotFallbackPath', () => { + it('keeps the legacy PNG fallback when only an id is available', () => { + expect(resolveScreenshotFallbackPath('legacy-shot')).toBe( + './screenshots/legacy-shot.png', + ); + }); + + it('uses the MIME-specific extension for screenshot references', () => { + expect( + resolveScreenshotFallbackPath({ + id: 'webp-shot', + mimeType: 'image/webp', + storage: 'inline', + }), + ).toBe('./screenshots/webp-shot.webp'); + expect( + resolveScreenshotFallbackPath({ + id: 'jpeg-shot', + mimeType: 'image/jpeg', + storage: 'inline', + }), + ).toBe('./screenshots/jpeg-shot.jpeg'); + }); + + it('prefers an explicit file-backed path', () => { + expect( + resolveScreenshotFallbackPath({ + id: 'webp-shot', + mimeType: 'image/webp', + storage: 'file', + path: './assets/custom.webp', + }), + ).toBe('./assets/custom.webp'); + }); +}); diff --git a/apps/report/src/utils/screenshot-source.ts b/apps/report/src/utils/screenshot-source.ts new file mode 100644 index 0000000000..7aeccfae87 --- /dev/null +++ b/apps/report/src/utils/screenshot-source.ts @@ -0,0 +1,31 @@ +import { + screenshotImageExtension, + screenshotImageFormatFromMimeType, +} from '@midscene/shared/img/image-format'; + +export interface ReportScreenshotSourceRef { + id: string; + mimeType?: unknown; + storage?: 'inline' | 'file'; + path?: string; +} + +export function resolveScreenshotFallbackPath( + refOrId: string | ReportScreenshotSourceRef, +): string { + if ( + typeof refOrId === 'object' && + refOrId.storage === 'file' && + refOrId.path + ) { + return refOrId.path; + } + + const id = typeof refOrId === 'string' ? refOrId : refOrId.id; + const format = + typeof refOrId === 'object' + ? screenshotImageFormatFromMimeType(refOrId.mimeType) + : undefined; + const extension = format ? screenshotImageExtension(format) : 'png'; + return `./screenshots/${id}.${extension}`; +} diff --git a/apps/site/docs/en/api.mdx b/apps/site/docs/en/api.mdx index cd0b0273c1..eb9185ae1a 100644 --- a/apps/site/docs/en/api.mdx +++ b/apps/site/docs/en/api.mdx @@ -1240,7 +1240,7 @@ interface RecordToReportOptions { screenshotBase64?: string; screenshots?: { /** - * PNG/JPEG data URI, or raw PNG base64 body. + * PNG/JPEG/WebP data URI, or raw PNG/WebP base64 body. */ base64: string; description?: string; @@ -1258,7 +1258,7 @@ function recordToReport( - `title?: string` - Optional, the title of the screenshot, if not provided, the title will be 'untitled'. - `options?: RecordToReportOptions` - Optional, a configuration object containing: - `content?: string` - The description of the screenshot. - - `screenshots?: Array<{ base64: string; description?: string }>` - Record one or more provided screenshots under one report entry. When this option is set, Midscene will not capture another screenshot automatically. `base64` should be a PNG/JPEG data URI such as `data:image/png;base64,...`; a raw base64 body is also accepted and treated as PNG. + - `screenshots?: Array<{ base64: string; description?: string }>` - Record one or more provided screenshots under one report entry. When this option is set, Midscene will not capture another screenshot automatically. `base64` should be a PNG/JPEG/WebP data URI such as `data:image/webp;base64,...`; raw WebP base64 is detected by its signature, while other raw base64 bodies retain the existing PNG default. - Compatibility: diff --git a/apps/site/docs/zh/api.mdx b/apps/site/docs/zh/api.mdx index 4a6e496ca9..769a2ddb91 100644 --- a/apps/site/docs/zh/api.mdx +++ b/apps/site/docs/zh/api.mdx @@ -1230,7 +1230,7 @@ interface RecordToReportOptions { screenshotBase64?: string; screenshots?: { /** - * PNG/JPEG data URI，或裸 PNG base64 body。 + * PNG/JPEG/WebP data URI，或裸 PNG/WebP base64 body。 */ base64: string; description?: string; @@ -1248,7 +1248,7 @@ function recordToReport( - `title?: string` - 可选，截图的标题，如果未提供，则标题为 'untitled'。 - `options?: RecordToReportOptions` - 可选，一个配置对象，包含： - `content?: string` - 截图的描述。 - - `screenshots?: Array<{ base64: string; description?: string }>` - 在同一个报告条目下记录一张或多张传入的截图。设置该选项后，Midscene 不会再自动截图。`base64` 推荐使用 PNG/JPEG data URI，例如 `data:image/png;base64,...`；也可以传裸 base64 body，此时会按 PNG 处理。 + - `screenshots?: Array<{ base64: string; description?: string }>` - 在同一个报告条目下记录一张或多张传入的截图。设置该选项后，Midscene 不会再自动截图。`base64` 推荐使用 PNG/JPEG/WebP data URI，例如 `data:image/webp;base64,...`；裸 WebP base64 会通过文件签名识别，其他裸 base64 body 保持现有的 PNG 默认行为。 - 兼容性： diff --git a/packages/core/src/agent/model-input-recorder.ts b/packages/core/src/agent/model-input-recorder.ts new file mode 100644 index 0000000000..fd1ef37edd --- /dev/null +++ b/packages/core/src/agent/model-input-recorder.ts @@ -0,0 +1,62 @@ +import { Buffer } from 'node:buffer'; +import type { ModelRuntime } from '@/ai-model/models'; +import { ScreenshotItem } from '@/screenshot-item'; +import type { ExecutionTask } from '@/types'; +import { parseBase64 } from '@midscene/shared/img'; +import { sha256Hex } from '@midscene/shared/utils'; + +const MODEL_INPUT_TIMING = 'model-input'; + +function screenshotContentHash(imageBase64: string): string { + const { body } = parseBase64(imageBase64); + return sha256Hex(Buffer.from(body, 'base64')); +} + +/** + * Bind a model runtime to one report task so the report retains the exact + * data-URI bytes passed to the provider after padding/cropping/resizing. + */ +export function recordModelInputsForTask( + modelRuntime: ModelRuntime, + task: ExecutionTask, + sourceScreenshot?: ScreenshotItem, +): ModelRuntime { + return { + ...modelRuntime, + onModelInputImages: (images) => { + modelRuntime.onModelInputImages?.(images); + + for (const [index, imageBase64] of images.entries()) { + if (!imageBase64.startsWith('data:image/')) { + continue; + } + + const contentHash = screenshotContentHash(imageBase64); + const alreadyRecorded = task.recorder?.some( + (item) => + item.timing === MODEL_INPUT_TIMING && + item.screenshot && + screenshotContentHash(item.screenshot.base64) === contentHash, + ); + if (alreadyRecorded) { + continue; + } + + const uiScreenshot = task.uiContext?.screenshot ?? sourceScreenshot; + const screenshot = + uiScreenshot && + screenshotContentHash(uiScreenshot.base64) === contentHash + ? uiScreenshot + : ScreenshotItem.create(imageBase64, Date.now()); + const recorderItem = { + type: 'screenshot' as const, + ts: Date.now(), + screenshot, + timing: MODEL_INPUT_TIMING, + description: `Model input ${index + 1} (exact bytes, sha256: ${contentHash})`, + }; + task.recorder = [...(task.recorder ?? []), recorderItem]; + } + }, + }; +} diff --git a/packages/core/src/agent/task-builder.ts b/packages/core/src/agent/task-builder.ts index d8cdb71e46..790b489ba2 100644 --- a/packages/core/src/agent/task-builder.ts +++ b/packages/core/src/agent/task-builder.ts @@ -25,6 +25,7 @@ import { sleep } from '@/utils'; import { generateElementByRect } from '@midscene/shared/extractor'; import { getDebug } from '@midscene/shared/logger'; import { assert } from '@midscene/shared/utils'; +import { recordModelInputsForTask } from './model-input-recorder'; import type { TaskCache } from './task-cache'; import { withUsageIntent } from './usage-intent'; import { @@ -554,7 +555,11 @@ export class TaskBuilder { context: uiContext, planLocatedElement, }, - defaultModel, + recordModelInputsForTask( + defaultModel, + task, + uiContext.screenshot, + ), abortSignal, ); applyDump(locateResult.dump); diff --git a/packages/core/src/agent/tasks.ts b/packages/core/src/agent/tasks.ts index 3821f9d28a..592dd5f492 100644 --- a/packages/core/src/agent/tasks.ts +++ b/packages/core/src/agent/tasks.ts @@ -38,6 +38,7 @@ import { ServiceError, aiActProgressScope } from '@/types'; import { getDebug } from '@midscene/shared/logger'; import { assert } from '@midscene/shared/utils'; import { ExecutionSession } from './execution-session'; +import { recordModelInputsForTask } from './model-input-recorder'; import { type AgentProgressPublisher, createAiActActionReporter, @@ -587,7 +588,11 @@ export class TaskExecutor { context: planningUiContext, actionContext: param.aiActContext, actionSpace, - modelRuntime: planningModel, + modelRuntime: recordModelInputsForTask( + planningModel, + executorContext.task, + planningUiContext.screenshot, + ), conversationHistory, includeLocateInPlanning, imagesIncludeCount, @@ -917,7 +922,7 @@ export class TaskExecutor { try { extractResult = await this.service.extract( demandInput, - modelRuntime, + recordModelInputsForTask(modelRuntime, task, uiContext.screenshot), opt, extraPageDescription, multimodalPrompt, diff --git a/packages/core/src/agent/utils.ts b/packages/core/src/agent/utils.ts index 235c480a9f..087bf23804 100644 --- a/packages/core/src/agent/utils.ts +++ b/packages/core/src/agent/utils.ts @@ -23,10 +23,9 @@ import { } from '@midscene/shared/env'; import { generateElementByRect } from '@midscene/shared/extractor'; import { - convertImgBufferToJpeg, - createImgBase64ByFormat, + canonicalizeScreenshotBase64, imageInfoOfBase64, - parseBase64, + normalizeBase64Image, resizeImgBase64, } from '@midscene/shared/img'; import { getDebug } from '@midscene/shared/logger'; @@ -37,28 +36,6 @@ import type { TaskCache } from './task-cache'; import { debug as cacheDebug } from './task-cache'; const agentDebug = getDebug('agent'); -const screenshotDataUrlPattern = /^data:image\/[a-zA-Z0-9.+-]+;base64,/i; - -const inferBase64ImageFormat = (base64Body: string) => { - if (base64Body.startsWith('iVBORw0KGgo')) { - return 'png'; - } - return 'jpeg'; -}; - -const normalizeScreenshotBase64 = (screenshotBase64: string) => { - const trimmedBase64 = screenshotBase64.trim(); - if (screenshotDataUrlPattern.test(trimmedBase64)) { - return trimmedBase64; - } - - const base64Body = trimmedBase64.replace(/\s/g, ''); - assert(base64Body, 'screenshotBase64 must include image data'); - return createImgBase64ByFormat( - inferBase64ImageFormat(base64Body), - base64Body, - ); -}; const legacyScrollTypeMap = { once: 'singleAction', @@ -203,23 +180,13 @@ export async function commonContextParser( shrunkShotToLogicalRatio, }; } else { - // For screenshots that do not need shrinking, convert PNG to JPEG to reduce the image payload in model requests and reports. (Shrunk images are already JPEG.) - // This mainly covers Android's default screenshot path, which produces PNG screenshots. - // Compared with conversion on Android, centralizing it here means each platform does not need to handle screenshot formats itself, and allows future output formats such as WebP. - // Built-in paths that already output JPEG are unaffected, and custom devices that output JPEG will not be compressed again. - // The Web platform already outputs JPEG, so it does not enter this branch. Other built-in device platforms run in Node, where Sharp conversion is fast enough that its extra cost is negligible. - let outputScreenshotBase64 = screenshotBase64; - const { mimeType, body } = parseBase64(screenshotBase64); - if (mimeType.toLowerCase() === 'image/png') { - const jpegBuffer = await convertImgBufferToJpeg( - Buffer.from(body, 'base64'), - 90, - ); - outputScreenshotBase64 = createImgBase64ByFormat( - 'jpeg', - jpegBuffer.toString('base64'), - ); - } + // PNG/raw producers are encoded once as WebP before the ScreenshotItem is + // shared by model requests and reports. Native JPEG sources are preserved + // to avoid a second lossy encode for MJPEG/HDC frames. + const outputScreenshotBase64 = await canonicalizeScreenshotBase64( + screenshotBase64, + { preserveJpeg: true }, + ); return { shotSize: { @@ -242,10 +209,13 @@ export async function createScreenshotBoundUIContext( screenshotSize?: Size; }, ): Promise { - const normalizedScreenshotBase64 = - normalizeScreenshotBase64(screenshotBase64); - const actualScreenshotSize = await imageInfoOfBase64( + const normalizedScreenshotBase64 = normalizeBase64Image(screenshotBase64); + const canonicalScreenshotBase64 = await canonicalizeScreenshotBase64( normalizedScreenshotBase64, + { preserveJpeg: true }, + ); + const actualScreenshotSize = await imageInfoOfBase64( + canonicalScreenshotBase64, ); if ( opt.screenshotSize && @@ -262,7 +232,7 @@ export async function createScreenshotBoundUIContext( } return { - screenshot: ScreenshotItem.create(normalizedScreenshotBase64, Date.now()), + screenshot: ScreenshotItem.create(canonicalScreenshotBase64, Date.now()), shotSize: actualScreenshotSize, shrunkShotToLogicalRatio: 1, _isFrozen: true, diff --git a/packages/core/src/ai-model/model-adapter/types.ts b/packages/core/src/ai-model/model-adapter/types.ts index bae75251fd..d93953cee9 100644 --- a/packages/core/src/ai-model/model-adapter/types.ts +++ b/packages/core/src/ai-model/model-adapter/types.ts @@ -245,6 +245,8 @@ export interface ModelRuntime { * such as order-sensitive judging and deep-locate search-area calls). */ onUsage?: (usage: AIUsageInfo) => void; + /** Exact image URLs included in a model request, after all preprocessing. */ + onModelInputImages?: (images: readonly string[]) => void; } export interface ModelAdapterDefinition { diff --git a/packages/core/src/ai-model/service-caller/index.ts b/packages/core/src/ai-model/service-caller/index.ts index 1c02963008..71312d6ffd 100644 --- a/packages/core/src/ai-model/service-caller/index.ts +++ b/packages/core/src/ai-model/service-caller/index.ts @@ -345,6 +345,22 @@ export async function callAI( }> { const { config: modelConfig, adapter } = modelRuntime; + if (modelRuntime.onModelInputImages) { + const imageUrls = messages.flatMap((message) => { + if (!Array.isArray(message.content)) { + return []; + } + return message.content.flatMap((part) => + part.type === 'image_url' && typeof part.image_url?.url === 'string' + ? [part.image_url.url] + : [], + ); + }); + if (imageUrls.length > 0) { + modelRuntime.onModelInputImages(imageUrls); + } + } + // Stable internal ID for this call, used by the agent to deduplicate usage // across the onUsage callback and the task-dump-based collectUsageMetrics() // path when the provider does not return a request_id. diff --git a/packages/core/src/dump/report-action-dump.ts b/packages/core/src/dump/report-action-dump.ts index cb7b5cca2d..b2d4c75025 100644 --- a/packages/core/src/dump/report-action-dump.ts +++ b/packages/core/src/dump/report-action-dump.ts @@ -6,6 +6,10 @@ import { writeFileSync, } from 'node:fs'; import { join } from 'node:path'; +import { + screenshotImageExtension, + screenshotImageFormatFromMimeType, +} from '@midscene/shared/img/image-format'; import { ScreenshotItem } from '../screenshot-item'; import type { ExecutionTask, @@ -13,7 +17,7 @@ import type { IReportActionDump, } from '../types'; import { restoreImageReferences } from './screenshot-restoration'; -import { ScreenshotStore } from './screenshot-store'; +import { type ScreenshotRef, ScreenshotStore } from './screenshot-store'; /** * Replacer function for JSON serialization that handles Page, Browser objects and ScreenshotItem @@ -253,10 +257,10 @@ export class ReportActionDump implements IReportActionDump { } /** - * Serialize the dump to files with screenshots as separate PNG files. + * Serialize the dump to files with screenshots as separate image files. * Creates: * - {basePath} - dump JSON with { $screenshot: id } references - * - {basePath}.screenshots/ - PNG files + * - {basePath}.screenshots/ - screenshot image files * * @param basePath - Base path for the dump file */ @@ -296,14 +300,21 @@ export class ReportActionDump implements IReportActionDump { const dumpString = readFileSync(basePath, 'utf-8'); const screenshotsDir = `${basePath}.screenshots`; - const loadFromExecutionScreenshotDir = (id: string, mimeType: string) => { - const ext = mimeType === 'image/jpeg' ? 'jpeg' : 'png'; + const loadFromExecutionScreenshotDir = ( + id: string, + mimeType: ScreenshotRef['mimeType'], + ) => { + const format = screenshotImageFormatFromMimeType(mimeType); + if (!format) { + throw new Error(`Unsupported screenshot mime type: ${mimeType}`); + } + const ext = screenshotImageExtension(format); const filePath = join(screenshotsDir, `${id}.${ext}`); if (!existsSync(filePath)) { return ''; } const data = readFileSync(filePath); - return `data:image/${ext};base64,${data.toString('base64')}`; + return `data:${mimeType};base64,${data.toString('base64')}`; }; // Restore image references diff --git a/packages/core/src/dump/screenshot-store.ts b/packages/core/src/dump/screenshot-store.ts index 0e7c8ae78a..c25f5d08b1 100644 --- a/packages/core/src/dump/screenshot-store.ts +++ b/packages/core/src/dump/screenshot-store.ts @@ -1,6 +1,12 @@ import { existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs'; import { writeFile as writeFileAsync } from 'node:fs/promises'; import { dirname, isAbsolute, join } from 'node:path'; +import { + type ScreenshotImageMimeType, + isScreenshotImageMimeType, + screenshotImageExtension, + screenshotImageFormatFromMimeType, +} from '@midscene/shared/img/image-format'; import type { ScreenshotItem } from '../screenshot-item'; import { extractImageByIdSync } from './html-utils'; @@ -8,7 +14,7 @@ export interface ScreenshotRef { type: 'midscene_screenshot_ref'; id: string; capturedAt: number; - mimeType: 'image/png' | 'image/jpeg'; + mimeType: ScreenshotImageMimeType; storage: 'inline' | 'file'; path?: string; } @@ -22,7 +28,7 @@ export function normalizeScreenshotRef(value: unknown): ScreenshotRef | null { typeof record.id === 'string' && typeof record.capturedAt === 'number' && (record.storage === 'inline' || record.storage === 'file') && - (record.mimeType === 'image/png' || record.mimeType === 'image/jpeg') + isScreenshotImageMimeType(record.mimeType) ) { if (record.storage === 'file' && typeof record.path !== 'string') { return null; @@ -48,7 +54,11 @@ type ResolvedScreenshotSource = }; function extensionByMimeType(mimeType: ScreenshotRef['mimeType']): string { - return mimeType === 'image/jpeg' ? 'jpeg' : 'png'; + const format = screenshotImageFormatFromMimeType(mimeType); + if (!format) { + throw new Error(`Unsupported screenshot mime type: ${mimeType}`); + } + return screenshotImageExtension(format); } export function resolveScreenshotSource( diff --git a/packages/core/src/report-cli.ts b/packages/core/src/report-cli.ts index 2e12dd5100..3c8b627aee 100644 --- a/packages/core/src/report-cli.ts +++ b/packages/core/src/report-cli.ts @@ -79,7 +79,7 @@ function writeAttachmentFromReport( const resolved = resolveScreenshotSource(attachment.sourceRef ?? null, { reportPath: opts.htmlPath, fallbackId: id, - fallbackMimeType: (mimeType || 'image/png') as 'image/png' | 'image/jpeg', + fallbackMimeType: mimeType || 'image/png', }); if (resolved.type === 'data-uri') { diff --git a/packages/core/src/report-markdown.ts b/packages/core/src/report-markdown.ts index 481558ef26..bab8478aad 100644 --- a/packages/core/src/report-markdown.ts +++ b/packages/core/src/report-markdown.ts @@ -9,12 +9,19 @@ import type { ModelBrief, ReportActionDump, } from '@/types'; +import { + type ScreenshotImageFormat, + type ScreenshotImageMimeType, + normalizeScreenshotBase64, + parseBase64, + screenshotImageExtension, + screenshotImageFormatFromMimeType, + screenshotImageMimeType, +} from '@midscene/shared/img'; import type { ScreenshotRef } from './dump/screenshot-store'; import { normalizeScreenshotRef } from './dump/screenshot-store'; -const screenshotDataUrlPattern = - /^data:image\/(png|jpeg|jpg);base64,([\s\S]*)$/i; -const rawBase64BodyPattern = /^[a-zA-Z0-9+/=\s]+$/; +const screenshotDataUrlPattern = /^data:image\/(?:png|jpe?g|webp);base64,/i; const jsonContextMaxStringLength = 12_000; type ExecutionTaskWithExtraUsage = ExecutionTask & { @@ -39,7 +46,7 @@ export interface MarkdownAttachment { * write the screenshot under this name to keep links in sync. See #2392. */ suggestedFileName: string; - mimeType?: string; + mimeType?: ScreenshotImageMimeType; /** * Reference to the screenshot in the source report, used to locate the * original bytes when copying them to the exported name. Absent for in-memory @@ -448,26 +455,11 @@ function extractLocateCenter( function tryExtractBase64(screenshot: unknown): string | undefined { if (typeof screenshot === 'string') { - const trimmedScreenshot = screenshot.trim(); - const dataUrlMatch = trimmedScreenshot.match(screenshotDataUrlPattern); - if (dataUrlMatch) { - const format = dataUrlMatch[1].toLowerCase() === 'jpg' ? 'jpeg' : 'png'; - const base64Body = dataUrlMatch[2].replace(/\s/g, ''); - if (!base64Body) { - return undefined; - } - return `data:image/${format};base64,${base64Body}`; - } - - if ( - trimmedScreenshot.startsWith('data:') || - !rawBase64BodyPattern.test(trimmedScreenshot) - ) { + try { + return normalizeScreenshotBase64(screenshot); + } catch { return undefined; } - - const base64Body = trimmedScreenshot.replace(/\s/g, ''); - return base64Body ? `data:image/png;base64,${base64Body}` : undefined; } if (!screenshot || typeof screenshot !== 'object') return undefined; @@ -478,6 +470,20 @@ function tryExtractBase64(screenshot: unknown): string | undefined { return undefined; } +function screenshotMetadataFromMimeType(mimeType: unknown): { + extension: ScreenshotImageFormat; + mimeType: ScreenshotImageMimeType; +} { + const format = screenshotImageFormatFromMimeType(mimeType); + if (!format) { + throw new Error(`Unsupported screenshot mime type: ${String(mimeType)}`); + } + return { + extension: screenshotImageExtension(format), + mimeType: screenshotImageMimeType(format), + }; +} + function restoredSourceRef(screenshot: unknown): ScreenshotRef | undefined { if (!screenshot || typeof screenshot !== 'object') { return undefined; @@ -507,7 +513,7 @@ function screenshotAttachment( attachment: { id: screenshot.id, suggestedFileName, - mimeType: `image/${ext === 'jpeg' ? 'jpeg' : 'png'}`, + mimeType: screenshot.mimeType, executionIndex, taskIndex, base64Data: tryExtractBase64(screenshot), @@ -517,7 +523,7 @@ function screenshotAttachment( const ref = normalizeScreenshotRef(screenshot); if (ref) { - const ext = ref.mimeType === 'image/jpeg' ? 'jpeg' : 'png'; + const { extension: ext } = screenshotMetadataFromMimeType(ref.mimeType); const suggestedFileName = `execution-${executionIndex + 1}-task-${taskIndex + 1}-${ref.id}.${ext}`; return { markdown: `\n![${markdownLabel}](${screenshotBaseDir}/${suggestedFileName})`, @@ -535,7 +541,9 @@ function screenshotAttachment( const sourceRef = restoredSourceRef(screenshot); if (sourceRef) { - const ext = sourceRef.mimeType === 'image/jpeg' ? 'jpeg' : 'png'; + const { extension: ext } = screenshotMetadataFromMimeType( + sourceRef.mimeType, + ); const suggestedFileName = `execution-${executionIndex + 1}-task-${taskIndex + 1}-${sourceRef.id}.${ext}`; return { markdown: `\n![${markdownLabel}](${screenshotBaseDir}/${suggestedFileName})`, @@ -553,7 +561,9 @@ function screenshotAttachment( const base64 = tryExtractBase64(screenshot); if (base64) { - const ext = base64.startsWith('data:image/jpeg') ? 'jpeg' : 'png'; + const { extension: ext, mimeType } = screenshotMetadataFromMimeType( + parseBase64(base64).mimeType, + ); const idSuffix = options?.fallbackIdSuffix ? `-${options.fallbackIdSuffix}` : ''; @@ -564,7 +574,7 @@ function screenshotAttachment( attachment: { id, suggestedFileName, - mimeType: `image/${ext}`, + mimeType, executionIndex, taskIndex, base64Data: base64, diff --git a/packages/core/src/report.ts b/packages/core/src/report.ts index 0d2ffd6935..8f79fafb1b 100644 --- a/packages/core/src/report.ts +++ b/packages/core/src/report.ts @@ -10,6 +10,11 @@ import { } from 'node:fs'; import * as path from 'node:path'; import { getMidsceneRunSubDir } from '@midscene/shared/common'; +import { + type ScreenshotImageFormat, + screenshotImageExtension, + screenshotImageFormatFromMimeType, +} from '@midscene/shared/img'; import { antiEscapeScriptTag, logMsg } from '@midscene/shared/utils'; import { getReportFileName } from './agent'; import { @@ -22,6 +27,7 @@ import { streamImageScriptsToFile, } from './dump/html-utils'; import { + type ScreenshotRef, normalizeScreenshotRef, resolveScreenshotSource, } from './dump/screenshot-store'; @@ -526,10 +532,14 @@ export function collectDedupedExecutions( }; } -function extensionByMimeType(mimeType: string): 'png' | 'jpeg' { - if (mimeType === 'image/png') return 'png'; - if (mimeType === 'image/jpeg') return 'jpeg'; - throw new Error(`Unsupported screenshot mime type: ${mimeType}`); +function extensionByMimeType( + mimeType: ScreenshotRef['mimeType'], +): ScreenshotImageFormat { + const format = screenshotImageFormatFromMimeType(mimeType); + if (!format) { + throw new Error(`Unsupported screenshot mime type: ${mimeType}`); + } + return screenshotImageExtension(format); } function externalizeScreenshotsInExecution( diff --git a/packages/core/src/screenshot-item.ts b/packages/core/src/screenshot-item.ts index 631417d457..ced17b2a68 100644 --- a/packages/core/src/screenshot-item.ts +++ b/packages/core/src/screenshot-item.ts @@ -1,4 +1,12 @@ import { readFileSync } from 'node:fs'; +import { + type ScreenshotImageFormat, + type ScreenshotImageMimeType, + inferScreenshotImageFormatFromBase64, + screenshotImageExtension, + screenshotImageFormatFromMimeType, + screenshotImageMimeType, +} from '@midscene/shared/img/image-format'; import { uuid } from '@midscene/shared/utils'; import { extractImageByIdSync } from './dump/html-utils'; import { @@ -13,13 +21,32 @@ import { */ export type ScreenshotSerializeFormat = ScreenshotRef; +const BASE64_SEPARATOR = ';base64,'; + /** - * Detect image format from base64 data URI prefix. + * Detect image format from a data URI or raw base64 body. */ -function detectFormat(base64: string): 'png' | 'jpeg' { - if (base64.startsWith('data:image/jpeg')) return 'jpeg'; - if (base64.startsWith('data:image/jpg')) return 'jpeg'; - return 'png'; +function detectFormat(base64: string): ScreenshotImageFormat { + const separatorIndex = base64.indexOf(BASE64_SEPARATOR); + const mimeType = + separatorIndex === -1 ? undefined : base64.slice(5, separatorIndex); + const detectedFormat = + separatorIndex === -1 + ? inferScreenshotImageFormatFromBase64(base64) + : screenshotImageFormatFromMimeType(mimeType); + + // Before WebP support, every non-JPEG value used PNG metadata. Preserve that + // behavior for temporary and test placeholders while detecting valid images. + return detectedFormat ?? 'png'; +} + +function rawBase64Body(base64: string): string { + const separatorIndex = base64.indexOf(BASE64_SEPARATOR); + const body = + separatorIndex === -1 + ? base64 + : base64.slice(separatorIndex + BASE64_SEPARATOR.length); + return body.replace(/\s/g, ''); } /** @@ -35,7 +62,7 @@ function detectFormat(base64: string): 'png' | 'jpeg' { export class ScreenshotItem { private _id: string; private _base64: string | null; - private _format: 'png' | 'jpeg'; + private _format: ScreenshotImageFormat; private _capturedAt: number; private _serializedRef: ScreenshotRef | null = null; private _persistedPath: string | null = null; @@ -57,14 +84,19 @@ export class ScreenshotItem { return this._id; } - /** Get the image format (png or jpeg) */ - get format(): 'png' | 'jpeg' { + /** Get the image format (PNG, JPEG, or WebP). */ + get format(): ScreenshotImageFormat { return this._format; } /** Get the file extension for this screenshot */ - get extension(): string { - return this._format === 'jpeg' ? 'jpeg' : 'png'; + get extension(): ScreenshotImageFormat { + return screenshotImageExtension(this._format); + } + + /** Get the MIME type for this screenshot. */ + get mimeType(): ScreenshotImageMimeType { + return screenshotImageMimeType(this._format); } /** Get screenshot capture timestamp in milliseconds */ @@ -83,7 +115,7 @@ export class ScreenshotItem { throw new Error(`Screenshot ${this._id}: file recovery path missing`); } const buffer = readFileSync(this._persistedPath); - return `data:image/${this._format};base64,${buffer.toString('base64')}`; + return `data:${this.mimeType};base64,${buffer.toString('base64')}`; }; const loadFromInline = (): string => { @@ -176,7 +208,7 @@ export class ScreenshotItem { type: 'midscene_screenshot_ref', id: this._id, capturedAt: this._capturedAt, - mimeType: this._format === 'jpeg' ? 'image/jpeg' : 'image/png', + mimeType: this.mimeType, storage: 'inline', } ); @@ -195,7 +227,7 @@ export class ScreenshotItem { type: 'midscene_screenshot_ref', id: this._id, capturedAt: this._capturedAt, - mimeType: this._format === 'jpeg' ? 'image/jpeg' : 'image/png', + mimeType: this.mimeType, storage, }; if (storage === 'file') { @@ -213,6 +245,6 @@ export class ScreenshotItem { * Useful for writing raw binary data to files. */ get rawBase64(): string { - return this.base64.replace(/^data:image\/(png|jpeg|jpg);base64,/, ''); + return rawBase64Body(this.base64); } } diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 7ea517660f..460e509e7b 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -489,7 +489,7 @@ export interface ExecutionRecorderItem { export interface RecordToReportScreenshot { /** - * PNG/JPEG data URI, or raw PNG base64 body. + * PNG/JPEG/WebP data URI, or raw PNG/WebP base64 body. */ base64: string; description?: string; @@ -740,7 +740,7 @@ export type ExecutionTaskPlanningLocate = /* How a report file stores screenshots: - `inline`: base64 image script tags embedded in the single HTML file -- `directory`: external PNG files under a sibling `screenshots/` dir +- `directory`: external image files under a sibling `screenshots/` dir */ export type ScreenshotMode = 'inline' | 'directory'; @@ -916,7 +916,7 @@ export interface AgentOpt { * Use directory-based report format with separate image files. * * When enabled: - * - Screenshots are saved as PNG files in a `screenshots/` subdirectory + * - Screenshots retain their image format in a `screenshots/` subdirectory * - Report is generated as `index.html` with relative image paths * - Reduces memory usage and report file size * diff --git a/packages/core/tests/unit-test/agent-describe-element.test.ts b/packages/core/tests/unit-test/agent-describe-element.test.ts index 967a597955..767478de4d 100644 --- a/packages/core/tests/unit-test/agent-describe-element.test.ts +++ b/packages/core/tests/unit-test/agent-describe-element.test.ts @@ -424,7 +424,7 @@ describe('element describer utils', () => { const describeContext = describe.mock.calls[0][2]?.context; expect(describeContext?.screenshot.base64).toMatch( - /^data:image\/png;base64,/, + /^data:image\/webp;base64,UklGR/, ); expect(describeContext?.shotSize).toEqual(fixtureScreenshotSize); diff --git a/packages/core/tests/unit-test/agent-dump-update.test.ts b/packages/core/tests/unit-test/agent-dump-update.test.ts index 9f9bda4066..77f2a5c809 100644 --- a/packages/core/tests/unit-test/agent-dump-update.test.ts +++ b/packages/core/tests/unit-test/agent-dump-update.test.ts @@ -255,7 +255,7 @@ describe('Agent dump update screenshot serialization', () => { screenshots: [{ base64: 'data:image/svg+xml;base64,custom' }], }), ).rejects.toThrow( - 'recordToReport: screenshot #1 base64 must be a PNG/JPEG data URI or raw PNG base64 string', + 'recordToReport: screenshot #1 base64 must be a PNG/JPEG/WebP data URI or raw PNG/WebP base64 string', ); expect(screenshotBase64).not.toHaveBeenCalled(); diff --git a/packages/core/tests/unit-test/common-context-parser-orientation.test.ts b/packages/core/tests/unit-test/common-context-parser-orientation.test.ts index cce92f67fa..b6648a52ee 100644 --- a/packages/core/tests/unit-test/common-context-parser-orientation.test.ts +++ b/packages/core/tests/unit-test/common-context-parser-orientation.test.ts @@ -4,13 +4,8 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; // Mock imageInfoOfBase64 to control screenshot dimensions vi.mock('@midscene/shared/img', () => ({ - convertImgBufferToJpeg: vi.fn(), - createImgBase64ByFormat: vi.fn(), + canonicalizeScreenshotBase64: vi.fn().mockResolvedValue('mock-base64-data'), imageInfoOfBase64: vi.fn(), - parseBase64: vi.fn(() => ({ - mimeType: 'image/jpeg', - body: 'mock-base64-data', - })), resizeImgBase64: vi.fn().mockResolvedValue('mock-resized-base64-data'), })); diff --git a/packages/core/tests/unit-test/common-context-parser-shrink-factor.test.ts b/packages/core/tests/unit-test/common-context-parser-shrink-factor.test.ts index ca672e0d90..d221df3754 100644 --- a/packages/core/tests/unit-test/common-context-parser-shrink-factor.test.ts +++ b/packages/core/tests/unit-test/common-context-parser-shrink-factor.test.ts @@ -3,25 +3,19 @@ import type { AbstractInterface } from '@/device'; import { beforeEach, describe, expect, it, vi } from 'vitest'; vi.mock('@midscene/shared/img', () => ({ - convertImgBufferToJpeg: vi.fn(), - createImgBase64ByFormat: vi.fn(), + canonicalizeScreenshotBase64: vi.fn(), imageInfoOfBase64: vi.fn(), - parseBase64: vi.fn(), resizeImgBase64: vi.fn().mockResolvedValue('mock-resized-base64-data'), })); import { - convertImgBufferToJpeg, - createImgBase64ByFormat, + canonicalizeScreenshotBase64, imageInfoOfBase64, - parseBase64, resizeImgBase64, } from '@midscene/shared/img'; -const mockedConvertToJpeg = vi.mocked(convertImgBufferToJpeg); -const mockedCreateBase64 = vi.mocked(createImgBase64ByFormat); +const mockedCanonicalize = vi.mocked(canonicalizeScreenshotBase64); const mockedImageInfo = vi.mocked(imageInfoOfBase64); -const mockedParseBase64 = vi.mocked(parseBase64); const mockedResizeImg = vi.mocked(resizeImgBase64); function createMockInterface( @@ -41,35 +35,24 @@ function createMockInterface( describe('commonContextParser screenshotShrinkFactor', () => { beforeEach(() => { vi.clearAllMocks(); - mockedParseBase64.mockReturnValue({ - mimeType: 'image/jpeg', - body: 'mock-base64-data', - }); + mockedCanonicalize.mockResolvedValue('mock-base64-data'); }); - it('converts PNG screenshots to JPEG quality 90 when not shrinking', async () => { + it('canonicalizes screenshots before sharing them with AI and reports', async () => { const mockInterface = createMockInterface(800, 400); - const pngBody = Buffer.from('png-image').toString('base64'); - const jpegBuffer = Buffer.from('jpeg-image'); mockedImageInfo.mockResolvedValue({ width: 2400, height: 1200 }); - mockedParseBase64.mockReturnValue({ - mimeType: 'image/png', - body: pngBody, - }); - mockedConvertToJpeg.mockResolvedValue(jpegBuffer); - mockedCreateBase64.mockReturnValue('data:image/jpeg;base64,jpeg-image'); + mockedCanonicalize.mockResolvedValue( + 'data:image/webp;base64,canonical-webp', + ); const result = await commonContextParser(mockInterface, {}); - expect(mockedConvertToJpeg).toHaveBeenCalledWith( - Buffer.from(pngBody, 'base64'), - 90, - ); - expect(mockedCreateBase64).toHaveBeenCalledWith( - 'jpeg', - jpegBuffer.toString('base64'), + expect(mockedCanonicalize).toHaveBeenCalledWith('mock-base64-data', { + preserveJpeg: true, + }); + expect(result.screenshot.base64).toBe( + 'data:image/webp;base64,canonical-webp', ); - expect(result.screenshot.base64).toBe('data:image/jpeg;base64,jpeg-image'); }); it('does not shrink when screenshotShrinkFactor is not provided', async () => { @@ -94,6 +77,7 @@ describe('commonContextParser screenshotShrinkFactor', () => { width: 1200, height: 600, }); + expect(mockedCanonicalize).not.toHaveBeenCalled(); expect(result.shotSize).toEqual({ width: 1200, height: 600 }); }); diff --git a/packages/core/tests/unit-test/gpt-image-detail.test.ts b/packages/core/tests/unit-test/gpt-image-detail.test.ts index 8820872d88..373a9f6f19 100644 --- a/packages/core/tests/unit-test/gpt-image-detail.test.ts +++ b/packages/core/tests/unit-test/gpt-image-detail.test.ts @@ -183,4 +183,16 @@ describe('GPT image detail handling', () => { expect(mockCreate.mock.calls[0][0]).toHaveProperty('temperature', 0.7); }); + + it('reports the exact image URL included in the model request', async () => { + const onModelInputImages = vi.fn(); + await callAI(imageMessage, { + ...getModelRuntime(baseModelConfig), + onModelInputImages, + }); + + expect(onModelInputImages).toHaveBeenCalledWith([ + 'https://example.com/shot.png', + ]); + }); }); diff --git a/packages/core/tests/unit-test/model-input-recorder.test.ts b/packages/core/tests/unit-test/model-input-recorder.test.ts new file mode 100644 index 0000000000..59e9197eed --- /dev/null +++ b/packages/core/tests/unit-test/model-input-recorder.test.ts @@ -0,0 +1,81 @@ +import { recordModelInputsForTask } from '@/agent/model-input-recorder'; +import type { ModelRuntime } from '@/ai-model/models'; +import { ScreenshotItem } from '@/screenshot-item'; +import type { ExecutionTask } from '@/types'; +import { describe, expect, it, vi } from 'vitest'; + +const pngBase64 = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA'; +const webpBase64 = + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; + +function createTask(): ExecutionTask { + const screenshot = ScreenshotItem.create(pngBase64, 100); + return { + taskId: 'task-1', + type: 'Planning', + subType: 'Plan', + status: 'running', + executor: vi.fn(), + uiContext: { + screenshot, + shotSize: { width: 5, height: 1 }, + shrunkShotToLogicalRatio: 1, + }, + }; +} + +describe('recordModelInputsForTask', () => { + it('reuses the report screenshot when the exact bytes are sent to AI', () => { + const task = createTask(); + const runtime = recordModelInputsForTask({} as ModelRuntime, task); + + runtime.onModelInputImages?.([pngBase64]); + + expect(task.recorder).toHaveLength(1); + expect(task.recorder?.[0].screenshot).toBe(task.uiContext?.screenshot); + expect(task.recorder?.[0].timing).toBe('model-input'); + expect(task.recorder?.[0].description).toMatch( + /^Model input 1 \(exact bytes, sha256: [a-f0-9]{64}\)$/, + ); + }); + + it('records a transformed model image and deduplicates request retries', () => { + const task = createTask(); + const parentCallback = vi.fn(); + const parentRuntime = {} as ModelRuntime; + parentRuntime.onModelInputImages = parentCallback; + const runtime = recordModelInputsForTask(parentRuntime, task); + + runtime.onModelInputImages?.([webpBase64]); + runtime.onModelInputImages?.([webpBase64]); + + expect(parentCallback).toHaveBeenCalledTimes(2); + expect(task.recorder).toHaveLength(1); + expect(task.recorder?.[0].screenshot?.base64).toBe(webpBase64); + expect(task.recorder?.[0].screenshot).not.toBe(task.uiContext?.screenshot); + }); + + it('reuses the executor context screenshot before it is bound to the task', () => { + const task = createTask(); + const sourceScreenshot = task.uiContext!.screenshot; + task.uiContext = undefined; + const runtime = recordModelInputsForTask( + {} as ModelRuntime, + task, + sourceScreenshot, + ); + + runtime.onModelInputImages?.([pngBase64]); + + expect(task.recorder?.[0].screenshot).toBe(sourceScreenshot); + }); + + it('does not turn remote reference image URLs into screenshot items', () => { + const task = createTask(); + const runtime = recordModelInputsForTask({} as ModelRuntime, task); + + runtime.onModelInputImages?.(['https://example.com/reference.png']); + + expect(task.recorder).toBeUndefined(); + }); +}); diff --git a/packages/core/tests/unit-test/report-action-dump-webp.test.ts b/packages/core/tests/unit-test/report-action-dump-webp.test.ts new file mode 100644 index 0000000000..874b2cfe04 --- /dev/null +++ b/packages/core/tests/unit-test/report-action-dump-webp.test.ts @@ -0,0 +1,76 @@ +import { existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { ReportActionDump } from '../../src/dump/report-action-dump'; +import { ScreenshotItem } from '../../src/screenshot-item'; +import { ExecutionDump } from '../../src/types'; + +const webpBody = + 'UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; +const webpBase64 = `data:image/webp;base64,${webpBody}`; + +describe('ReportActionDump WebP file serialization', () => { + let temporaryDirectory: string; + + beforeEach(() => { + temporaryDirectory = join( + tmpdir(), + `midscene-dump-webp-${Date.now()}-${Math.random()}`, + ); + mkdirSync(temporaryDirectory, { recursive: true }); + }); + + afterEach(() => { + rmSync(temporaryDirectory, { recursive: true, force: true }); + }); + + it('writes and restores WebP files without changing their bytes or MIME type', () => { + const screenshot = ScreenshotItem.create(webpBase64, 100); + const dump = new ReportActionDump({ + sdkVersion: '1.0.0-test', + groupName: 'webp-dump', + modelBriefs: [], + executions: [ + new ExecutionDump({ + logTime: 100, + name: 'webp-execution', + tasks: [ + { + taskId: 'webp-task', + type: 'Insight', + subType: 'Locate', + param: { prompt: 'target' }, + uiContext: { + screenshot, + shotSize: { width: 2, height: 3 }, + shrunkShotToLogicalRatio: 1, + }, + executor: async () => undefined, + recorder: [], + status: 'finished', + }, + ], + }), + ], + }); + const dumpPath = join(temporaryDirectory, 'dump.json'); + + dump.serializeToFiles(dumpPath); + + const screenshotPath = join( + `${dumpPath}.screenshots`, + `${screenshot.id}.webp`, + ); + expect(existsSync(screenshotPath)).toBe(true); + expect(readFileSync(screenshotPath).toString('base64')).toBe(webpBody); + + const restored = JSON.parse( + ReportActionDump.fromFilesAsInlineJson(dumpPath), + ); + expect(restored.executions[0].tasks[0].uiContext.screenshot).toMatchObject({ + base64: webpBase64, + capturedAt: 100, + }); + }); +}); diff --git a/packages/core/tests/unit-test/report-cli.test.ts b/packages/core/tests/unit-test/report-cli.test.ts index f80b55149b..2497db54cb 100644 --- a/packages/core/tests/unit-test/report-cli.test.ts +++ b/packages/core/tests/unit-test/report-cli.test.ts @@ -24,6 +24,9 @@ function fakeBase64(sizeBytes: number): string { return `data:image/png;base64,${'A'.repeat(sizeBytes)}`; } +const webpBase64 = + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; + function createExecution( id: string, screenshot: ScreenshotItem | ScreenshotRef, @@ -223,6 +226,45 @@ describe('createReportCliCommands', () => { expect(existsSync(join(outputDir, 'report.md'))).toBe(true); }); + it('exports WebP screenshots and markdown links with the .webp suffix', async () => { + const reportPath = join(tmpDir, 'input-report-sdk-webp', 'index.html'); + mkdirSync(join(tmpDir, 'input-report-sdk-webp'), { recursive: true }); + const screenshot = ScreenshotItem.create(webpBase64, Date.now()); + const dump = new ReportActionDump({ + groupName: 'sdk-webp-test', + sdkVersion: '1.0.0-test', + modelBriefs: [], + executions: [createExecution('exec-sdk-webp', screenshot)], + }); + writeFileSync( + reportPath, + [ + generateImageScriptTag(screenshot.id, screenshot.base64), + generateDumpScriptTag(dump.serialize(), { + 'data-group-id': 'webp-group', + }), + ].join('\n'), + 'utf-8', + ); + + const outputDir = join(tmpDir, 'output-sdk-webp'); + const result = await reportFileToMarkdown({ + htmlPath: reportPath, + outputDir, + }); + const expectedFileName = `execution-1-task-1-${screenshot.id}.webp`; + + expect(result.screenshotFiles).toEqual([ + join(outputDir, 'screenshots', expectedFileName), + ]); + expect(readFileSync(result.screenshotFiles[0]).toString('base64')).toBe( + webpBase64.split(',')[1], + ); + expect(readFileSync(join(outputDir, 'report.md'), 'utf-8')).toContain( + `./screenshots/${expectedFileName}`, + ); + }); + it('throws SDK-friendly validation errors for reportFileToMarkdown', async () => { await expect( reportFileToMarkdown({ diff --git a/packages/core/tests/unit-test/report-markdown.test.ts b/packages/core/tests/unit-test/report-markdown.test.ts index aa23792285..df9cc13d4f 100644 --- a/packages/core/tests/unit-test/report-markdown.test.ts +++ b/packages/core/tests/unit-test/report-markdown.test.ts @@ -28,6 +28,9 @@ function createTask(overrides: Record = {}) { } describe('report-markdown', () => { + const webpBase64 = + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; + it('handles single execution markdown with screenshot file links', () => { const screenshot = ScreenshotItem.create( 'data:image/png;base64,Zm9v', @@ -85,6 +88,34 @@ describe('report-markdown', () => { ); }); + it('exports WebP screenshots with coherent MIME and file metadata', () => { + const screenshot = ScreenshotItem.create(webpBase64, 1710000000000); + const execution: IExecutionDump = { + logTime: 1710000000000, + name: 'webp execution', + tasks: [ + createTask({ + uiContext: { + screenshot, + shotSize: { width: 2, height: 3 }, + }, + }), + ], + }; + + const result = executionToMarkdown(execution); + + expect(result.attachments).toHaveLength(1); + expect(result.attachments[0]).toMatchObject({ + suggestedFileName: expect.stringMatching(/\.webp$/), + mimeType: 'image/webp', + base64Data: webpBase64, + }); + expect(result.markdown).toContain( + `./screenshots/${result.attachments[0].suggestedFileName}`, + ); + }); + it('merges all executions into one markdown and keeps file snapshot', async () => { const report: IReportActionDump = { sdkVersion: '1.0.0', diff --git a/packages/core/tests/unit-test/report-split.test.ts b/packages/core/tests/unit-test/report-split.test.ts index 9c9f869c41..6c56364f6a 100644 --- a/packages/core/tests/unit-test/report-split.test.ts +++ b/packages/core/tests/unit-test/report-split.test.ts @@ -18,6 +18,9 @@ function fakeBase64(sizeBytes: number): string { return `data:image/png;base64,${'A'.repeat(sizeBytes)}`; } +const webpBase64 = + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; + function createExecution( id: string, screenshot: ScreenshotItem | ScreenshotRef, @@ -129,6 +132,48 @@ describe('splitReportHtmlByExecution', () => { } }); + it('externalizes inline WebP screenshots as .webp files', () => { + const reportPath = join(tmpDir, 'webp-report', 'index.html'); + mkdirSync(join(tmpDir, 'webp-report'), { recursive: true }); + const screenshot = ScreenshotItem.create(webpBase64, Date.now()); + const dump = new ReportActionDump({ + groupName: 'webp-split-test', + sdkVersion: '1.0.0-test', + modelBriefs: [], + executions: [createExecution('webp-exec', screenshot)], + }); + writeFileSync( + reportPath, + [ + generateImageScriptTag(screenshot.id, screenshot.base64), + generateDumpScriptTag(dump.serialize(), { + 'data-group-id': 'webp-group', + }), + ].join('\n'), + 'utf-8', + ); + + const result = splitReportHtmlByExecution({ + htmlPath: reportPath, + outputDir: join(tmpDir, 'webp-output'), + }); + const outputDump = JSON.parse( + readFileSync(result.executionJsonFiles[0], 'utf-8'), + ); + const outputRef = outputDump.executions[0].tasks[0].uiContext.screenshot; + + expect(outputRef).toMatchObject({ + mimeType: 'image/webp', + path: `./screenshots/${screenshot.id}.webp`, + }); + expect(result.screenshotFiles).toEqual([ + join(tmpDir, 'webp-output', 'screenshots', `${screenshot.id}.webp`), + ]); + expect(readFileSync(result.screenshotFiles[0]).toString('base64')).toBe( + webpBase64.split(',')[1], + ); + }); + it('should process large report incrementally without accumulating all dump scripts', () => { const reportPath = join(tmpDir, 'large-report', 'index.html'); mkdirSync(join(tmpDir, 'large-report'), { recursive: true }); diff --git a/packages/core/tests/unit-test/screenshot-item.test.ts b/packages/core/tests/unit-test/screenshot-item.test.ts index d20c0246f1..6f43ddc9d7 100644 --- a/packages/core/tests/unit-test/screenshot-item.test.ts +++ b/packages/core/tests/unit-test/screenshot-item.test.ts @@ -6,6 +6,8 @@ import { ScreenshotItem } from '../../src/screenshot-item'; describe('ScreenshotItem', () => { const testBase64 = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA'; + const webpBase64 = + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; describe('create', () => { it('should create a ScreenshotItem from base64 string', () => { @@ -19,6 +21,43 @@ describe('ScreenshotItem', () => { const item = ScreenshotItem.create(testBase64, capturedAt); expect(item.capturedAt).toBe(capturedAt); }); + + it('preserves empty screenshot placeholders as PNG metadata', () => { + const item = ScreenshotItem.create('', 123); + + expect(item.base64).toBe(''); + expect(item.rawBase64).toBe(''); + expect(item.format).toBe('png'); + expect(item.extension).toBe('png'); + expect(item.mimeType).toBe('image/png'); + expect(item.toSerializable()).toMatchObject({ + capturedAt: 123, + mimeType: 'image/png', + }); + }); + + it('preserves unrecognized screenshot placeholders as PNG metadata', () => { + const item = ScreenshotItem.create('mock-screenshot', 123); + + expect(item.base64).toBe('mock-screenshot'); + expect(item.rawBase64).toBe('mock-screenshot'); + expect(item.format).toBe('png'); + expect(item.extension).toBe('png'); + expect(item.mimeType).toBe('image/png'); + }); + + it('classifies WebP screenshots without corrupting their metadata or body', () => { + const item = ScreenshotItem.create(webpBase64, 123); + + expect(item.format).toBe('webp'); + expect(item.extension).toBe('webp'); + expect(item.mimeType).toBe('image/webp'); + expect(item.rawBase64).toBe(webpBase64.split(',')[1]); + expect(item.toSerializable()).toMatchObject({ + capturedAt: 123, + mimeType: 'image/webp', + }); + }); }); describe('base64 getter', () => { @@ -126,6 +165,11 @@ describe('ScreenshotItem', () => { expect(item.rawBase64).toBe('/9j/4AAQ'); }); + it('should strip data URI prefix from WebP', () => { + const item = ScreenshotItem.create(webpBase64, Date.now()); + expect(item.rawBase64).toBe(webpBase64.split(',')[1]); + }); + it('should return unchanged if no prefix', () => { const item = ScreenshotItem.create( 'iVBORw0KGgoAAAANSUhEUgAAAAUA', diff --git a/packages/core/tests/unit-test/screenshot-store.test.ts b/packages/core/tests/unit-test/screenshot-store.test.ts index f63f801157..898904ee39 100644 --- a/packages/core/tests/unit-test/screenshot-store.test.ts +++ b/packages/core/tests/unit-test/screenshot-store.test.ts @@ -13,6 +13,9 @@ import { ScreenshotItem } from '../../src/screenshot-item'; describe('ScreenshotStore', () => { const pngBase64 = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA'; + const webpBody = + 'UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; + const webpBase64 = `data:image/webp;base64,${webpBody}`; let tmpRoot: string; beforeEach(() => { @@ -44,6 +47,53 @@ describe('ScreenshotStore', () => { expect(store.loadBase64(ref)).toContain('data:image/png;base64,'); }); + it('persists WebP bytes with a .webp path and restores the WebP MIME type', async () => { + const reportPath = join(tmpRoot, 'index.html'); + const screenshotsDir = join(tmpRoot, 'screenshots'); + const item = ScreenshotItem.create(webpBase64, 100); + const store = new ScreenshotStore({ + mode: 'directory', + reportPath, + screenshotsDir, + }); + + const ref = await store.persist(item); + const filePath = join(screenshotsDir, `${item.id}.webp`); + + expect(ref).toMatchObject({ + mimeType: 'image/webp', + path: `./screenshots/${item.id}.webp`, + }); + expect(readFileSync(filePath).toString('base64')).toBe(webpBody); + expect(store.loadBase64(ref)).toBe(webpBase64); + }); + + it('resolves sibling WebP files for inline references', () => { + const reportPath = join(tmpRoot, 'index.html'); + const screenshotsDir = join(tmpRoot, 'screenshots'); + mkdirSync(screenshotsDir, { recursive: true }); + writeFileSync(reportPath, ''); + writeFileSync( + join(screenshotsDir, 'sibling-webp.webp'), + Buffer.from(webpBody, 'base64'), + ); + const store = new ScreenshotStore({ + mode: 'inline', + reportPath, + writeInlineImage: () => {}, + }); + + expect( + store.loadBase64({ + type: 'midscene_screenshot_ref', + id: 'sibling-webp', + capturedAt: 100, + mimeType: 'image/webp', + storage: 'inline', + }), + ).toBe(webpBase64); + }); + it('deduplicates same screenshot persistence by id', async () => { const reportPath = join(tmpRoot, 'index.html'); const screenshotsDir = join(tmpRoot, 'screenshots'); diff --git a/packages/core/tests/unit-test/tasks-null-data.test.ts b/packages/core/tests/unit-test/tasks-null-data.test.ts index 83c5e6861a..ebd52451a6 100644 --- a/packages/core/tests/unit-test/tasks-null-data.test.ts +++ b/packages/core/tests/unit-test/tasks-null-data.test.ts @@ -41,6 +41,12 @@ const expectEmptyUIContext = () => shrunkShotToLogicalRatio: 1, }); +const expectRecordedModelRuntime = (config: IModelConfig) => + expect.objectContaining({ + config: expect.objectContaining(config), + onModelInputImages: expect.any(Function), + }); + const createMockUsage = (totalTokens: number): AIUsageInfo => ({ prompt_tokens: 0, completion_tokens: 0, @@ -214,7 +220,7 @@ describe('TaskExecutor - Null Data Handling', () => { StatementIsTruthy: 'Boolean, based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images, whether the following statement is true: Page title is correct', }, - getModelRuntime(mockModelConfig), + expectRecordedModelRuntime(mockModelConfig), {}, '', undefined, @@ -269,7 +275,7 @@ describe('TaskExecutor - Null Data Handling', () => { StatementIsTruthy: "Boolean, the user wants to do some 'wait for' operation. based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images, please check whether the following statement is true: Element is visible", }, - getModelRuntime(mockModelConfig), + expectRecordedModelRuntime(mockModelConfig), {}, '', undefined, @@ -636,7 +642,7 @@ describe('TaskExecutor - Null Data Handling', () => { Number: 'Number, based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images, Extract the price', }, - getModelRuntime(mockModelConfig), + expectRecordedModelRuntime(mockModelConfig), {}, '', undefined, @@ -731,7 +737,7 @@ describe('TaskExecutor - Null Data Handling', () => { Number: 'Number, based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images, Extract the price', }, - getModelRuntime(mockModelConfig), + expectRecordedModelRuntime(mockModelConfig), {}, '', undefined, @@ -787,7 +793,7 @@ describe('TaskExecutor - Null Data Handling', () => { Boolean: 'Boolean, based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images, there is a like button', }, - getModelRuntime(mockModelConfig), + expectRecordedModelRuntime(mockModelConfig), {}, '', undefined, diff --git a/packages/shared/src/agent-tools/types.ts b/packages/shared/src/agent-tools/types.ts index de007edaa8..78cf044a90 100644 --- a/packages/shared/src/agent-tools/types.ts +++ b/packages/shared/src/agent-tools/types.ts @@ -98,7 +98,7 @@ export type UserPromptLike = export interface RecordToReportScreenshot { /** - * PNG/JPEG data URI, or raw PNG base64 body. + * PNG/JPEG/WebP data URI, or raw PNG/WebP base64 body. */ base64: string; description?: string; diff --git a/packages/shared/src/cli/screenshot-file.ts b/packages/shared/src/cli/screenshot-file.ts index 4ecb0d38eb..db843c3041 100644 --- a/packages/shared/src/cli/screenshot-file.ts +++ b/packages/shared/src/cli/screenshot-file.ts @@ -1,6 +1,11 @@ import { existsSync, mkdirSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { + type ScreenshotImageFormat, + screenshotImageFormatFromExtension, + screenshotImageFormatFromMimeType, +} from '../img/image-format'; export interface WriteCliScreenshotFileOptions { id?: unknown; @@ -20,14 +25,12 @@ function safeScreenshotFilenamePart(value: unknown): string { function extensionFromImageMetadata( mimeType: unknown, extension: unknown, -): 'png' | 'jpeg' { - if (extension === 'jpeg' || extension === 'jpg') { - return 'jpeg'; +): ScreenshotImageFormat { + const extensionFormat = screenshotImageFormatFromExtension(extension); + if (extensionFormat) { + return extensionFormat; } - if (extension === 'png') { - return 'png'; - } - return mimeType === 'image/jpeg' ? 'jpeg' : 'png'; + return screenshotImageFormatFromMimeType(mimeType) ?? 'png'; } export function writeCliScreenshotFile( diff --git a/packages/shared/src/cli/verbose-screenshot.ts b/packages/shared/src/cli/verbose-screenshot.ts index b27d62234d..f77bfc621a 100644 --- a/packages/shared/src/cli/verbose-screenshot.ts +++ b/packages/shared/src/cli/verbose-screenshot.ts @@ -71,7 +71,9 @@ function screenshotRawBase64(value: unknown): string | undefined { } const base64 = getStringProperty(value, 'base64'); - const match = base64?.match(/^data:image\/(?:png|jpeg|jpg);base64,(.+)$/); + const match = base64?.match( + /^data:image\/(?:png|jpeg|jpg|webp);base64,(.+)$/, + ); return match?.[1]; } diff --git a/packages/shared/src/img/box-select.ts b/packages/shared/src/img/box-select.ts index 1b76deb366..0afa332281 100644 --- a/packages/shared/src/img/box-select.ts +++ b/packages/shared/src/img/box-select.ts @@ -758,9 +758,11 @@ async function encodeRgbaWithSharp( const output = await Sharp(Buffer.from(pixels), { raw: { width, height, channels: 4 }, }) - .jpeg({ quality: 90, chromaSubsampling: '4:4:4' }) + // Keep synthetic marker edges and colors exact; these pixels carry model + // semantics and are more important than the small extra payload. + .webp({ lossless: true, effort: 1 }) .toBuffer(); - return createImgBase64ByFormat('jpeg', output.toString('base64')); + return createImgBase64ByFormat('webp', output.toString('base64')); } export const compositeElementInfoImg = async (options: { diff --git a/packages/shared/src/img/browser-webp-encoder.ts b/packages/shared/src/img/browser-webp-encoder.ts new file mode 100644 index 0000000000..be2cdb38e0 --- /dev/null +++ b/packages/shared/src/img/browser-webp-encoder.ts @@ -0,0 +1,114 @@ +export interface BrowserWebpEncodeInput { + pixels: ArrayLike; + width: number; + height: number; + /** Encoder quality from 0 to 100. Defaults to 90. */ + quality?: number; +} + +/** + * Encode RGBA pixels with the WebP encoder provided by the browser. + * + * Keep this function self-contained so browser contract tests can execute the + * production implementation in a page or Worker without a test-only copy. + */ +export async function encodeRgbaToWebp({ + pixels, + width, + height, + quality = 90, +}: BrowserWebpEncodeInput): Promise { + if ( + !Number.isSafeInteger(width) || + !Number.isSafeInteger(height) || + width <= 0 || + height <= 0 + ) { + throw new Error('WebP image dimensions must be positive safe integers'); + } + + if (!Number.isFinite(quality) || quality < 0 || quality > 100) { + throw new Error('WebP quality must be between 0 and 100'); + } + + const expectedPixelCount = width * height * 4; + if ( + !Number.isSafeInteger(expectedPixelCount) || + pixels.length !== expectedPixelCount + ) { + throw new Error( + `WebP RGBA pixel length must be ${expectedPixelCount}, got ${pixels.length}`, + ); + } + + const normalizedQuality = quality / 100; + let outputBlob: Blob; + + if (typeof OffscreenCanvas !== 'undefined') { + const canvas = new OffscreenCanvas(width, height); + const context = canvas.getContext('2d'); + if (!context) { + throw new Error('Failed to get an OffscreenCanvas 2d context'); + } + + const imageData = context.createImageData(width, height); + imageData.data.set(pixels); + context.putImageData(imageData, 0, 0); + outputBlob = await canvas.convertToBlob({ + type: 'image/webp', + quality: normalizedQuality, + }); + } else if (typeof document !== 'undefined') { + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const context = canvas.getContext('2d'); + if (!context) { + throw new Error('Failed to get an HTMLCanvasElement 2d context'); + } + + const imageData = context.createImageData(width, height); + imageData.data.set(pixels); + context.putImageData(imageData, 0, 0); + outputBlob = await new Promise((resolve, reject) => { + canvas.toBlob( + (blob) => { + if (blob) { + resolve(blob); + } else { + reject(new Error('HTMLCanvasElement failed to encode WebP')); + } + }, + 'image/webp', + normalizedQuality, + ); + }); + } else { + throw new Error( + 'WebP encoding requires OffscreenCanvas or HTMLCanvasElement', + ); + } + + if (outputBlob.type.toLowerCase() !== 'image/webp') { + throw new Error( + `Browser WebP encoder returned ${outputBlob.type || 'an unknown MIME type'}`, + ); + } + + const output = new Uint8Array(await outputBlob.arrayBuffer()); + const isWebp = + output.length >= 12 && + output[0] === 0x52 && + output[1] === 0x49 && + output[2] === 0x46 && + output[3] === 0x46 && + output[8] === 0x57 && + output[9] === 0x45 && + output[10] === 0x42 && + output[11] === 0x50; + if (!isWebp) { + throw new Error('Browser WebP encoder returned invalid WebP bytes'); + } + + return output; +} diff --git a/packages/shared/src/img/canvas-fallback.ts b/packages/shared/src/img/canvas-fallback.ts index bfc0f4bf02..62267aaed4 100644 --- a/packages/shared/src/img/canvas-fallback.ts +++ b/packages/shared/src/img/canvas-fallback.ts @@ -4,6 +4,11 @@ */ import { getDebug } from '../logger'; +import { + detectScreenshotImageFormatFromBuffer, + inferScreenshotImageFormatFromBase64, + screenshotImageMimeType, +} from './image-format'; const debug = getDebug('img:canvas-fallback'); @@ -42,6 +47,10 @@ export class CanvasImage { get_bytes_jpeg(quality: number): Uint8Array { const dataUrl = this.canvas.toDataURL('image/jpeg', quality / 100); + return CanvasImage.bytesFromDataUrl(dataUrl); + } + + private static bytesFromDataUrl(dataUrl: string): Uint8Array { const base64 = dataUrl.split(',')[1]; const binary = atob(base64); const bytes = new Uint8Array(binary.length); @@ -89,7 +98,9 @@ export class CanvasImage { if (base64Body.startsWith('data:')) { img.src = base64Body; } else { - img.src = `data:image/png;base64,${base64Body}`; + const format = + inferScreenshotImageFormatFromBase64(base64Body) ?? 'png'; + img.src = `data:${screenshotImageMimeType(format)};base64,${base64Body}`; } }); } @@ -99,7 +110,10 @@ export class CanvasImage { */ static async new_from_byteslice(bytes: Uint8Array): Promise { return new Promise((resolve, reject) => { - const blob = new Blob([bytes], { type: 'image/png' }); + const format = detectScreenshotImageFormatFromBuffer(bytes) ?? 'png'; + const blob = new Blob([bytes], { + type: screenshotImageMimeType(format), + }); const url = URL.createObjectURL(blob); const img = new Image(); diff --git a/packages/shared/src/img/image-format.ts b/packages/shared/src/img/image-format.ts new file mode 100644 index 0000000000..256fadbdef --- /dev/null +++ b/packages/shared/src/img/image-format.ts @@ -0,0 +1,131 @@ +export type ScreenshotImageFormat = 'png' | 'jpeg' | 'webp'; + +export type ScreenshotImageMimeType = 'image/png' | 'image/jpeg' | 'image/webp'; + +const mimeTypeByFormat: Record = + { + png: 'image/png', + jpeg: 'image/jpeg', + webp: 'image/webp', + }; + +export function screenshotImageMimeType( + format: ScreenshotImageFormat, +): ScreenshotImageMimeType { + return mimeTypeByFormat[format]; +} + +export function screenshotImageExtension( + format: ScreenshotImageFormat, +): ScreenshotImageFormat { + return format; +} + +export function screenshotImageFormatFromExtension( + extension: unknown, +): ScreenshotImageFormat | undefined { + if (typeof extension !== 'string') { + return undefined; + } + + switch (extension.toLowerCase()) { + case 'png': + return 'png'; + case 'jpeg': + case 'jpg': + return 'jpeg'; + case 'webp': + return 'webp'; + default: + return undefined; + } +} + +export function screenshotImageFormatFromMimeType( + mimeType: unknown, +): ScreenshotImageFormat | undefined { + if (typeof mimeType !== 'string') { + return undefined; + } + + switch (mimeType.toLowerCase()) { + case 'image/png': + return 'png'; + case 'image/jpeg': + case 'image/jpg': + return 'jpeg'; + case 'image/webp': + return 'webp'; + default: + return undefined; + } +} + +export function isScreenshotImageMimeType( + mimeType: unknown, +): mimeType is ScreenshotImageMimeType { + return ( + mimeType === 'image/png' || + mimeType === 'image/jpeg' || + mimeType === 'image/webp' + ); +} + +export function inferScreenshotImageFormatFromBase64( + base64Body: string, +): ScreenshotImageFormat | undefined { + const normalizedBody = base64Body.replace(/\s/g, ''); + if (normalizedBody.startsWith('iVBORw0KGgo')) { + return 'png'; + } + if (normalizedBody.startsWith('/9j/')) { + return 'jpeg'; + } + if (normalizedBody.startsWith('UklGR')) { + return 'webp'; + } + return undefined; +} + +export function detectScreenshotImageFormatFromBuffer( + buffer: Uint8Array, +): ScreenshotImageFormat | undefined { + if ( + buffer.length >= 8 && + buffer[0] === 0x89 && + buffer[1] === 0x50 && + buffer[2] === 0x4e && + buffer[3] === 0x47 && + buffer[4] === 0x0d && + buffer[5] === 0x0a && + buffer[6] === 0x1a && + buffer[7] === 0x0a + ) { + return 'png'; + } + + if ( + buffer.length >= 3 && + buffer[0] === 0xff && + buffer[1] === 0xd8 && + buffer[2] === 0xff + ) { + return 'jpeg'; + } + + if ( + buffer.length >= 12 && + buffer[0] === 0x52 && + buffer[1] === 0x49 && + buffer[2] === 0x46 && + buffer[3] === 0x46 && + buffer[8] === 0x57 && + buffer[9] === 0x45 && + buffer[10] === 0x42 && + buffer[11] === 0x50 + ) { + return 'webp'; + } + + return undefined; +} diff --git a/packages/shared/src/img/index.ts b/packages/shared/src/img/index.ts index 9079c4e7f6..4754cc8a9e 100644 --- a/packages/shared/src/img/index.ts +++ b/packages/shared/src/img/index.ts @@ -2,13 +2,29 @@ export { imageInfoOfBase64, isValidPNGImageBuffer, isValidJPEGImageBuffer, + isValidWebPImageBuffer, isValidImageBuffer, validateScreenshotBuffer, type ValidateScreenshotBufferOptions, } from './info'; +export { + detectScreenshotImageFormatFromBuffer, + inferScreenshotImageFormatFromBase64, + isScreenshotImageMimeType, + screenshotImageExtension, + screenshotImageFormatFromExtension, + screenshotImageFormatFromMimeType, + screenshotImageMimeType, + type ScreenshotImageFormat, + type ScreenshotImageMimeType, +} from './image-format'; export { resizeAndConvertImgBuffer, convertImgBufferToJpeg, + convertImgBufferToWebp, + canonicalizeScreenshotBase64, + DEFAULT_WEBP_SCREENSHOT_EFFORT, + DEFAULT_WEBP_SCREENSHOT_QUALITY, resizeImgBase64, zoomForGPT4o, saveBase64Image, @@ -24,6 +40,8 @@ export { normalizeBase64Image, normalizeScreenshotBase64, type NormalizeScreenshotBase64Options, + type CanonicalizeScreenshotOptions, + type WebpScreenshotEncodeOptions, } from './transform'; export { processImageElementInfo, @@ -31,3 +49,7 @@ export { compositePointMarkerImg, annotateRects, } from './box-select'; +export { + encodeRgbaToWebp, + type BrowserWebpEncodeInput, +} from './browser-webp-encoder'; diff --git a/packages/shared/src/img/info.ts b/packages/shared/src/img/info.ts index a06d1b42fe..75cc13d656 100644 --- a/packages/shared/src/img/info.ts +++ b/packages/shared/src/img/info.ts @@ -4,6 +4,7 @@ import type { Size } from '../types'; import { ifInNode } from '../utils'; import getPhoton from './get-photon'; import getSharp from './get-sharp'; +import { detectScreenshotImageFormatFromBuffer } from './image-format'; export interface ImageInfo extends Size {} @@ -118,12 +119,25 @@ export function isValidJPEGImageBuffer(buffer: Buffer): boolean { } /** - * Check if the Buffer is a valid image (PNG or JPEG) + * Check if the Buffer has a WebP signature. * @param buffer The Buffer to check - * @returns true if the Buffer is a valid PNG or JPEG image, otherwise false + * @returns true if the Buffer has a WebP signature, otherwise false + */ +export function isValidWebPImageBuffer(buffer: Buffer): boolean { + return detectScreenshotImageFormatFromBuffer(buffer) === 'webp'; +} + +/** + * Check if the Buffer is a supported screenshot image (PNG, JPEG, or WebP) + * @param buffer The Buffer to check + * @returns true if the Buffer has a supported image signature, otherwise false */ export function isValidImageBuffer(buffer: Buffer): boolean { - return isValidPNGImageBuffer(buffer) || isValidJPEGImageBuffer(buffer); + return ( + isValidPNGImageBuffer(buffer) || + isValidJPEGImageBuffer(buffer) || + isValidWebPImageBuffer(buffer) + ); } export interface ValidateScreenshotBufferOptions { diff --git a/packages/shared/src/img/transform.ts b/packages/shared/src/img/transform.ts index caeb583015..c44c9c3064 100644 --- a/packages/shared/src/img/transform.ts +++ b/packages/shared/src/img/transform.ts @@ -7,11 +7,62 @@ import type { PhotonImage as PhotonImageType } from '@silvia-odwyer/photon'; import { getDebug } from '../logger'; import type { Rect } from '../types'; import { ifInNode } from '../utils'; +import { encodeRgbaToWebp } from './browser-webp-encoder'; import getPhoton from './get-photon'; import getSharp from './get-sharp'; +import { + type ScreenshotImageFormat, + detectScreenshotImageFormatFromBuffer, + inferScreenshotImageFormatFromBase64, + screenshotImageMimeType, +} from './image-format'; const imgDebug = getDebug('img'); +export const DEFAULT_WEBP_SCREENSHOT_QUALITY = 90; +export const DEFAULT_WEBP_SCREENSHOT_EFFORT = 1; + +export interface WebpScreenshotEncodeOptions { + /** Encoder quality from 0 to 100. Defaults to 90. */ + quality?: number; + /** Sharp encoder CPU effort from 0 to 6. Defaults to 1. */ + effort?: number; +} + +export interface CanonicalizeScreenshotOptions + extends WebpScreenshotEncodeOptions { + /** Keep a valid JPEG source byte-for-byte instead of applying another lossy encode. */ + preserveJpeg?: boolean; +} + +function assertWebpBuffer(buffer: Uint8Array, label: string): void { + if (detectScreenshotImageFormatFromBuffer(buffer) !== 'webp') { + throw new Error(`${label} did not produce a valid WebP image`); + } +} + +interface BrowserImagePixels { + get_raw_pixels(): Uint8Array; + get_width(): number; + get_height(): number; +} + +async function encodeBrowserImageToWebp( + image: BrowserImagePixels, + quality = DEFAULT_WEBP_SCREENSHOT_QUALITY, +): Promise { + const output = Buffer.from( + await encodeRgbaToWebp({ + pixels: image.get_raw_pixels(), + width: image.get_width(), + height: image.get_height(), + quality, + }), + ); + assertWebpBuffer(output, 'Browser image encoder'); + return output; +} + /** * Saves a Base64-encoded image to a file * @@ -33,7 +84,7 @@ export async function saveBase64Image(options: { /** * Resizes an image from Buffer, maybe return a new format - * - If the image is Resized, the returned format will be jpg. + * - If the image is resized, the returned format will be WebP. * - If the image is not Resized, it will return to its original format. * @returns { buffer: resized buffer, format: the new format} */ @@ -78,8 +129,12 @@ export async function resizeAndConvertImgBuffer( const resizedBuffer = await Sharp(inputData) .resize(newSize.width, newSize.height) - .jpeg({ quality: 90 }) + .webp({ + quality: DEFAULT_WEBP_SCREENSHOT_QUALITY, + effort: DEFAULT_WEBP_SCREENSHOT_EFFORT, + }) .toBuffer(); + assertWebpBuffer(resizedBuffer, 'Sharp resize'); const resizeEndTime = Date.now(); imgDebug( @@ -88,8 +143,7 @@ export async function resizeAndConvertImgBuffer( return { buffer: resizedBuffer, - // by Sharp.jpeg() - format: 'jpeg', + format: 'webp', }; } @@ -126,8 +180,7 @@ export async function resizeAndConvertImgBuffer( SamplingFilter.CatmullRom, ); - const outputBytes = outputImage.get_bytes_jpeg(90); - const resizedBuffer = Buffer.from(outputBytes); + const resizedBuffer = await encodeBrowserImageToWebp(outputImage); // Free memory inputImage.free(); @@ -141,8 +194,7 @@ export async function resizeAndConvertImgBuffer( return { buffer: resizedBuffer, - // by Photon.get_bytes_jpeg() - format: 'jpeg', + format: 'webp', }; } @@ -173,50 +225,53 @@ export async function convertImgBufferToJpeg( } } +/** Convert an image buffer to a validated WebP image without resizing it. */ +export async function convertImgBufferToWebp( + inputData: Buffer, + options: WebpScreenshotEncodeOptions = {}, +): Promise { + const quality = options.quality ?? DEFAULT_WEBP_SCREENSHOT_QUALITY; + const effort = options.effort ?? DEFAULT_WEBP_SCREENSHOT_EFFORT; + + if (ifInNode) { + const Sharp = await getSharp(); + const output = await Sharp(inputData).webp({ quality, effort }).toBuffer(); + assertWebpBuffer(output, 'Sharp'); + return output; + } + + const mimeType = detectImageMimeTypeFromBuffer(inputData); + if (!mimeType) { + throw new Error('Cannot encode WebP from an unsupported image buffer'); + } + const photonImage = await photonFromBase64( + `data:${mimeType};base64,${inputData.toString('base64')}`, + ); + try { + return await encodeBrowserImageToWebp(photonImage, quality); + } finally { + photonImage.free(); + } +} + const base64ImageDataUrlPattern = /^data:image\/[a-zA-Z0-9.+-]+;base64,/i; const supportedScreenshotDataUriPattern = - /^data:image\/(png|jpe?g);base64,([\s\S]*)$/i; + /^data:image\/(png|jpe?g|webp);base64,([\s\S]*)$/i; const rawBase64BodyPattern = /^[A-Za-z0-9+/=\s]+$/; -export const inferBase64ImageFormat = (base64Body: string) => { - if (base64Body.startsWith('iVBORw0KGgo')) { - return 'png'; - } - return 'jpeg'; -}; +export const inferBase64ImageFormat = ( + base64Body: string, +): ScreenshotImageFormat => + inferScreenshotImageFormatFromBase64(base64Body) ?? 'jpeg'; function detectImageMimeTypeFromBuffer(buffer: Buffer): string | undefined { - if ( - buffer.length >= 8 && - buffer[0] === 0x89 && - buffer[1] === 0x50 && - buffer[2] === 0x4e && - buffer[3] === 0x47 && - buffer[4] === 0x0d && - buffer[5] === 0x0a && - buffer[6] === 0x1a && - buffer[7] === 0x0a - ) { - return 'image/png'; - } - if ( - buffer.length >= 3 && - buffer[0] === 0xff && - buffer[1] === 0xd8 && - buffer[2] === 0xff - ) { - return 'image/jpeg'; + const screenshotFormat = detectScreenshotImageFormatFromBuffer(buffer); + if (screenshotFormat) { + return screenshotImageMimeType(screenshotFormat); } if (buffer.length >= 6 && buffer.subarray(0, 3).toString('ascii') === 'GIF') { return 'image/gif'; } - if ( - buffer.length >= 12 && - buffer.subarray(0, 4).toString('ascii') === 'RIFF' && - buffer.subarray(8, 12).toString('ascii') === 'WEBP' - ) { - return 'image/webp'; - } if (buffer.length >= 2 && buffer[0] === 0x42 && buffer[1] === 0x4d) { return 'image/bmp'; } @@ -243,10 +298,10 @@ export const normalizeScreenshotBase64 = ( const dataUriMatch = trimmedBase64.match(supportedScreenshotDataUriPattern); if (dataUriMatch) { - const imageFormat = + const imageFormat: ScreenshotImageFormat = dataUriMatch[1].toLowerCase() === 'jpg' ? 'jpeg' - : dataUriMatch[1].toLowerCase(); + : (dataUriMatch[1].toLowerCase() as ScreenshotImageFormat); const body = dataUriMatch[2]; if (!normalizeBase64Body(body)) { throw new Error(`${label} cannot be empty`); @@ -256,17 +311,22 @@ export const normalizeScreenshotBase64 = ( if (trimmedBase64.startsWith('data:')) { throw new Error( - `${label} must be a PNG/JPEG data URI or raw PNG base64 string`, + `${label} must be a PNG/JPEG/WebP data URI or raw PNG/WebP base64 string`, ); } if (!rawBase64BodyPattern.test(trimmedBase64)) { throw new Error( - `${label} must be a PNG/JPEG data URI or raw PNG base64 string`, + `${label} must be a PNG/JPEG/WebP data URI or raw PNG/WebP base64 string`, ); } - return createImgBase64ByFormat('png', trimmedBase64); + const base64Body = normalizeBase64Body(trimmedBase64); + const inferredFormat = inferScreenshotImageFormatFromBase64(base64Body); + return createImgBase64ByFormat( + inferredFormat === 'webp' ? 'webp' : 'png', + base64Body, + ); }; export const normalizeBase64Image = (base64: string) => { @@ -283,6 +343,38 @@ export const normalizeBase64Image = (base64: string) => { ); }; +/** + * Normalize a screenshot at the AI/report boundary. + * + * Valid WebP is passed through byte-for-byte. Callers can also preserve JPEG + * sources to avoid a second lossy encode for native MJPEG/HDC streams. + */ +export async function canonicalizeScreenshotBase64( + inputBase64: string, + options: CanonicalizeScreenshotOptions = {}, +): Promise { + const { body } = parseBase64(inputBase64); + const inputBuffer = Buffer.from(body, 'base64'); + const inputFormat = detectScreenshotImageFormatFromBuffer(inputBuffer); + if (!inputFormat) { + throw new Error('Cannot canonicalize an unsupported screenshot image'); + } + + if ( + inputFormat === 'webp' || + (inputFormat === 'jpeg' && options.preserveJpeg) + ) { + return createImgBase64ByFormat(inputFormat, body); + } + + const startedAt = Date.now(); + const output = await convertImgBufferToWebp(inputBuffer, options); + imgDebug( + `canonicalizeScreenshot done, ${inputFormat}->webp, bytes: ${inputBuffer.length}->${output.length}, cost: ${Date.now() - startedAt}ms`, + ); + return createImgBase64ByFormat('webp', output.toString('base64')); +} + export async function resizeImgBase64( inputBase64: string, newSize: { @@ -425,12 +517,16 @@ export async function paddingToMatchBlockByBase64( bottom: targetHeight - height, background: { r: 255, g: 255, b: 255, alpha: 1 }, }) - .jpeg({ quality: 90 }) + .webp({ + quality: DEFAULT_WEBP_SCREENSHOT_QUALITY, + effort: DEFAULT_WEBP_SCREENSHOT_EFFORT, + }) .toBuffer(); + assertWebpBuffer(output, 'Sharp padding'); return { width: targetWidth, height: targetHeight, - imageBase64: createImgBase64ByFormat('jpeg', output.toString('base64')), + imageBase64: createImgBase64ByFormat('webp', output.toString('base64')), }; } @@ -473,12 +569,16 @@ export async function cropByRect( width, height, }) - .jpeg({ quality: 90 }) + .webp({ + quality: DEFAULT_WEBP_SCREENSHOT_QUALITY, + effort: DEFAULT_WEBP_SCREENSHOT_EFFORT, + }) .toBuffer(); + assertWebpBuffer(output, 'Sharp crop'); return { width, height, - imageBase64: createImgBase64ByFormat('jpeg', output.toString('base64')), + imageBase64: createImgBase64ByFormat('webp', output.toString('base64')), }; } @@ -503,11 +603,10 @@ export async function cropByRect( export async function photonToBase64( image: PhotonImageType, - quality = 90, + quality = DEFAULT_WEBP_SCREENSHOT_QUALITY, ): Promise { - const bytes = image.get_bytes_jpeg(quality); - const base64Body = Buffer.from(bytes).toString('base64'); - return `data:image/jpeg;base64,${base64Body}`; + const bytes = await encodeBrowserImageToWebp(image, quality); + return createImgBase64ByFormat('webp', bytes.toString('base64')); } export const httpImg2Base64 = async (url: string): Promise => { @@ -657,17 +756,22 @@ export async function scaleImage( kernel: 'lanczos3', fit: 'fill', }) - .jpeg({ - quality: 90, + .webp({ + quality: DEFAULT_WEBP_SCREENSHOT_QUALITY, + effort: DEFAULT_WEBP_SCREENSHOT_EFFORT, }) .toBuffer(); + assertWebpBuffer(resizedBuffer, 'Sharp scale'); const scaleEndTime = Date.now(); imgDebug( `scaleImage done (Sharp): ${originalWidth}x${originalHeight} -> ${newWidth}x${newHeight} (scale=${scale}), cost: ${scaleEndTime - scaleStartTime}ms`, ); - const base64 = `data:image/jpeg;base64,${resizedBuffer.toString('base64')}`; + const base64 = createImgBase64ByFormat( + 'webp', + resizedBuffer.toString('base64'), + ); return { width: newWidth, @@ -703,8 +807,7 @@ export async function scaleImage( SamplingFilter.CatmullRom, ); - const outputBytes = outputImage.get_bytes_jpeg(90); - const resizedBuffer = Buffer.from(outputBytes); + const resizedBuffer = await encodeBrowserImageToWebp(outputImage); // Free memory inputImage.free(); @@ -715,7 +818,10 @@ export async function scaleImage( `scaleImage done (Photon): ${originalWidth}x${originalHeight} -> ${newWidth}x${newHeight} (scale=${scale}), cost: ${scaleEndTime - scaleStartTime}ms`, ); - const base64 = `data:image/jpeg;base64,${resizedBuffer.toString('base64')}`; + const base64 = createImgBase64ByFormat( + 'webp', + resizedBuffer.toString('base64'), + ); return { width: newWidth, diff --git a/packages/shared/src/utils.ts b/packages/shared/src/utils.ts index dec7ef4eb3..31b5d54c10 100644 --- a/packages/shared/src/utils.ts +++ b/packages/shared/src/utils.ts @@ -12,6 +12,11 @@ export function uuid(): string { return generateUUID(); } +/** Return the lowercase SHA-256 digest of UTF-8 text or raw bytes. */ +export function sha256Hex(input: string | Uint8Array): string { + return sha256.create().update(input).hex(); +} + const hashMap: Record = {}; // id - combined export function generateHashId(rect: any, content = ''): string { diff --git a/packages/shared/tests/unit-test/image/index.test.ts b/packages/shared/tests/unit-test/image/index.test.ts index f5f9adba62..a20c4c93cd 100644 --- a/packages/shared/tests/unit-test/image/index.test.ts +++ b/packages/shared/tests/unit-test/image/index.test.ts @@ -8,6 +8,7 @@ import { httpImg2Base64, imageInfoOfBase64, isValidPNGImageBuffer, + isValidWebPImageBuffer, localImg2Base64, resizeAndConvertImgBuffer, resizeImgBase64, @@ -22,6 +23,10 @@ import { } from '../../../src/img/transform'; import { getFixture } from '../../utils'; +const webpBase64 = + 'UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; +const webpDataUri = `data:image/webp;base64,${webpBase64}`; + describe('imageInfoOfBase64', () => { it('returns correct dimensions for PNG image', async () => { const image = getFixture('icon.png'); @@ -41,6 +46,13 @@ describe('imageInfoOfBase64', () => { expect(info.height).toBe(905); }); + it('returns correct dimensions for WebP image', async () => { + await expect(imageInfoOfBase64(webpDataUri)).resolves.toEqual({ + width: 2, + height: 3, + }); + }); + it('works with base64 string without data URI header', async () => { const image = getFixture('icon.png'); const base64WithHeader = localImg2Base64(image); @@ -320,6 +332,15 @@ describe('image utils', () => { ).toThrow('Screenshot buffer has invalid image format'); }); + it('isValidWebPImageBuffer accepts WebP and rejects malformed RIFF data', () => { + expect(isValidWebPImageBuffer(Buffer.from(webpBase64, 'base64'))).toBe( + true, + ); + expect(isValidWebPImageBuffer(Buffer.from('RIFF1234NOPE', 'ascii'))).toBe( + false, + ); + }); + it('validateScreenshotBuffer accepts valid screenshots above the minimum size', () => { const buffer = readFileSync(getFixture('icon.png')); @@ -466,7 +487,7 @@ describe('resizeAndConvertImgBuffer', () => { ); expect(format).toBe('png'); }); - it('Sharp resize will get jpeg format', async () => { + it('Sharp resize will get WebP format', async () => { const { format, buffer } = await resizeAndConvertImgBuffer( 'png', imageBuffer, @@ -475,7 +496,9 @@ describe('resizeAndConvertImgBuffer', () => { height: 1, }, ); - expect(format).toBe('jpeg'); + expect(format).toBe('webp'); + expect(buffer.subarray(0, 4).toString('ascii')).toBe('RIFF'); + expect(buffer.subarray(8, 12).toString('ascii')).toBe('WEBP'); }); }); diff --git a/packages/shared/tests/unit-test/screenshot-file.test.ts b/packages/shared/tests/unit-test/screenshot-file.test.ts new file mode 100644 index 0000000000..cd81bd43f4 --- /dev/null +++ b/packages/shared/tests/unit-test/screenshot-file.test.ts @@ -0,0 +1,62 @@ +import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { writeCliScreenshotFile } from '../../src/cli/screenshot-file'; +import { collectScreenshotRefs } from '../../src/cli/verbose-screenshot'; + +const webpBase64 = + 'UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; + +describe('CLI WebP screenshot files', () => { + const temporaryDirectories: string[] = []; + + const makeTemporaryDirectory = () => { + const directory = mkdtempSync(join(tmpdir(), 'midscene-webp-')); + temporaryDirectories.push(directory); + return directory; + }; + + afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it('uses the .webp extension from screenshot MIME metadata', () => { + const directoryPath = makeTemporaryDirectory(); + const filePath = writeCliScreenshotFile(webpBase64, { + id: 'webp-shot', + mimeType: 'image/webp', + directoryPath, + }); + + expect(filePath).toBe(join(directoryPath, 'webp-shot.webp')); + expect(readFileSync(filePath).toString('base64')).toBe(webpBase64); + }); + + it('exports inline WebP screenshots for verbose output', () => { + const directoryPath = makeTemporaryDirectory(); + const screenshot = { + base64: `data:image/webp;base64,${webpBase64}`, + extension: 'webp', + toSerializable: () => ({ + type: 'midscene_screenshot_ref' as const, + id: 'inline-webp', + capturedAt: 1, + mimeType: 'image/webp', + storage: 'inline', + }), + }; + + const [collected] = collectScreenshotRefs(screenshot, { + exportMode: 'report', + reportFile: join(directoryPath, 'report.html'), + }); + + expect(collected.file).toBe('inline-webp.webp'); + expect( + existsSync(join(directoryPath, 'screenshots', 'inline-webp.webp')), + ).toBe(true); + }); +}); diff --git a/packages/shared/tests/unit-test/transform.test.ts b/packages/shared/tests/unit-test/transform.test.ts index cb7bbedf9f..e595175d7e 100644 --- a/packages/shared/tests/unit-test/transform.test.ts +++ b/packages/shared/tests/unit-test/transform.test.ts @@ -1,6 +1,8 @@ import path from 'node:path'; import { describe, expect, it, vi } from 'vitest'; +import { encodeRgbaToWebp } from '../../src/img/browser-webp-encoder'; import { + canonicalizeScreenshotBase64, inferBase64ImageFormat, normalizeBase64Image, normalizeScreenshotBase64, @@ -8,6 +10,39 @@ import { scaleImage, } from '../../src/img/transform'; +describe('encodeRgbaToWebp', () => { + it('validates dimensions before selecting a browser encoder', async () => { + await expect( + encodeRgbaToWebp({ + pixels: [], + width: 0, + height: 1, + }), + ).rejects.toThrow('WebP image dimensions must be positive safe integers'); + }); + + it('validates the RGBA pixel length', async () => { + await expect( + encodeRgbaToWebp({ + pixels: [255, 255, 255], + width: 1, + height: 1, + }), + ).rejects.toThrow('WebP RGBA pixel length must be 4, got 3'); + }); + + it('validates encoder quality', async () => { + await expect( + encodeRgbaToWebp({ + pixels: [255, 255, 255, 255], + width: 1, + height: 1, + quality: 101, + }), + ).rejects.toThrow('WebP quality must be between 0 and 100'); + }); +}); + describe('preapareImageUrl', () => { it('url is not a string will throw an error', async () => { await expect(preProcessImageUrl(1 as any, false)).rejects.toThrowError( @@ -95,16 +130,25 @@ describe('normalizeBase64Image', () => { 'data:image/jpeg;base64,/9j/4AAQSkZJRg==', ); }); + + it('wraps bare WebP base64 with the WebP MIME type', () => { + expect(normalizeBase64Image(' UklGRjQAAABXRUJQ VlA4IA== ')).toBe( + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4IA==', + ); + }); }); describe('normalizeScreenshotBase64', () => { - it('accepts PNG and JPEG data urls', () => { + it('accepts PNG, JPEG, and WebP data urls', () => { expect( normalizeScreenshotBase64(' data:image/png;base64,aaa\r\nbbb '), ).toBe('data:image/png;base64,aaabbb'); expect(normalizeScreenshotBase64('data:image/jpeg;base64,/9j/4AAQ')).toBe( 'data:image/jpeg;base64,/9j/4AAQ', ); + expect( + normalizeScreenshotBase64('data:image/webp;base64,UklGRjQAAA=='), + ).toBe('data:image/webp;base64,UklGRjQAAA=='); }); it('normalizes jpg data urls to jpeg', () => { @@ -119,6 +163,12 @@ describe('normalizeScreenshotBase64', () => { ); }); + it('recognizes raw WebP base64', () => { + expect(normalizeScreenshotBase64(' UklGRjQAAA BXRUJQ ')).toBe( + 'data:image/webp;base64,UklGRjQAAABXRUJQ', + ); + }); + it('uses the provided label in validation errors', () => { expect(() => normalizeScreenshotBase64(' ', { label: 'custom screenshot' }), @@ -129,14 +179,15 @@ describe('normalizeScreenshotBase64', () => { label: 'custom screenshot', }), ).toThrow( - 'custom screenshot must be a PNG/JPEG data URI or raw PNG base64 string', + 'custom screenshot must be a PNG/JPEG/WebP data URI or raw PNG/WebP base64 string', ); }); }); describe('inferBase64ImageFormat', () => { - it('detects png payloads and otherwise falls back to jpeg', () => { + it('detects PNG and WebP payloads and otherwise falls back to JPEG', () => { expect(inferBase64ImageFormat('iVBORw0KGgoaaa')).toBe('png'); + expect(inferBase64ImageFormat('UklGRjQAAABXRUJQ')).toBe('webp'); expect(inferBase64ImageFormat('/9j/4AAQSkZJRg==')).toBe('jpeg'); }); }); @@ -152,7 +203,7 @@ describe('scaleImage', () => { expect(result.width).toBe(2); expect(result.height).toBe(2); expect(result.imageBase64).toMatchInlineSnapshot( - `"data:image/jpeg;base64,/9j/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAACAAIDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAn/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFAEBAAAAAAAAAAAAAAAAAAAAAP/EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhEDEQA/AKpgA//Z"`, + `"data:image/webp;base64,UklGRioAAABXRUJQVlA4IB4AAACQAQCdASoCAAIAAMASJaQAAzoO0gAA/v/+JwoMAAA="`, ); }); @@ -162,7 +213,7 @@ describe('scaleImage', () => { expect(result.width).toBe(3); expect(result.height).toBe(3); expect(result.imageBase64).toMatchInlineSnapshot( - `"data:image/jpeg;base64,/9j/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAADAAMDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAn/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFAEBAAAAAAAAAAAAAAAAAAAAAP/EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhEDEQA/AKpgA//Z"`, + `"data:image/webp;base64,UklGRioAAABXRUJQVlA4IB4AAACQAQCdASoDAAMAAMASJaQAAzoO0gAA/v/+JwoMAAA="`, ); }); @@ -209,3 +260,30 @@ describe('scaleImage', () => { vi.restoreAllMocks(); }); }); + +describe('canonicalizeScreenshotBase64', () => { + const onePixelWhiteImage = + 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wCEAAMDAwMDAwMDAwMEBAQEBAYFBQUFBgkGBwYHBgkOCAoICAoIDgwPDAsMDwwWEQ8PERYZFRQVGR4bGx4mJCYyMkMBAwMDAwMDAwMDAwQEBAQEBgUFBQUGCQYHBgcGCQ4ICggICggODA8MCwwPDBYRDw8RFhkVFBUZHhsbHiYkJjIyQ//CABEIAAEAAQMBIgACEQEDEQH/xAAnAAEBAAAAAAAAAAAAAAAAAAAACQEBAAAAAAAAAAAAAAAAAAAAAP/aAAwDAQACEAMQAAAAqmD/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAE/AH//xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAECAQE/AH//xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAEDAQE/AH//2Q=='; + + it('passes an existing WebP through byte-for-byte', async () => { + const webp = await canonicalizeScreenshotBase64(onePixelWhiteImage); + expect(await canonicalizeScreenshotBase64(webp)).toBe(webp); + }); + + it('preserves native JPEG only when auto policy requests it', async () => { + expect( + await canonicalizeScreenshotBase64(onePixelWhiteImage, { + preserveJpeg: true, + }), + ).toBe(onePixelWhiteImage); + expect(await canonicalizeScreenshotBase64(onePixelWhiteImage)).toMatch( + /^data:image\/webp;base64,UklGR/, + ); + }); + + it('rejects unsupported image bytes instead of returning a blank image', async () => { + await expect( + canonicalizeScreenshotBase64('data:image/png;base64,aGVsbG8='), + ).rejects.toThrow('unsupported screenshot image'); + }); +}); diff --git a/packages/visualizer/src/hooks/usePlaygroundExecution.ts b/packages/visualizer/src/hooks/usePlaygroundExecution.ts index 965b7f316d..a0b475ad8a 100644 --- a/packages/visualizer/src/hooks/usePlaygroundExecution.ts +++ b/packages/visualizer/src/hooks/usePlaygroundExecution.ts @@ -10,6 +10,10 @@ import { parseImageScripts, restoreImageReferences, } from '@midscene/core/dump'; +import { + screenshotImageExtension, + screenshotImageFormatFromMimeType, +} from '@midscene/shared/img/image-format'; import { useCallback } from 'react'; import { useEnvConfig } from '../store/store'; import type { @@ -154,7 +158,11 @@ async function loadReportReplay( } const dump = (await response.json()) as IReportActionDump; result.dump = restoreImageReferences(dump, (ref) => { - const extension = ref.mimeType === 'image/jpeg' ? 'jpeg' : 'png'; + const format = screenshotImageFormatFromMimeType(ref.mimeType); + if (!format) { + throw new Error(`Unsupported screenshot mime type: ${ref.mimeType}`); + } + const extension = screenshotImageExtension(format); return new URL( `screenshots/${encodeURIComponent(ref.id)}.${extension}`, result.report!.url, diff --git a/packages/visualizer/tests/playground-execution-stop.test.ts b/packages/visualizer/tests/playground-execution-stop.test.ts index 9047546cfe..f9b401ba95 100644 --- a/packages/visualizer/tests/playground-execution-stop.test.ts +++ b/packages/visualizer/tests/playground-execution-stop.test.ts @@ -95,7 +95,7 @@ function Harness({ return null; } -function replayDump() { +function replayDump(mimeType: 'image/png' | 'image/webp' = 'image/png') { return { sdkVersion: 'test', groupName: 'Playground run', @@ -113,7 +113,7 @@ function replayDump() { type: 'midscene_screenshot_ref', id: 'shot-1', capturedAt: 1, - mimeType: 'image/png', + mimeType, storage: 'inline', }, }, @@ -276,7 +276,7 @@ describe('usePlaygroundExecution stop handling', () => { const fetchMock = vi.fn(async () => Promise.resolve({ ok: true, - json: async () => replayDump(), + json: async () => replayDump('image/webp'), }), ); vi.stubGlobal('fetch', fetchMock); @@ -331,7 +331,7 @@ describe('usePlaygroundExecution stop handling', () => { const restoredDump = allScriptsFromDumpMock.mock.calls[0]?.[0] as any; expect( restoredDump.executions[0].tasks[0].uiContext.screenshot.base64, - ).toBe('http://localhost/reports/report-1/screenshots/shot-1.png'); + ).toBe('http://localhost/reports/report-1/screenshots/shot-1.webp'); await act(async () => root.unmount()); });