diff --git a/apps/report/src/App.tsx b/apps/report/src/App.tsx index 9434cd4032..f09b140d27 100644 --- a/apps/report/src/App.tsx +++ b/apps/report/src/App.tsx @@ -54,12 +54,16 @@ import { getEmptyDumpDescription, parseDumpAttributes, } from './utils/report-dump'; +import { + type ReportScreenshotSourceRef, + resolveScreenshotFallbackPath, +} from './utils/screenshot-source'; // Shared image cache across all test cases — resolved images are cached by id const imageCache = new Map(); function resolveImageFromDom( - refOrId: string | { id: string; storage?: 'inline' | 'file'; path?: string }, + refOrId: string | ReportScreenshotSourceRef, ): string { const id = typeof refOrId === 'string' ? refOrId : refOrId.id; const cached = imageCache.get(id); @@ -74,12 +78,7 @@ function resolveImageFromDom( return data; } - if (typeof refOrId === 'object' && refOrId?.storage === 'file') { - return refOrId.path || `./screenshots/${id}.png`; - } - - // Fallback to directory path - return `./screenshots/${id}.png`; + return resolveScreenshotFallbackPath(refOrId); } let globalRenderCount = 1; diff --git a/apps/report/src/components/playground/index.tsx b/apps/report/src/components/playground/index.tsx index 3b23220cef..59a445db37 100644 --- a/apps/report/src/components/playground/index.tsx +++ b/apps/report/src/components/playground/index.tsx @@ -14,6 +14,10 @@ import { } from '@midscene/core/dump'; import { type PlaygroundSDK, noReplayAPIs } from '@midscene/playground'; import type { ServerResponse } from '@midscene/playground'; +import { + screenshotImageExtension, + screenshotImageFormatFromMimeType, +} from '@midscene/shared/img/image-format'; import { ContextPreview, Logo, @@ -55,7 +59,11 @@ async function loadReferencedReplay(result: PlaygroundResult) { } const dump = (await response.json()) as IReportActionDump; result.dump = restoreImageReferences(dump, (ref) => { - const extension = ref.mimeType === 'image/jpeg' ? 'jpeg' : 'png'; + const format = screenshotImageFormatFromMimeType(ref.mimeType); + if (!format) { + throw new Error(`Unsupported screenshot mime type: ${ref.mimeType}`); + } + const extension = screenshotImageExtension(format); return new URL( `screenshots/${encodeURIComponent(ref.id)}.${extension}`, result.report!.url, diff --git a/apps/report/src/components/timeline/build-timeline-screenshots.test.ts b/apps/report/src/components/timeline/build-timeline-screenshots.test.ts index cae0e5a947..3f85fad707 100644 --- a/apps/report/src/components/timeline/build-timeline-screenshots.test.ts +++ b/apps/report/src/components/timeline/build-timeline-screenshots.test.ts @@ -4,6 +4,8 @@ import { buildTimelineScreenshots } from './build-timeline-screenshots'; const onePixelPngBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAFgwJ/lz8yrwAAAABJRU5ErkJggg=='; +const webpBase64 = + 'UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; interface TaskFixtureOptions { id: string; @@ -103,6 +105,19 @@ describe('buildTimelineScreenshots', () => { ); }); + it('labels raw WebP recorder screenshots with the WebP MIME type', () => { + const task = makeTask({ + id: 'raw-webp-recorder', + startTs: 1000, + recorder: [{ ts: 1200, screenshot: webpBase64 }], + }); + + const { allScreenshots } = buildTimelineScreenshots([task]); + + expect(allScreenshots).toHaveLength(1); + expect(allScreenshots[0].img).toBe(`data:image/webp;base64,${webpBase64}`); + }); + it('keeps already-prefixed data URLs unchanged', () => { const dataUrl = `data:image/png;base64,${onePixelPngBase64}`; const task = makeTask({ diff --git a/apps/report/src/components/timeline/build-timeline-screenshots.ts b/apps/report/src/components/timeline/build-timeline-screenshots.ts index 093a58df47..93b5d3937b 100644 --- a/apps/report/src/components/timeline/build-timeline-screenshots.ts +++ b/apps/report/src/components/timeline/build-timeline-screenshots.ts @@ -1,4 +1,8 @@ import type { ExecutionTask } from '@midscene/core'; +import { + inferScreenshotImageFormatFromBase64, + screenshotImageMimeType, +} from '@midscene/shared/img/image-format'; export interface TimelineScreenshot { id: string; @@ -30,11 +34,8 @@ const imageSrcFromString = (value: string): string => { } const body = trimmed.replace(/\s/g, ''); - const mimeType = body.startsWith('/9j/') - ? 'image/jpeg' - : body.startsWith('UklGR') - ? 'image/webp' - : 'image/png'; + const format = inferScreenshotImageFormatFromBase64(body) ?? 'png'; + const mimeType = screenshotImageMimeType(format); return `data:${mimeType};base64,${body}`; }; diff --git a/apps/report/src/utils/markdown-export.test.ts b/apps/report/src/utils/markdown-export.test.ts index 1f9d49be17..8aee18695b 100644 --- a/apps/report/src/utils/markdown-export.test.ts +++ b/apps/report/src/utils/markdown-export.test.ts @@ -105,6 +105,27 @@ describe('markdown-export helpers', () => { ); }); + it('packages WebP data URI attachments without changing their bytes', () => { + const files = buildMarkdownArchiveFiles('# webp report', [ + { + id: 'webp-inline', + suggestedFileName: 'webp-inline.webp', + mimeType: 'image/webp', + executionIndex: 0, + taskIndex: 0, + base64Data: `data:image/webp;base64,${btoa('webp-bytes')}`, + }, + ]); + + expect(Object.keys(files).sort()).toEqual([ + 'report.md', + 'screenshots/webp-inline.webp', + ]); + expect( + new TextDecoder().decode(files['screenshots/webp-inline.webp']), + ).toBe('webp-bytes'); + }); + it('builds display items from markdown attachment names and paths', () => { const items = getMarkdownAttachmentDisplayItems([ { diff --git a/apps/report/src/utils/markdown-export.ts b/apps/report/src/utils/markdown-export.ts index 53cfb7c632..179a3e2f86 100644 --- a/apps/report/src/utils/markdown-export.ts +++ b/apps/report/src/utils/markdown-export.ts @@ -1,7 +1,7 @@ import type { MarkdownAttachment } from '@midscene/core'; const defaultScreenshotBaseDir = './screenshots'; -const dataUrlBase64Pattern = /^data:image\/(?:png|jpeg|jpg);base64,/i; +const dataUrlBase64Pattern = /^data:image\/(?:png|jpeg|jpg|webp);base64,/i; const rawBase64Pattern = /^[a-zA-Z0-9+/=\s]+$/; type MarkdownExport = { diff --git a/apps/report/src/utils/screenshot-source.test.ts b/apps/report/src/utils/screenshot-source.test.ts new file mode 100644 index 0000000000..c811d836bb --- /dev/null +++ b/apps/report/src/utils/screenshot-source.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'vitest'; +import { resolveScreenshotFallbackPath } from './screenshot-source'; + +describe('resolveScreenshotFallbackPath', () => { + it('keeps the legacy PNG fallback when only an id is available', () => { + expect(resolveScreenshotFallbackPath('legacy-shot')).toBe( + './screenshots/legacy-shot.png', + ); + }); + + it('uses the MIME-specific extension for screenshot references', () => { + expect( + resolveScreenshotFallbackPath({ + id: 'webp-shot', + mimeType: 'image/webp', + storage: 'inline', + }), + ).toBe('./screenshots/webp-shot.webp'); + expect( + resolveScreenshotFallbackPath({ + id: 'jpeg-shot', + mimeType: 'image/jpeg', + storage: 'inline', + }), + ).toBe('./screenshots/jpeg-shot.jpeg'); + }); + + it('prefers an explicit file-backed path', () => { + expect( + resolveScreenshotFallbackPath({ + id: 'webp-shot', + mimeType: 'image/webp', + storage: 'file', + path: './assets/custom.webp', + }), + ).toBe('./assets/custom.webp'); + }); +}); diff --git a/apps/report/src/utils/screenshot-source.ts b/apps/report/src/utils/screenshot-source.ts new file mode 100644 index 0000000000..7aeccfae87 --- /dev/null +++ b/apps/report/src/utils/screenshot-source.ts @@ -0,0 +1,31 @@ +import { + screenshotImageExtension, + screenshotImageFormatFromMimeType, +} from '@midscene/shared/img/image-format'; + +export interface ReportScreenshotSourceRef { + id: string; + mimeType?: unknown; + storage?: 'inline' | 'file'; + path?: string; +} + +export function resolveScreenshotFallbackPath( + refOrId: string | ReportScreenshotSourceRef, +): string { + if ( + typeof refOrId === 'object' && + refOrId.storage === 'file' && + refOrId.path + ) { + return refOrId.path; + } + + const id = typeof refOrId === 'string' ? refOrId : refOrId.id; + const format = + typeof refOrId === 'object' + ? screenshotImageFormatFromMimeType(refOrId.mimeType) + : undefined; + const extension = format ? screenshotImageExtension(format) : 'png'; + return `./screenshots/${id}.${extension}`; +} diff --git a/apps/site/docs/en/api.mdx b/apps/site/docs/en/api.mdx index cd0b0273c1..eb9185ae1a 100644 --- a/apps/site/docs/en/api.mdx +++ b/apps/site/docs/en/api.mdx @@ -1240,7 +1240,7 @@ interface RecordToReportOptions { screenshotBase64?: string; screenshots?: { /** - * PNG/JPEG data URI, or raw PNG base64 body. + * PNG/JPEG/WebP data URI, or raw PNG/WebP base64 body. */ base64: string; description?: string; @@ -1258,7 +1258,7 @@ function recordToReport( - `title?: string` - Optional, the title of the screenshot, if not provided, the title will be 'untitled'. - `options?: RecordToReportOptions` - Optional, a configuration object containing: - `content?: string` - The description of the screenshot. - - `screenshots?: Array<{ base64: string; description?: string }>` - Record one or more provided screenshots under one report entry. When this option is set, Midscene will not capture another screenshot automatically. `base64` should be a PNG/JPEG data URI such as `data:image/png;base64,...`; a raw base64 body is also accepted and treated as PNG. + - `screenshots?: Array<{ base64: string; description?: string }>` - Record one or more provided screenshots under one report entry. When this option is set, Midscene will not capture another screenshot automatically. `base64` should be a PNG/JPEG/WebP data URI such as `data:image/webp;base64,...`; raw WebP base64 is detected by its signature, while other raw base64 bodies retain the existing PNG default. - Compatibility: diff --git a/apps/site/docs/zh/api.mdx b/apps/site/docs/zh/api.mdx index 4a6e496ca9..769a2ddb91 100644 --- a/apps/site/docs/zh/api.mdx +++ b/apps/site/docs/zh/api.mdx @@ -1230,7 +1230,7 @@ interface RecordToReportOptions { screenshotBase64?: string; screenshots?: { /** - * PNG/JPEG data URI，或裸 PNG base64 body。 + * PNG/JPEG/WebP data URI，或裸 PNG/WebP base64 body。 */ base64: string; description?: string; @@ -1248,7 +1248,7 @@ function recordToReport( - `title?: string` - 可选，截图的标题，如果未提供，则标题为 'untitled'。 - `options?: RecordToReportOptions` - 可选，一个配置对象，包含： - `content?: string` - 截图的描述。 - - `screenshots?: Array<{ base64: string; description?: string }>` - 在同一个报告条目下记录一张或多张传入的截图。设置该选项后，Midscene 不会再自动截图。`base64` 推荐使用 PNG/JPEG data URI，例如 `data:image/png;base64,...`；也可以传裸 base64 body，此时会按 PNG 处理。 + - `screenshots?: Array<{ base64: string; description?: string }>` - 在同一个报告条目下记录一张或多张传入的截图。设置该选项后，Midscene 不会再自动截图。`base64` 推荐使用 PNG/JPEG/WebP data URI，例如 `data:image/webp;base64,...`；裸 WebP base64 会通过文件签名识别，其他裸 base64 body 保持现有的 PNG 默认行为。 - 兼容性： diff --git a/packages/core/src/report-cli.ts b/packages/core/src/report-cli.ts index 2e12dd5100..3c8b627aee 100644 --- a/packages/core/src/report-cli.ts +++ b/packages/core/src/report-cli.ts @@ -79,7 +79,7 @@ function writeAttachmentFromReport( const resolved = resolveScreenshotSource(attachment.sourceRef ?? null, { reportPath: opts.htmlPath, fallbackId: id, - fallbackMimeType: (mimeType || 'image/png') as 'image/png' | 'image/jpeg', + fallbackMimeType: mimeType || 'image/png', }); if (resolved.type === 'data-uri') { diff --git a/packages/core/src/report-markdown.ts b/packages/core/src/report-markdown.ts index 481558ef26..bab8478aad 100644 --- a/packages/core/src/report-markdown.ts +++ b/packages/core/src/report-markdown.ts @@ -9,12 +9,19 @@ import type { ModelBrief, ReportActionDump, } from '@/types'; +import { + type ScreenshotImageFormat, + type ScreenshotImageMimeType, + normalizeScreenshotBase64, + parseBase64, + screenshotImageExtension, + screenshotImageFormatFromMimeType, + screenshotImageMimeType, +} from '@midscene/shared/img'; import type { ScreenshotRef } from './dump/screenshot-store'; import { normalizeScreenshotRef } from './dump/screenshot-store'; -const screenshotDataUrlPattern = - /^data:image\/(png|jpeg|jpg);base64,([\s\S]*)$/i; -const rawBase64BodyPattern = /^[a-zA-Z0-9+/=\s]+$/; +const screenshotDataUrlPattern = /^data:image\/(?:png|jpe?g|webp);base64,/i; const jsonContextMaxStringLength = 12_000; type ExecutionTaskWithExtraUsage = ExecutionTask & { @@ -39,7 +46,7 @@ export interface MarkdownAttachment { * write the screenshot under this name to keep links in sync. See #2392. */ suggestedFileName: string; - mimeType?: string; + mimeType?: ScreenshotImageMimeType; /** * Reference to the screenshot in the source report, used to locate the * original bytes when copying them to the exported name. Absent for in-memory @@ -448,26 +455,11 @@ function extractLocateCenter( function tryExtractBase64(screenshot: unknown): string | undefined { if (typeof screenshot === 'string') { - const trimmedScreenshot = screenshot.trim(); - const dataUrlMatch = trimmedScreenshot.match(screenshotDataUrlPattern); - if (dataUrlMatch) { - const format = dataUrlMatch[1].toLowerCase() === 'jpg' ? 'jpeg' : 'png'; - const base64Body = dataUrlMatch[2].replace(/\s/g, ''); - if (!base64Body) { - return undefined; - } - return `data:image/${format};base64,${base64Body}`; - } - - if ( - trimmedScreenshot.startsWith('data:') || - !rawBase64BodyPattern.test(trimmedScreenshot) - ) { + try { + return normalizeScreenshotBase64(screenshot); + } catch { return undefined; } - - const base64Body = trimmedScreenshot.replace(/\s/g, ''); - return base64Body ? `data:image/png;base64,${base64Body}` : undefined; } if (!screenshot || typeof screenshot !== 'object') return undefined; @@ -478,6 +470,20 @@ function tryExtractBase64(screenshot: unknown): string | undefined { return undefined; } +function screenshotMetadataFromMimeType(mimeType: unknown): { + extension: ScreenshotImageFormat; + mimeType: ScreenshotImageMimeType; +} { + const format = screenshotImageFormatFromMimeType(mimeType); + if (!format) { + throw new Error(`Unsupported screenshot mime type: ${String(mimeType)}`); + } + return { + extension: screenshotImageExtension(format), + mimeType: screenshotImageMimeType(format), + }; +} + function restoredSourceRef(screenshot: unknown): ScreenshotRef | undefined { if (!screenshot || typeof screenshot !== 'object') { return undefined; @@ -507,7 +513,7 @@ function screenshotAttachment( attachment: { id: screenshot.id, suggestedFileName, - mimeType: `image/${ext === 'jpeg' ? 'jpeg' : 'png'}`, + mimeType: screenshot.mimeType, executionIndex, taskIndex, base64Data: tryExtractBase64(screenshot), @@ -517,7 +523,7 @@ function screenshotAttachment( const ref = normalizeScreenshotRef(screenshot); if (ref) { - const ext = ref.mimeType === 'image/jpeg' ? 'jpeg' : 'png'; + const { extension: ext } = screenshotMetadataFromMimeType(ref.mimeType); const suggestedFileName = `execution-${executionIndex + 1}-task-${taskIndex + 1}-${ref.id}.${ext}`; return { markdown: `\n![${markdownLabel}](${screenshotBaseDir}/${suggestedFileName})`, @@ -535,7 +541,9 @@ function screenshotAttachment( const sourceRef = restoredSourceRef(screenshot); if (sourceRef) { - const ext = sourceRef.mimeType === 'image/jpeg' ? 'jpeg' : 'png'; + const { extension: ext } = screenshotMetadataFromMimeType( + sourceRef.mimeType, + ); const suggestedFileName = `execution-${executionIndex + 1}-task-${taskIndex + 1}-${sourceRef.id}.${ext}`; return { markdown: `\n![${markdownLabel}](${screenshotBaseDir}/${suggestedFileName})`, @@ -553,7 +561,9 @@ function screenshotAttachment( const base64 = tryExtractBase64(screenshot); if (base64) { - const ext = base64.startsWith('data:image/jpeg') ? 'jpeg' : 'png'; + const { extension: ext, mimeType } = screenshotMetadataFromMimeType( + parseBase64(base64).mimeType, + ); const idSuffix = options?.fallbackIdSuffix ? `-${options.fallbackIdSuffix}` : ''; @@ -564,7 +574,7 @@ function screenshotAttachment( attachment: { id, suggestedFileName, - mimeType: `image/${ext}`, + mimeType, executionIndex, taskIndex, base64Data: base64, diff --git a/packages/core/src/report.ts b/packages/core/src/report.ts index 0d2ffd6935..8f79fafb1b 100644 --- a/packages/core/src/report.ts +++ b/packages/core/src/report.ts @@ -10,6 +10,11 @@ import { } from 'node:fs'; import * as path from 'node:path'; import { getMidsceneRunSubDir } from '@midscene/shared/common'; +import { + type ScreenshotImageFormat, + screenshotImageExtension, + screenshotImageFormatFromMimeType, +} from '@midscene/shared/img'; import { antiEscapeScriptTag, logMsg } from '@midscene/shared/utils'; import { getReportFileName } from './agent'; import { @@ -22,6 +27,7 @@ import { streamImageScriptsToFile, } from './dump/html-utils'; import { + type ScreenshotRef, normalizeScreenshotRef, resolveScreenshotSource, } from './dump/screenshot-store'; @@ -526,10 +532,14 @@ export function collectDedupedExecutions( }; } -function extensionByMimeType(mimeType: string): 'png' | 'jpeg' { - if (mimeType === 'image/png') return 'png'; - if (mimeType === 'image/jpeg') return 'jpeg'; - throw new Error(`Unsupported screenshot mime type: ${mimeType}`); +function extensionByMimeType( + mimeType: ScreenshotRef['mimeType'], +): ScreenshotImageFormat { + const format = screenshotImageFormatFromMimeType(mimeType); + if (!format) { + throw new Error(`Unsupported screenshot mime type: ${mimeType}`); + } + return screenshotImageExtension(format); } function externalizeScreenshotsInExecution( diff --git a/packages/core/tests/unit-test/report-cli.test.ts b/packages/core/tests/unit-test/report-cli.test.ts index f80b55149b..2497db54cb 100644 --- a/packages/core/tests/unit-test/report-cli.test.ts +++ b/packages/core/tests/unit-test/report-cli.test.ts @@ -24,6 +24,9 @@ function fakeBase64(sizeBytes: number): string { return `data:image/png;base64,${'A'.repeat(sizeBytes)}`; } +const webpBase64 = + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; + function createExecution( id: string, screenshot: ScreenshotItem | ScreenshotRef, @@ -223,6 +226,45 @@ describe('createReportCliCommands', () => { expect(existsSync(join(outputDir, 'report.md'))).toBe(true); }); + it('exports WebP screenshots and markdown links with the .webp suffix', async () => { + const reportPath = join(tmpDir, 'input-report-sdk-webp', 'index.html'); + mkdirSync(join(tmpDir, 'input-report-sdk-webp'), { recursive: true }); + const screenshot = ScreenshotItem.create(webpBase64, Date.now()); + const dump = new ReportActionDump({ + groupName: 'sdk-webp-test', + sdkVersion: '1.0.0-test', + modelBriefs: [], + executions: [createExecution('exec-sdk-webp', screenshot)], + }); + writeFileSync( + reportPath, + [ + generateImageScriptTag(screenshot.id, screenshot.base64), + generateDumpScriptTag(dump.serialize(), { + 'data-group-id': 'webp-group', + }), + ].join('\n'), + 'utf-8', + ); + + const outputDir = join(tmpDir, 'output-sdk-webp'); + const result = await reportFileToMarkdown({ + htmlPath: reportPath, + outputDir, + }); + const expectedFileName = `execution-1-task-1-${screenshot.id}.webp`; + + expect(result.screenshotFiles).toEqual([ + join(outputDir, 'screenshots', expectedFileName), + ]); + expect(readFileSync(result.screenshotFiles[0]).toString('base64')).toBe( + webpBase64.split(',')[1], + ); + expect(readFileSync(join(outputDir, 'report.md'), 'utf-8')).toContain( + `./screenshots/${expectedFileName}`, + ); + }); + it('throws SDK-friendly validation errors for reportFileToMarkdown', async () => { await expect( reportFileToMarkdown({ diff --git a/packages/core/tests/unit-test/report-markdown.test.ts b/packages/core/tests/unit-test/report-markdown.test.ts index aa23792285..df9cc13d4f 100644 --- a/packages/core/tests/unit-test/report-markdown.test.ts +++ b/packages/core/tests/unit-test/report-markdown.test.ts @@ -28,6 +28,9 @@ function createTask(overrides: Record = {}) { } describe('report-markdown', () => { + const webpBase64 = + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; + it('handles single execution markdown with screenshot file links', () => { const screenshot = ScreenshotItem.create( 'data:image/png;base64,Zm9v', @@ -85,6 +88,34 @@ describe('report-markdown', () => { ); }); + it('exports WebP screenshots with coherent MIME and file metadata', () => { + const screenshot = ScreenshotItem.create(webpBase64, 1710000000000); + const execution: IExecutionDump = { + logTime: 1710000000000, + name: 'webp execution', + tasks: [ + createTask({ + uiContext: { + screenshot, + shotSize: { width: 2, height: 3 }, + }, + }), + ], + }; + + const result = executionToMarkdown(execution); + + expect(result.attachments).toHaveLength(1); + expect(result.attachments[0]).toMatchObject({ + suggestedFileName: expect.stringMatching(/\.webp$/), + mimeType: 'image/webp', + base64Data: webpBase64, + }); + expect(result.markdown).toContain( + `./screenshots/${result.attachments[0].suggestedFileName}`, + ); + }); + it('merges all executions into one markdown and keeps file snapshot', async () => { const report: IReportActionDump = { sdkVersion: '1.0.0', diff --git a/packages/core/tests/unit-test/report-split.test.ts b/packages/core/tests/unit-test/report-split.test.ts index 9c9f869c41..6c56364f6a 100644 --- a/packages/core/tests/unit-test/report-split.test.ts +++ b/packages/core/tests/unit-test/report-split.test.ts @@ -18,6 +18,9 @@ function fakeBase64(sizeBytes: number): string { return `data:image/png;base64,${'A'.repeat(sizeBytes)}`; } +const webpBase64 = + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; + function createExecution( id: string, screenshot: ScreenshotItem | ScreenshotRef, @@ -129,6 +132,48 @@ describe('splitReportHtmlByExecution', () => { } }); + it('externalizes inline WebP screenshots as .webp files', () => { + const reportPath = join(tmpDir, 'webp-report', 'index.html'); + mkdirSync(join(tmpDir, 'webp-report'), { recursive: true }); + const screenshot = ScreenshotItem.create(webpBase64, Date.now()); + const dump = new ReportActionDump({ + groupName: 'webp-split-test', + sdkVersion: '1.0.0-test', + modelBriefs: [], + executions: [createExecution('webp-exec', screenshot)], + }); + writeFileSync( + reportPath, + [ + generateImageScriptTag(screenshot.id, screenshot.base64), + generateDumpScriptTag(dump.serialize(), { + 'data-group-id': 'webp-group', + }), + ].join('\n'), + 'utf-8', + ); + + const result = splitReportHtmlByExecution({ + htmlPath: reportPath, + outputDir: join(tmpDir, 'webp-output'), + }); + const outputDump = JSON.parse( + readFileSync(result.executionJsonFiles[0], 'utf-8'), + ); + const outputRef = outputDump.executions[0].tasks[0].uiContext.screenshot; + + expect(outputRef).toMatchObject({ + mimeType: 'image/webp', + path: `./screenshots/${screenshot.id}.webp`, + }); + expect(result.screenshotFiles).toEqual([ + join(tmpDir, 'webp-output', 'screenshots', `${screenshot.id}.webp`), + ]); + expect(readFileSync(result.screenshotFiles[0]).toString('base64')).toBe( + webpBase64.split(',')[1], + ); + }); + it('should process large report incrementally without accumulating all dump scripts', () => { const reportPath = join(tmpDir, 'large-report', 'index.html'); mkdirSync(join(tmpDir, 'large-report'), { recursive: true }); diff --git a/packages/shared/src/agent-tools/types.ts b/packages/shared/src/agent-tools/types.ts index de007edaa8..78cf044a90 100644 --- a/packages/shared/src/agent-tools/types.ts +++ b/packages/shared/src/agent-tools/types.ts @@ -98,7 +98,7 @@ export type UserPromptLike = export interface RecordToReportScreenshot { /** - * PNG/JPEG data URI, or raw PNG base64 body. + * PNG/JPEG/WebP data URI, or raw PNG/WebP base64 body. */ base64: string; description?: string; diff --git a/packages/shared/src/cli/screenshot-file.ts b/packages/shared/src/cli/screenshot-file.ts index 4ecb0d38eb..db843c3041 100644 --- a/packages/shared/src/cli/screenshot-file.ts +++ b/packages/shared/src/cli/screenshot-file.ts @@ -1,6 +1,11 @@ import { existsSync, mkdirSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { + type ScreenshotImageFormat, + screenshotImageFormatFromExtension, + screenshotImageFormatFromMimeType, +} from '../img/image-format'; export interface WriteCliScreenshotFileOptions { id?: unknown; @@ -20,14 +25,12 @@ function safeScreenshotFilenamePart(value: unknown): string { function extensionFromImageMetadata( mimeType: unknown, extension: unknown, -): 'png' | 'jpeg' { - if (extension === 'jpeg' || extension === 'jpg') { - return 'jpeg'; +): ScreenshotImageFormat { + const extensionFormat = screenshotImageFormatFromExtension(extension); + if (extensionFormat) { + return extensionFormat; } - if (extension === 'png') { - return 'png'; - } - return mimeType === 'image/jpeg' ? 'jpeg' : 'png'; + return screenshotImageFormatFromMimeType(mimeType) ?? 'png'; } export function writeCliScreenshotFile( diff --git a/packages/shared/src/cli/verbose-screenshot.ts b/packages/shared/src/cli/verbose-screenshot.ts index b27d62234d..f77bfc621a 100644 --- a/packages/shared/src/cli/verbose-screenshot.ts +++ b/packages/shared/src/cli/verbose-screenshot.ts @@ -71,7 +71,9 @@ function screenshotRawBase64(value: unknown): string | undefined { } const base64 = getStringProperty(value, 'base64'); - const match = base64?.match(/^data:image\/(?:png|jpeg|jpg);base64,(.+)$/); + const match = base64?.match( + /^data:image\/(?:png|jpeg|jpg|webp);base64,(.+)$/, + ); return match?.[1]; } diff --git a/packages/shared/tests/unit-test/screenshot-file.test.ts b/packages/shared/tests/unit-test/screenshot-file.test.ts new file mode 100644 index 0000000000..cd81bd43f4 --- /dev/null +++ b/packages/shared/tests/unit-test/screenshot-file.test.ts @@ -0,0 +1,62 @@ +import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { writeCliScreenshotFile } from '../../src/cli/screenshot-file'; +import { collectScreenshotRefs } from '../../src/cli/verbose-screenshot'; + +const webpBase64 = + 'UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; + +describe('CLI WebP screenshot files', () => { + const temporaryDirectories: string[] = []; + + const makeTemporaryDirectory = () => { + const directory = mkdtempSync(join(tmpdir(), 'midscene-webp-')); + temporaryDirectories.push(directory); + return directory; + }; + + afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it('uses the .webp extension from screenshot MIME metadata', () => { + const directoryPath = makeTemporaryDirectory(); + const filePath = writeCliScreenshotFile(webpBase64, { + id: 'webp-shot', + mimeType: 'image/webp', + directoryPath, + }); + + expect(filePath).toBe(join(directoryPath, 'webp-shot.webp')); + expect(readFileSync(filePath).toString('base64')).toBe(webpBase64); + }); + + it('exports inline WebP screenshots for verbose output', () => { + const directoryPath = makeTemporaryDirectory(); + const screenshot = { + base64: `data:image/webp;base64,${webpBase64}`, + extension: 'webp', + toSerializable: () => ({ + type: 'midscene_screenshot_ref' as const, + id: 'inline-webp', + capturedAt: 1, + mimeType: 'image/webp', + storage: 'inline', + }), + }; + + const [collected] = collectScreenshotRefs(screenshot, { + exportMode: 'report', + reportFile: join(directoryPath, 'report.html'), + }); + + expect(collected.file).toBe('inline-webp.webp'); + expect( + existsSync(join(directoryPath, 'screenshots', 'inline-webp.webp')), + ).toBe(true); + }); +}); diff --git a/packages/visualizer/src/hooks/usePlaygroundExecution.ts b/packages/visualizer/src/hooks/usePlaygroundExecution.ts index 965b7f316d..a0b475ad8a 100644 --- a/packages/visualizer/src/hooks/usePlaygroundExecution.ts +++ b/packages/visualizer/src/hooks/usePlaygroundExecution.ts @@ -10,6 +10,10 @@ import { parseImageScripts, restoreImageReferences, } from '@midscene/core/dump'; +import { + screenshotImageExtension, + screenshotImageFormatFromMimeType, +} from '@midscene/shared/img/image-format'; import { useCallback } from 'react'; import { useEnvConfig } from '../store/store'; import type { @@ -154,7 +158,11 @@ async function loadReportReplay( } const dump = (await response.json()) as IReportActionDump; result.dump = restoreImageReferences(dump, (ref) => { - const extension = ref.mimeType === 'image/jpeg' ? 'jpeg' : 'png'; + const format = screenshotImageFormatFromMimeType(ref.mimeType); + if (!format) { + throw new Error(`Unsupported screenshot mime type: ${ref.mimeType}`); + } + const extension = screenshotImageExtension(format); return new URL( `screenshots/${encodeURIComponent(ref.id)}.${extension}`, result.report!.url, diff --git a/packages/visualizer/tests/playground-execution-stop.test.ts b/packages/visualizer/tests/playground-execution-stop.test.ts index 9047546cfe..f9b401ba95 100644 --- a/packages/visualizer/tests/playground-execution-stop.test.ts +++ b/packages/visualizer/tests/playground-execution-stop.test.ts @@ -95,7 +95,7 @@ function Harness({ return null; } -function replayDump() { +function replayDump(mimeType: 'image/png' | 'image/webp' = 'image/png') { return { sdkVersion: 'test', groupName: 'Playground run', @@ -113,7 +113,7 @@ function replayDump() { type: 'midscene_screenshot_ref', id: 'shot-1', capturedAt: 1, - mimeType: 'image/png', + mimeType, storage: 'inline', }, }, @@ -276,7 +276,7 @@ describe('usePlaygroundExecution stop handling', () => { const fetchMock = vi.fn(async () => Promise.resolve({ ok: true, - json: async () => replayDump(), + json: async () => replayDump('image/webp'), }), ); vi.stubGlobal('fetch', fetchMock); @@ -331,7 +331,7 @@ describe('usePlaygroundExecution stop handling', () => { const restoredDump = allScriptsFromDumpMock.mock.calls[0]?.[0] as any; expect( restoredDump.executions[0].tasks[0].uiContext.screenshot.base64, - ).toBe('http://localhost/reports/report-1/screenshots/shot-1.png'); + ).toBe('http://localhost/reports/report-1/screenshots/shot-1.webp'); await act(async () => root.unmount()); });