feat(report): support WebP screenshot consumers

Author: quanruzhuoxiu

apps/report/src/App.tsx

@@ -54,12 +54,16 @@ import {
   getEmptyDumpDescription,
   parseDumpAttributes,
 } from './utils/report-dump';
+import {
+  type ReportScreenshotSourceRef,
+  resolveScreenshotFallbackPath,
+} from './utils/screenshot-source';
 
 // Shared image cache across all test cases — resolved images are cached by id
 const imageCache = new Map<string, string>();
 
 function resolveImageFromDom(
-  refOrId: string | { id: string; storage?: 'inline' | 'file'; path?: string },
+  refOrId: string | ReportScreenshotSourceRef,
 ): string {
   const id = typeof refOrId === 'string' ? refOrId : refOrId.id;
   const cached = imageCache.get(id);
@@ -74,12 +78,7 @@ function resolveImageFromDom(
     return data;
   }
 
-  if (typeof refOrId === 'object' && refOrId?.storage === 'file') {
-    return refOrId.path || `./screenshots/${id}.png`;
-  }
-
-  // Fallback to directory path
-  return `./screenshots/${id}.png`;
+  return resolveScreenshotFallbackPath(refOrId);
 }
 
 let globalRenderCount = 1;

apps/report/src/components/playground/index.tsx

@@ -14,6 +14,10 @@ import {
 } from '@midscene/core/dump';
 import { type PlaygroundSDK, noReplayAPIs } from '@midscene/playground';
 import type { ServerResponse } from '@midscene/playground';
+import {
+  screenshotImageExtension,
+  screenshotImageFormatFromMimeType,
+} from '@midscene/shared/img/image-format';
 import {
   ContextPreview,
   Logo,
@@ -55,7 +59,11 @@ async function loadReferencedReplay(result: PlaygroundResult) {
   }
   const dump = (await response.json()) as IReportActionDump;
   result.dump = restoreImageReferences(dump, (ref) => {
-    const extension = ref.mimeType === 'image/jpeg' ? 'jpeg' : 'png';
+    const format = screenshotImageFormatFromMimeType(ref.mimeType);
+    if (!format) {
+      throw new Error(`Unsupported screenshot mime type: ${ref.mimeType}`);
+    }
+    const extension = screenshotImageExtension(format);
     return new URL(
       `screenshots/${encodeURIComponent(ref.id)}.${extension}`,
       result.report!.url,

apps/report/src/components/timeline/build-timeline-screenshots.test.ts

@@ -4,6 +4,8 @@ import { buildTimelineScreenshots } from './build-timeline-screenshots';
 
 const onePixelPngBase64 =
   'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAFgwJ/lz8yrwAAAABJRU5ErkJggg==';
+const webpBase64 =
+  'UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA';
 
 interface TaskFixtureOptions {
   id: string;
@@ -103,6 +105,19 @@ describe('buildTimelineScreenshots', () => {
     );
   });
 
+  it('labels raw WebP recorder screenshots with the WebP MIME type', () => {
+    const task = makeTask({
+      id: 'raw-webp-recorder',
+      startTs: 1000,
+      recorder: [{ ts: 1200, screenshot: webpBase64 }],
+    });
+
+    const { allScreenshots } = buildTimelineScreenshots([task]);
+
+    expect(allScreenshots).toHaveLength(1);
+    expect(allScreenshots[0].img).toBe(`data:image/webp;base64,${webpBase64}`);
+  });
+
   it('keeps already-prefixed data URLs unchanged', () => {
     const dataUrl = `data:image/png;base64,${onePixelPngBase64}`;
     const task = makeTask({

apps/report/src/components/timeline/build-timeline-screenshots.ts

@@ -1,4 +1,8 @@
 import type { ExecutionTask } from '@midscene/core';
+import {
+  inferScreenshotImageFormatFromBase64,
+  screenshotImageMimeType,
+} from '@midscene/shared/img/image-format';
 
 export interface TimelineScreenshot {
   id: string;
@@ -30,11 +34,8 @@ const imageSrcFromString = (value: string): string => {
   }
 
   const body = trimmed.replace(/\s/g, '');
-  const mimeType = body.startsWith('/9j/')
-    ? 'image/jpeg'
-    : body.startsWith('UklGR')
-      ? 'image/webp'
-      : 'image/png';
+  const format = inferScreenshotImageFormatFromBase64(body) ?? 'png';
+  const mimeType = screenshotImageMimeType(format);
   return `data:${mimeType};base64,${body}`;
 };
 

apps/report/src/utils/markdown-export.test.ts

@@ -105,6 +105,27 @@ describe('markdown-export helpers', () => {
     );
   });
 
+  it('packages WebP data URI attachments without changing their bytes', () => {
+    const files = buildMarkdownArchiveFiles('# webp report', [
+      {
+        id: 'webp-inline',
+        suggestedFileName: 'webp-inline.webp',
+        mimeType: 'image/webp',
+        executionIndex: 0,
+        taskIndex: 0,
+        base64Data: `data:image/webp;base64,${btoa('webp-bytes')}`,
+      },
+    ]);
+
+    expect(Object.keys(files).sort()).toEqual([
+      'report.md',
+      'screenshots/webp-inline.webp',
+    ]);
+    expect(
+      new TextDecoder().decode(files['screenshots/webp-inline.webp']),
+    ).toBe('webp-bytes');
+  });
+
   it('builds display items from markdown attachment names and paths', () => {
     const items = getMarkdownAttachmentDisplayItems([
       {

apps/report/src/utils/markdown-export.ts

@@ -1,7 +1,7 @@
 import type { MarkdownAttachment } from '@midscene/core';
 
 const defaultScreenshotBaseDir = './screenshots';
-const dataUrlBase64Pattern = /^data:image\/(?:png|jpeg|jpg);base64,/i;
+const dataUrlBase64Pattern = /^data:image\/(?:png|jpeg|jpg|webp);base64,/i;
 const rawBase64Pattern = /^[a-zA-Z0-9+/=\s]+$/;
 
 type MarkdownExport = {

apps/report/src/utils/screenshot-source.test.ts

@@ -0,0 +1,38 @@
+import { describe, expect, it } from 'vitest';
+import { resolveScreenshotFallbackPath } from './screenshot-source';
+
+describe('resolveScreenshotFallbackPath', () => {
+  it('keeps the legacy PNG fallback when only an id is available', () => {
+    expect(resolveScreenshotFallbackPath('legacy-shot')).toBe(
+      './screenshots/legacy-shot.png',
+    );
+  });
+
+  it('uses the MIME-specific extension for screenshot references', () => {
+    expect(
+      resolveScreenshotFallbackPath({
+        id: 'webp-shot',
+        mimeType: 'image/webp',
+        storage: 'inline',
+      }),
+    ).toBe('./screenshots/webp-shot.webp');
+    expect(
+      resolveScreenshotFallbackPath({
+        id: 'jpeg-shot',
+        mimeType: 'image/jpeg',
+        storage: 'inline',
+      }),
+    ).toBe('./screenshots/jpeg-shot.jpeg');
+  });
+
+  it('prefers an explicit file-backed path', () => {
+    expect(
+      resolveScreenshotFallbackPath({
+        id: 'webp-shot',
+        mimeType: 'image/webp',
+        storage: 'file',
+        path: './assets/custom.webp',
+      }),
+    ).toBe('./assets/custom.webp');
+  });
+});

apps/report/src/utils/screenshot-source.ts

@@ -0,0 +1,31 @@
+import {
+  screenshotImageExtension,
+  screenshotImageFormatFromMimeType,
+} from '@midscene/shared/img/image-format';
+
+export interface ReportScreenshotSourceRef {
+  id: string;
+  mimeType?: unknown;
+  storage?: 'inline' | 'file';
+  path?: string;
+}
+
+export function resolveScreenshotFallbackPath(
+  refOrId: string | ReportScreenshotSourceRef,
+): string {
+  if (
+    typeof refOrId === 'object' &&
+    refOrId.storage === 'file' &&
+    refOrId.path
+  ) {
+    return refOrId.path;
+  }
+
+  const id = typeof refOrId === 'string' ? refOrId : refOrId.id;
+  const format =
+    typeof refOrId === 'object'
+      ? screenshotImageFormatFromMimeType(refOrId.mimeType)
+      : undefined;
+  const extension = format ? screenshotImageExtension(format) : 'png';
+  return `./screenshots/${id}.${extension}`;
+}

apps/site/docs/en/api.mdx

@@ -1240,7 +1240,7 @@ interface RecordToReportOptions {
   screenshotBase64?: string;
   screenshots?: {
     /**
-     * PNG/JPEG data URI, or raw PNG base64 body.
+     * PNG/JPEG/WebP data URI, or raw PNG/WebP base64 body.
      */
     base64: string;
     description?: string;
@@ -1258,7 +1258,7 @@ function recordToReport(
   - `title?: string` - Optional, the title of the screenshot, if not provided, the title will be 'untitled'.
   - `options?: RecordToReportOptions` - Optional, a configuration object containing:
     - `content?: string` - The description of the screenshot.
-    - `screenshots?: Array<{ base64: string; description?: string }>` - Record one or more provided screenshots under one report entry. When this option is set, Midscene will not capture another screenshot automatically. `base64` should be a PNG/JPEG data URI such as `data:image/png;base64,...`; a raw base64 body is also accepted and treated as PNG.
+    - `screenshots?: Array<{ base64: string; description?: string }>` - Record one or more provided screenshots under one report entry. When this option is set, Midscene will not capture another screenshot automatically. `base64` should be a PNG/JPEG/WebP data URI such as `data:image/webp;base64,...`; raw WebP base64 is detected by its signature, while other raw base64 bodies retain the existing PNG default.
 
 - Compatibility:
 

apps/site/docs/zh/api.mdx

@@ -1230,7 +1230,7 @@ interface RecordToReportOptions {
   screenshotBase64?: string;
   screenshots?: {
     /**
-     * PNG/JPEG data URI，或裸 PNG base64 body。
+     * PNG/JPEG/WebP data URI，或裸 PNG/WebP base64 body。
      */
     base64: string;
     description?: string;
@@ -1248,7 +1248,7 @@ function recordToReport(
   - `title?: string` - 可选，截图的标题，如果未提供，则标题为 'untitled'。
   - `options?: RecordToReportOptions` - 可选，一个配置对象，包含：
     - `content?: string` - 截图的描述。
-    - `screenshots?: Array<{ base64: string; description?: string }>` - 在同一个报告条目下记录一张或多张传入的截图。设置该选项后，Midscene 不会再自动截图。`base64` 推荐使用 PNG/JPEG data URI，例如 `data:image/png;base64,...`；也可以传裸 base64 body，此时会按 PNG 处理。
+    - `screenshots?: Array<{ base64: string; description?: string }>` - 在同一个报告条目下记录一张或多张传入的截图。设置该选项后，Midscene 不会再自动截图。`base64` 推荐使用 PNG/JPEG/WebP data URI，例如 `data:image/webp;base64,...`；裸 WebP base64 会通过文件签名识别，其他裸 base64 body 保持现有的 PNG 默认行为。
 
 - 兼容性：