feat(report): support WebP screenshot consumers

apps/report/src/App.tsx

@@ -54,12 +54,16 @@ import {
   getEmptyDumpDescription,
   parseDumpAttributes,
 } from './utils/report-dump';
+import {
+  type ReportScreenshotSourceRef,
+  resolveScreenshotFallbackPath,
+} from './utils/screenshot-source';
 
 // Shared image cache across all test cases — resolved images are cached by id
 const imageCache = new Map<string, string>();
 
 function resolveImageFromDom(
-  refOrId: string | { id: string; storage?: 'inline' | 'file'; path?: string },
+  refOrId: string | ReportScreenshotSourceRef,
 ): string {
   const id = typeof refOrId === 'string' ? refOrId : refOrId.id;
   const cached = imageCache.get(id);
@@ -74,12 +78,7 @@ function resolveImageFromDom(
     return data;
   }
 
-  if (typeof refOrId === 'object' && refOrId?.storage === 'file') {
-    return refOrId.path || `./screenshots/${id}.png`;
-  }
-
-  // Fallback to directory path
-  return `./screenshots/${id}.png`;
+  return resolveScreenshotFallbackPath(refOrId);
 }
 
 let globalRenderCount = 1;

apps/report/src/components/playground/index.tsx

@@ -14,6 +14,10 @@ import {
 } from '@midscene/core/dump';
 import { type PlaygroundSDK, noReplayAPIs } from '@midscene/playground';
 import type { ServerResponse } from '@midscene/playground';
+import {
+  screenshotImageExtension,
+  screenshotImageFormatFromMimeType,
+} from '@midscene/shared/img/image-format';
 import {
   ContextPreview,
   Logo,
@@ -55,7 +59,11 @@ async function loadReferencedReplay(result: PlaygroundResult) {
   }
   const dump = (await response.json()) as IReportActionDump;
   result.dump = restoreImageReferences(dump, (ref) => {
-    const extension = ref.mimeType === 'image/jpeg' ? 'jpeg' : 'png';
+    const format = screenshotImageFormatFromMimeType(ref.mimeType);
+    if (!format) {
+      throw new Error(`Unsupported screenshot mime type: ${ref.mimeType}`);
+    }
+    const extension = screenshotImageExtension(format);
     return new URL(
       `screenshots/${encodeURIComponent(ref.id)}.${extension}`,
       result.report!.url,

apps/report/src/components/timeline/build-timeline-screenshots.test.ts

@@ -4,6 +4,8 @@ import { buildTimelineScreenshots } from './build-timeline-screenshots';
 
 const onePixelPngBase64 =
   'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAFgwJ/lz8yrwAAAABJRU5ErkJggg==';
+const webpBase64 =
+  'UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA';
 
 interface TaskFixtureOptions {
   id: string;
@@ -103,6 +105,19 @@ describe('buildTimelineScreenshots', () => {
     );
   });
 
+  it('labels raw WebP recorder screenshots with the WebP MIME type', () => {
+    const task = makeTask({
+      id: 'raw-webp-recorder',
+      startTs: 1000,
+      recorder: [{ ts: 1200, screenshot: webpBase64 }],
+    });
+
+    const { allScreenshots } = buildTimelineScreenshots([task]);
+
+    expect(allScreenshots).toHaveLength(1);
+    expect(allScreenshots[0].img).toBe(`data:image/webp;base64,${webpBase64}`);
+  });
+
   it('keeps already-prefixed data URLs unchanged', () => {
     const dataUrl = `data:image/png;base64,${onePixelPngBase64}`;
     const task = makeTask({

apps/report/src/components/timeline/build-timeline-screenshots.ts

@@ -1,4 +1,8 @@
 import type { ExecutionTask } from '@midscene/core';
+import {
+  inferScreenshotImageFormatFromBase64,
+  screenshotImageMimeType,
+} from '@midscene/shared/img/image-format';
 
 export interface TimelineScreenshot {
   id: string;
@@ -30,11 +34,8 @@ const imageSrcFromString = (value: string): string => {
   }
 
   const body = trimmed.replace(/\s/g, '');
-  const mimeType = body.startsWith('/9j/')
-    ? 'image/jpeg'
-    : body.startsWith('UklGR')
-      ? 'image/webp'
-      : 'image/png';
+  const format = inferScreenshotImageFormatFromBase64(body) ?? 'png';
+  const mimeType = screenshotImageMimeType(format);
   return `data:${mimeType};base64,${body}`;
 };
 

apps/report/src/utils/markdown-export.test.ts

@@ -105,6 +105,27 @@ describe('markdown-export helpers', () => {
     );
   });
 
+  it('packages WebP data URI attachments without changing their bytes', () => {
+    const files = buildMarkdownArchiveFiles('# webp report', [
+      {
+        id: 'webp-inline',
+        suggestedFileName: 'webp-inline.webp',
+        mimeType: 'image/webp',
+        executionIndex: 0,
+        taskIndex: 0,
+        base64Data: `data:image/webp;base64,${btoa('webp-bytes')}`,
+      },
+    ]);
+
+    expect(Object.keys(files).sort()).toEqual([
+      'report.md',
+      'screenshots/webp-inline.webp',
+    ]);
+    expect(
+      new TextDecoder().decode(files['screenshots/webp-inline.webp']),
+    ).toBe('webp-bytes');
+  });
+
   it('builds display items from markdown attachment names and paths', () => {
     const items = getMarkdownAttachmentDisplayItems([
       {

apps/report/src/utils/markdown-export.ts

@@ -1,7 +1,7 @@
 import type { MarkdownAttachment } from '@midscene/core';
 
 const defaultScreenshotBaseDir = './screenshots';
-const dataUrlBase64Pattern = /^data:image\/(?:png|jpeg|jpg);base64,/i;
+const dataUrlBase64Pattern = /^data:image\/(?:png|jpeg|jpg|webp);base64,/i;
 const rawBase64Pattern = /^[a-zA-Z0-9+/=\s]+$/;
 
 type MarkdownExport = {

apps/report/src/utils/screenshot-source.test.ts

@@ -0,0 +1,38 @@
+import { describe, expect, it } from 'vitest';
+import { resolveScreenshotFallbackPath } from './screenshot-source';
+
+describe('resolveScreenshotFallbackPath', () => {
+  it('keeps the legacy PNG fallback when only an id is available', () => {
+    expect(resolveScreenshotFallbackPath('legacy-shot')).toBe(
+      './screenshots/legacy-shot.png',
+    );
+  });
+
+  it('uses the MIME-specific extension for screenshot references', () => {
+    expect(
+      resolveScreenshotFallbackPath({
+        id: 'webp-shot',
+        mimeType: 'image/webp',
+        storage: 'inline',
+      }),
+    ).toBe('./screenshots/webp-shot.webp');
+    expect(
+      resolveScreenshotFallbackPath({
+        id: 'jpeg-shot',
+        mimeType: 'image/jpeg',
+        storage: 'inline',
+      }),
+    ).toBe('./screenshots/jpeg-shot.jpeg');
+  });
+
+  it('prefers an explicit file-backed path', () => {
+    expect(
+      resolveScreenshotFallbackPath({
+        id: 'webp-shot',
+        mimeType: 'image/webp',
+        storage: 'file',
+        path: './assets/custom.webp',
+      }),
+    ).toBe('./assets/custom.webp');
+  });
+});

apps/report/src/utils/screenshot-source.ts

@@ -0,0 +1,31 @@
+import {
+  screenshotImageExtension,
+  screenshotImageFormatFromMimeType,
+} from '@midscene/shared/img/image-format';
+
+export interface ReportScreenshotSourceRef {
+  id: string;
+  mimeType?: unknown;
+  storage?: 'inline' | 'file';
+  path?: string;
+}
+
+export function resolveScreenshotFallbackPath(
+  refOrId: string | ReportScreenshotSourceRef,
+): string {
+  if (
+    typeof refOrId === 'object' &&
+    refOrId.storage === 'file' &&
+    refOrId.path
+  ) {
+    return refOrId.path;
+  }
+
+  const id = typeof refOrId === 'string' ? refOrId : refOrId.id;
+  const format =
+    typeof refOrId === 'object'
+      ? screenshotImageFormatFromMimeType(refOrId.mimeType)
+      : undefined;
+  const extension = format ? screenshotImageExtension(format) : 'png';
+  return `./screenshots/${id}.${extension}`;
+}

apps/site/docs/en/api.mdx

@@ -1240,7 +1240,7 @@ interface RecordToReportOptions {
   screenshotBase64?: string;
   screenshots?: {
     /**
-     * PNG/JPEG data URI, or raw PNG base64 body.
+     * PNG/JPEG/WebP data URI, or raw PNG/WebP base64 body.
      */
     base64: string;
     description?: string;
@@ -1258,7 +1258,7 @@ function recordToReport(
   - `title?: string` - Optional, the title of the screenshot, if not provided, the title will be 'untitled'.
   - `options?: RecordToReportOptions` - Optional, a configuration object containing:
     - `content?: string` - The description of the screenshot.
-    - `screenshots?: Array<{ base64: string; description?: string }>` - Record one or more provided screenshots under one report entry. When this option is set, Midscene will not capture another screenshot automatically. `base64` should be a PNG/JPEG data URI such as `data:image/png;base64,...`; a raw base64 body is also accepted and treated as PNG.
+    - `screenshots?: Array<{ base64: string; description?: string }>` - Record one or more provided screenshots under one report entry. When this option is set, Midscene will not capture another screenshot automatically. `base64` should be a PNG/JPEG/WebP data URI such as `data:image/webp;base64,...`; raw WebP base64 is detected by its signature, while other raw base64 bodies retain the existing PNG default.
 
 - Compatibility:
 

apps/site/docs/zh/api.mdx

@@ -1230,7 +1230,7 @@ interface RecordToReportOptions {
   screenshotBase64?: string;
   screenshots?: {
     /**
-     * PNG/JPEG data URI，或裸 PNG base64 body。
+     * PNG/JPEG/WebP data URI，或裸 PNG/WebP base64 body。
      */
     base64: string;
     description?: string;
@@ -1248,7 +1248,7 @@ function recordToReport(
   - `title?: string` - 可选，截图的标题，如果未提供，则标题为 'untitled'。
   - `options?: RecordToReportOptions` - 可选，一个配置对象，包含：
     - `content?: string` - 截图的描述。
-    - `screenshots?: Array<{ base64: string; description?: string }>` - 在同一个报告条目下记录一张或多张传入的截图。设置该选项后，Midscene 不会再自动截图。`base64` 推荐使用 PNG/JPEG data URI，例如 `data:image/png;base64,...`；也可以传裸 base64 body，此时会按 PNG 处理。
+    - `screenshots?: Array<{ base64: string; description?: string }>` - 在同一个报告条目下记录一张或多张传入的截图。设置该选项后，Midscene 不会再自动截图。`base64` 推荐使用 PNG/JPEG/WebP data URI，例如 `data:image/webp;base64,...`；裸 WebP base64 会通过文件签名识别，其他裸 base64 body 保持现有的 PNG 默认行为。
 
 - 兼容性：
 

packages/core/src/report-cli.ts

@@ -79,7 +79,7 @@ function writeAttachmentFromReport(
   const resolved = resolveScreenshotSource(attachment.sourceRef ?? null, {
     reportPath: opts.htmlPath,
     fallbackId: id,
-    fallbackMimeType: (mimeType || 'image/png') as 'image/png' | 'image/jpeg',
+    fallbackMimeType: mimeType || 'image/png',
   });
 
   if (resolved.type === 'data-uri') {

packages/core/src/report-markdown.ts

@@ -9,12 +9,19 @@ import type {
   ModelBrief,
   ReportActionDump,
 } from '@/types';
+import {
+  type ScreenshotImageFormat,
+  type ScreenshotImageMimeType,
+  normalizeScreenshotBase64,
+  parseBase64,
+  screenshotImageExtension,
+  screenshotImageFormatFromMimeType,
+  screenshotImageMimeType,
+} from '@midscene/shared/img';
 import type { ScreenshotRef } from './dump/screenshot-store';
 import { normalizeScreenshotRef } from './dump/screenshot-store';
 
-const screenshotDataUrlPattern =
-  /^data:image\/(png|jpeg|jpg);base64,([\s\S]*)$/i;
-const rawBase64BodyPattern = /^[a-zA-Z0-9+/=\s]+$/;
+const screenshotDataUrlPattern = /^data:image\/(?:png|jpe?g|webp);base64,/i;
 const jsonContextMaxStringLength = 12_000;
 
 type ExecutionTaskWithExtraUsage = ExecutionTask & {
@@ -39,7 +46,7 @@ export interface MarkdownAttachment {
    * write the screenshot under this name to keep links in sync. See #2392.
    */
   suggestedFileName: string;
-  mimeType?: string;
+  mimeType?: ScreenshotImageMimeType;
   /**
    * Reference to the screenshot in the source report, used to locate the
    * original bytes when copying them to the exported name. Absent for in-memory
@@ -448,26 +455,11 @@ function extractLocateCenter(
 
 function tryExtractBase64(screenshot: unknown): string | undefined {
   if (typeof screenshot === 'string') {
-    const trimmedScreenshot = screenshot.trim();
-    const dataUrlMatch = trimmedScreenshot.match(screenshotDataUrlPattern);
-    if (dataUrlMatch) {
-      const format = dataUrlMatch[1].toLowerCase() === 'jpg' ? 'jpeg' : 'png';
-      const base64Body = dataUrlMatch[2].replace(/\s/g, '');
-      if (!base64Body) {
-        return undefined;
-      }
-      return `data:image/${format};base64,${base64Body}`;
-    }
-
-    if (
-      trimmedScreenshot.startsWith('data:') ||
-      !rawBase64BodyPattern.test(trimmedScreenshot)
-    ) {
+    try {
+      return normalizeScreenshotBase64(screenshot);
+    } catch {
       return undefined;
     }
-
-    const base64Body = trimmedScreenshot.replace(/\s/g, '');
-    return base64Body ? `data:image/png;base64,${base64Body}` : undefined;
   }
 
   if (!screenshot || typeof screenshot !== 'object') return undefined;
@@ -478,6 +470,20 @@ function tryExtractBase64(screenshot: unknown): string | undefined {
   return undefined;
 }
 
+function screenshotMetadataFromMimeType(mimeType: unknown): {
+  extension: ScreenshotImageFormat;
+  mimeType: ScreenshotImageMimeType;
+} {
+  const format = screenshotImageFormatFromMimeType(mimeType);
+  if (!format) {
+    throw new Error(`Unsupported screenshot mime type: ${String(mimeType)}`);
+  }
+  return {
+    extension: screenshotImageExtension(format),
+    mimeType: screenshotImageMimeType(format),
+  };
+}
+
 function restoredSourceRef(screenshot: unknown): ScreenshotRef | undefined {
   if (!screenshot || typeof screenshot !== 'object') {
     return undefined;
@@ -507,7 +513,7 @@ function screenshotAttachment(
       attachment: {
         id: screenshot.id,
         suggestedFileName,
-        mimeType: `image/${ext === 'jpeg' ? 'jpeg' : 'png'}`,
+        mimeType: screenshot.mimeType,
         executionIndex,
         taskIndex,
         base64Data: tryExtractBase64(screenshot),
@@ -517,7 +523,7 @@ function screenshotAttachment(
 
   const ref = normalizeScreenshotRef(screenshot);
   if (ref) {
-    const ext = ref.mimeType === 'image/jpeg' ? 'jpeg' : 'png';
+    const { extension: ext } = screenshotMetadataFromMimeType(ref.mimeType);
     const suggestedFileName = `execution-${executionIndex + 1}-task-${taskIndex + 1}-${ref.id}.${ext}`;
     return {
       markdown: `\n![${markdownLabel}](${screenshotBaseDir}/${suggestedFileName})`,
@@ -535,7 +541,9 @@ function screenshotAttachment(
 
   const sourceRef = restoredSourceRef(screenshot);
   if (sourceRef) {
-    const ext = sourceRef.mimeType === 'image/jpeg' ? 'jpeg' : 'png';
+    const { extension: ext } = screenshotMetadataFromMimeType(
+      sourceRef.mimeType,
+    );
     const suggestedFileName = `execution-${executionIndex + 1}-task-${taskIndex + 1}-${sourceRef.id}.${ext}`;
     return {
       markdown: `\n![${markdownLabel}](${screenshotBaseDir}/${suggestedFileName})`,
@@ -553,7 +561,9 @@ function screenshotAttachment(
 
   const base64 = tryExtractBase64(screenshot);
   if (base64) {
-    const ext = base64.startsWith('data:image/jpeg') ? 'jpeg' : 'png';
+    const { extension: ext, mimeType } = screenshotMetadataFromMimeType(
+      parseBase64(base64).mimeType,
+    );
     const idSuffix = options?.fallbackIdSuffix
       ? `-${options.fallbackIdSuffix}`
       : '';
@@ -564,7 +574,7 @@ function screenshotAttachment(
       attachment: {
         id,
         suggestedFileName,
-        mimeType: `image/${ext}`,
+        mimeType,
         executionIndex,
         taskIndex,
         base64Data: base64,