Skip to content

Commit d84b795

Browse files
quanruzhuoxiuquanruzhuoxiu
committed
feat(web-integration): use WebP for browser screenshot producers
1 parent 88e097e commit d84b795

19 files changed

Lines changed: 1004 additions & 152 deletions

‎apps/chrome-extension/src/extension/recorder/screenshot-export.ts‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
import {
2+
type ScreenshotImageFormat,
3+
type ScreenshotImageMimeType,
4+
inferScreenshotImageFormatFromBase64,
5+
screenshotImageExtension,
6+
screenshotImageMimeType,
7+
} from '@midscene/shared/img/image-format';
8+
import type { RecordingSession } from '../../store';
9+
10+
export interface RecorderScreenshotAsset {
11+
body: string;
12+
extension: ScreenshotImageFormat;
13+
mimeType: ScreenshotImageMimeType;
14+
}
15+
16+
export const recorderScreenshotAsset = (
17+
screenshotBase64: string,
18+
): RecorderScreenshotAsset => {
19+
const separator = ';base64,';
20+
const separatorIndex = screenshotBase64.indexOf(separator);
21+
const body = (
22+
separatorIndex === -1
23+
? screenshotBase64
24+
: screenshotBase64.slice(separatorIndex + separator.length)
25+
).replace(/\s/g, '');
26+
const format = inferScreenshotImageFormatFromBase64(body);
27+
if (!format) {
28+
throw new Error('Unsupported recorder screenshot image format');
29+
}
30+
31+
return {
32+
body,
33+
extension: screenshotImageExtension(format),
34+
mimeType: screenshotImageMimeType(format),
35+
};
36+
};
37+
38+
export const generateEventsMarkdownTable = (
39+
sessions: RecordingSession[],
40+
): string => {
41+
let markdown = '# Test Events Report\n\n';
42+
43+
sessions.forEach((session, sessionIndex) => {
44+
if (session.events.length === 0) return;
45+
46+
markdown += `## ${session.name}\n\n`;
47+
if (session.description) {
48+
markdown += `**Description:** ${session.description}\n\n`;
49+
}
50+
markdown += `**Created:** ${new Date(session.createdAt).toLocaleString()}\n\n`;
51+
52+
markdown += '| Page | Screenshot Before | Screenshot After | Action |\n';
53+
markdown += '|------|------------|------------|--------|\n';
54+
55+
session.events.forEach((event, eventIndex) => {
56+
const page = event.title || event.url || '';
57+
const screenshotBefore = event.screenshotBefore
58+
? `![](./images/screenshot_${sessionIndex}_${eventIndex}_before.${recorderScreenshotAsset(event.screenshotBefore).extension})`
59+
: 'N/A';
60+
const screenshotAfter = event.screenshotAfter
61+
? `![](./images/screenshot_${sessionIndex}_${eventIndex}_after.${recorderScreenshotAsset(event.screenshotAfter).extension})`
62+
: 'N/A';
63+
let action = '';
64+
switch (event.type) {
65+
case 'click':
66+
action = `Click on ${event.elementDescription || 'element'}`;
67+
break;
68+
case 'input':
69+
action = `Input "${event.value}" into ${event.elementDescription || 'field'}`;
70+
break;
71+
case 'navigation':
72+
action = `Navigate to ${event.url}`;
73+
break;
74+
default:
75+
action = `${event.type} on ${event.elementDescription || 'element'}`;
76+
}
77+
78+
markdown += `| ${page} | ${screenshotBefore} | ${screenshotAfter} | ${action} |\n`;
79+
});
80+
81+
if (session.generatedCode?.yaml || session.generatedCode?.playwright) {
82+
markdown += '## Generated Code\n\n';
83+
if (session.generatedCode?.yaml) {
84+
markdown += '### YAML\n\n';
85+
markdown += `\`\`\`yaml\n${session.generatedCode.yaml}\n\`\`\`\n\n`;
86+
}
87+
if (session.generatedCode?.playwright) {
88+
markdown += '### Playwright\n\n';
89+
markdown += `\`\`\`playwright\n${session.generatedCode.playwright}\n\`\`\`\n\n`;
90+
}
91+
}
92+
93+
markdown += '\n\n\n';
94+
});
95+
96+
return markdown;
97+
};

‎apps/chrome-extension/src/extension/recorder/utils.ts‎

Lines changed: 16 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,11 @@ import JSZip from 'jszip';
1212
import type { ChatCompletionContentPart } from 'openai/resources/index';
1313
import type { RecordingSession } from '../../store';
1414
import { recordLogger } from './logger';
15+
import {
16+
type RecorderScreenshotAsset,
17+
generateEventsMarkdownTable,
18+
recorderScreenshotAsset,
19+
} from './screenshot-export';
1520
import { isChromeExtension, safeChromeAPI } from './types';
1621

1722
// Generate default session name with current time
@@ -713,74 +718,12 @@ const generateDetailedSequentialMindmap = (
713718
return mermaid;
714719
};
715720

716-
// Generate markdown table for event details
717-
const generateEventsMarkdownTable = (sessions: RecordingSession[]): string => {
718-
let markdown = '# Test Events Report\n\n';
719-
720-
sessions.forEach((session, sessionIndex) => {
721-
if (session.events.length === 0) return;
722-
723-
markdown += `## ${session.name}\n\n`;
724-
if (session.description) {
725-
markdown += `**Description:** ${session.description}\n\n`;
726-
}
727-
markdown += `**Created:** ${new Date(session.createdAt).toLocaleString()}\n\n`;
728-
729-
markdown += '| Page | Screenshot Before | Screenshot After | Action |\n';
730-
markdown += '|------|------------|------------|--------|\n';
731-
732-
session.events.forEach((event, eventIndex) => {
733-
let expected = 'N/A';
734-
const page = event.title || event.url || '';
735-
const screenshotBefore = event.screenshotBefore
736-
? `![](./images/screenshot_${sessionIndex}_${eventIndex}_before.png)`
737-
: 'N/A';
738-
const screenshotAfter = event.screenshotAfter
739-
? `![](./images/screenshot_${sessionIndex}_${eventIndex}_after.png)`
740-
: 'N/A';
741-
if (event.type === 'navigation') {
742-
expected = `Navigate to ${event.url}`;
743-
}
744-
745-
let action = '';
746-
switch (event.type) {
747-
case 'click':
748-
action = `Click on ${event.elementDescription || 'element'}`;
749-
break;
750-
case 'input':
751-
action = `Input "${event.value}" into ${event.elementDescription || 'field'}`;
752-
break;
753-
case 'navigation':
754-
action = `Navigate to ${event.url}`;
755-
break;
756-
default:
757-
action = `${event.type} on ${event.elementDescription || 'element'}`;
758-
}
759-
760-
markdown += `| ${page} | ${screenshotBefore} | ${screenshotAfter} | ${action} |\n`;
761-
});
762-
763-
if (session.generatedCode?.yaml || session.generatedCode?.playwright) {
764-
markdown += '## Generated Code\n\n';
765-
if (session.generatedCode?.yaml) {
766-
markdown += '### YAML\n\n';
767-
markdown += `\`\`\`yaml\n${session.generatedCode.yaml}\n\`\`\`\n\n`;
768-
}
769-
if (session.generatedCode?.playwright) {
770-
markdown += '### Playwright\n\n';
771-
markdown += `\`\`\`playwright\n${session.generatedCode.playwright}\n\`\`\`\n\n`;
772-
}
773-
}
774-
775-
markdown += '\n\n\n';
776-
});
777-
778-
return markdown;
779-
};
780-
781721
// Convert base64 to blob
782-
const base64ToBlob = (base64: string, mimeType: string): Blob => {
783-
const byteCharacters = atob(base64.split(',')[1]);
722+
const base64BodyToBlob = (
723+
base64Body: string,
724+
mimeType: RecorderScreenshotAsset['mimeType'],
725+
): Blob => {
726+
const byteCharacters = atob(base64Body);
784727
const byteNumbers = new Array(byteCharacters.length);
785728
for (let i = 0; i < byteCharacters.length; i++) {
786729
byteNumbers[i] = byteCharacters.charCodeAt(i);
@@ -810,15 +753,16 @@ export const exportAllEventsToZip = async (sessions: RecordingSession[]) => {
810753
// Process each session and extract images
811754
sessionsWithEvents.forEach((session, sessionIndex) => {
812755
session.events.forEach((event, eventIndex) => {
813-
const ext = 'png';
814756
if (event.screenshotBefore) {
815-
const fileName = `screenshot_${sessionIndex}_${eventIndex}_before.${ext}`;
816-
const blob = base64ToBlob(event.screenshotBefore, `image/${ext}`);
757+
const asset = recorderScreenshotAsset(event.screenshotBefore);
758+
const fileName = `screenshot_${sessionIndex}_${eventIndex}_before.${asset.extension}`;
759+
const blob = base64BodyToBlob(asset.body, asset.mimeType);
817760
imagesFolder?.file(fileName, blob);
818761
}
819762
if (event.screenshotAfter) {
820-
const fileName = `screenshot_${sessionIndex}_${eventIndex}_after.${ext}`;
821-
const blob = base64ToBlob(event.screenshotAfter, `image/${ext}`);
763+
const asset = recorderScreenshotAsset(event.screenshotAfter);
764+
const fileName = `screenshot_${sessionIndex}_${eventIndex}_after.${asset.extension}`;
765+
const blob = base64BodyToBlob(asset.body, asset.mimeType);
822766
imagesFolder?.file(fileName, blob);
823767
}
824768
});

‎apps/chrome-extension/src/scripts/worker.ts‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import type { UIContext } from '@midscene/core';
44
import { uuid } from '@midscene/shared/utils';
55
import { BridgeConnector, type BridgeStatus } from '../utils/bridgeConnector';
66
import { registerAlarmListener, safeSetupKeepalive } from '../utils/keepalive';
7+
import { canonicalizeRecorderScreenshot } from '../utils/screenshot';
78
import { workerMessageTypes } from '../utils/workerMessageTypes';
89

910
// save screenshot
@@ -448,8 +449,22 @@ chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
448449
chrome.runtime.lastError,
449450
);
450451
sendResponse(null);
452+
} else if (!dataUrl) {
453+
console.error(
454+
'[ServiceWorker] Screenshot capture returned empty data',
455+
);
456+
sendResponse(null);
451457
} else {
452-
sendResponse(dataUrl);
458+
void canonicalizeRecorderScreenshot(dataUrl).then(
459+
(webpDataUrl) => sendResponse(webpDataUrl),
460+
(error) => {
461+
console.error(
462+
'[ServiceWorker] Failed to encode recorder screenshot as WebP:',
463+
error,
464+
);
465+
sendResponse(null);
466+
},
467+
);
453468
}
454469
},
455470
);

‎apps/chrome-extension/src/utils/screenshot.ts‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
const WEBP_QUALITY = 0.9;
2+
const DATA_URL_PATTERN = /^data:image\/(png|jpe?g|webp);base64,([\s\S]+)$/i;
3+
4+
function decodeScreenshotDataUrl(dataUrl: string): {
5+
bytes: Uint8Array;
6+
mimeType: string;
7+
} {
8+
const match = DATA_URL_PATTERN.exec(dataUrl);
9+
if (!match) {
10+
throw new Error(
11+
'Recorder screenshot must be a PNG, JPEG, or WebP data URL',
12+
);
13+
}
14+
15+
const binary = atob(match[2]);
16+
const bytes = Uint8Array.from(binary, (character) => character.charCodeAt(0));
17+
return {
18+
bytes,
19+
mimeType: `image/${match[1].toLowerCase().replace('jpg', 'jpeg')}`,
20+
};
21+
}
22+
23+
function isWebp(bytes: Uint8Array): boolean {
24+
return (
25+
bytes.length >= 12 &&
26+
String.fromCharCode(...bytes.subarray(0, 4)) === 'RIFF' &&
27+
String.fromCharCode(...bytes.subarray(8, 12)) === 'WEBP'
28+
);
29+
}
30+
31+
function encodeBase64(bytes: Uint8Array): string {
32+
let binary = '';
33+
const chunkSize = 0x8000;
34+
for (let offset = 0; offset < bytes.length; offset += chunkSize) {
35+
binary += String.fromCharCode(
36+
...bytes.subarray(offset, offset + chunkSize),
37+
);
38+
}
39+
return btoa(binary);
40+
}
41+
42+
/** Convert captureVisibleTab's PNG output to the recorder's canonical WebP. */
43+
export async function canonicalizeRecorderScreenshot(
44+
dataUrl: string,
45+
): Promise<string> {
46+
const source = decodeScreenshotDataUrl(dataUrl);
47+
if (source.mimeType === 'image/webp') {
48+
if (!isWebp(source.bytes)) {
49+
throw new Error('Recorder screenshot has an invalid WebP signature');
50+
}
51+
return dataUrl;
52+
}
53+
54+
const bitmap = await createImageBitmap(
55+
new Blob([source.bytes], { type: source.mimeType }),
56+
);
57+
try {
58+
const canvas = new OffscreenCanvas(bitmap.width, bitmap.height);
59+
const context = canvas.getContext('2d');
60+
if (!context) {
61+
throw new Error('Recorder screenshot WebP encoder is unavailable');
62+
}
63+
context.drawImage(bitmap, 0, 0);
64+
65+
const output = await canvas.convertToBlob({
66+
type: 'image/webp',
67+
quality: WEBP_QUALITY,
68+
});
69+
const bytes = new Uint8Array(await output.arrayBuffer());
70+
if (output.type !== 'image/webp' || !isWebp(bytes)) {
71+
throw new Error('Recorder screenshot encoder returned invalid WebP');
72+
}
73+
return `data:image/webp;base64,${encodeBase64(bytes)}`;
74+
} finally {
75+
bitmap.close();
76+
}
77+
}

‎apps/chrome-extension/tests/recorder-export-image-format.test.ts‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
import { describe, expect, it } from 'vitest';
2+
import {
3+
generateEventsMarkdownTable,
4+
recorderScreenshotAsset,
5+
} from '../src/extension/recorder/screenshot-export';
6+
import type { RecordingSession } from '../src/store';
7+
8+
const webpBody =
9+
'UklGRioAAABXRUJQVlA4IB4AAAAwAQCdASoBAAEAAUAmJQBOgCHwAP7+hNQAAAA=';
10+
const pngBody =
11+
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==';
12+
13+
describe('Chrome recorder screenshot export', () => {
14+
it('derives the exported extension and MIME type from the image bytes', () => {
15+
expect(
16+
recorderScreenshotAsset(`data:image/png;base64,${webpBody}`),
17+
).toEqual({
18+
body: webpBody,
19+
extension: 'webp',
20+
mimeType: 'image/webp',
21+
});
22+
expect(recorderScreenshotAsset(`data:image/png;base64,${pngBody}`)).toEqual(
23+
{
24+
body: pngBody,
25+
extension: 'png',
26+
mimeType: 'image/png',
27+
},
28+
);
29+
});
30+
31+
it('uses each screenshot actual extension in the Markdown table', () => {
32+
const sessions = [
33+
{
34+
id: 'session-1',
35+
name: 'WebP export',
36+
createdAt: 1,
37+
updatedAt: 1,
38+
status: 'completed',
39+
events: [
40+
{
41+
type: 'click',
42+
timestamp: 1,
43+
hashId: 'event-1',
44+
screenshotBefore: `data:image/webp;base64,${webpBody}`,
45+
screenshotAfter: `data:image/png;base64,${pngBody}`,
46+
},
47+
],
48+
},
49+
] as RecordingSession[];
50+
51+
const markdown = generateEventsMarkdownTable(sessions);
52+
53+
expect(markdown).toContain('![](./images/screenshot_0_0_before.webp)');
54+
expect(markdown).toContain('![](./images/screenshot_0_0_after.png)');
55+
expect(markdown).not.toContain('screenshot_0_0_before.png');
56+
});
57+
});

0 commit comments

Comments
 (0)