chore: extract large snapshots in the trace as resources

Author: dgozman

packages/playwright-core/src/server/trace/recorder/snapshotter.ts

@@ -131,22 +131,30 @@ export class Snapshotter {
         frameId: frame.guid,
         frameUrl: data.url,
         doctype: data.doctype,
-        html: data.html,
         viewport: data.viewport,
         timestamp: monotonicTime(),
         wallTime: data.wallTime,
         collectionTime: data.collectionTime,
-        resourceOverrides: [],
+        resourceOverrides: data.resourceOverrides.length ? [] : undefined,
         isMainFrame: page.mainFrame() === frame
       };
+      const htmlJson = JSON.stringify(data.html);
+      if (htmlJson.length < 200) {
+        snapshot.html = data.html;
+      } else {
+        const buffer = Buffer.from(htmlJson);
+        const sha1 = calculateSha1(buffer) + '.json';
+        snapshot.sha1 = sha1;
+        this._delegate.onSnapshotterBlob({ sha1, buffer });
+      }
       for (const { url, content, contentType } of data.resourceOverrides) {
         if (typeof content === 'string') {
           const buffer = Buffer.from(content);
           const sha1 = calculateSha1(buffer) + '.' + (mime.getExtension(contentType) || 'dat');
           this._delegate.onSnapshotterBlob({ sha1, buffer });
-          snapshot.resourceOverrides.push({ url, sha1 });
+          snapshot.resourceOverrides!.push({ url, sha1 });
         } else {
-          snapshot.resourceOverrides.push({ url, ref: content });
+          snapshot.resourceOverrides!.push({ url, ref: content });
         }
       }
       this._delegate.onFrameSnapshot(snapshot);

packages/playwright-core/src/tools/trace/traceCli.ts

@@ -690,7 +690,7 @@ export async function traceSnapshot(traceFile: string, actionId: string, options
 
   const snapshotKey = `${snapshotName}@${callId}`;
 
-  const rendered = renderer.render();
+  const rendered = await renderer.render();
   const defaultName = `snapshot-${actionId}-${snapshotName}.html`;
 
   if (options.serve) {
@@ -704,10 +704,11 @@ export async function traceSnapshot(traceFile: string, actionId: string, options
       const url = new URL('http://localhost' + request.url!);
       const searchParams = url.searchParams;
       searchParams.set('name', snapshotKey);
-      const snapshotResponse = snapshotServer.serveSnapshot(pageId, searchParams, '/snapshot');
-      response.statusCode = snapshotResponse.status;
-      snapshotResponse.headers.forEach((value, key) => response.setHeader(key, value));
-      snapshotResponse.text().then(text => response.end(text));
+      snapshotServer.serveSnapshot(pageId, searchParams, '/snapshot').then(snapshotResponse => {
+        response.statusCode = snapshotResponse.status;
+        snapshotResponse.headers.forEach((value, key) => response.setHeader(key, value));
+        snapshotResponse.text().then(text => response.end(text));
+      });
       return true;
     });
 

packages/playwright-core/src/utils/isomorphic/lruCache.ts

@@ -25,7 +25,7 @@ export class LRUCache<K, V> {
     this._size = 0;
   }
 
-  getOrCompute(key: K, compute: () => { value: V, size: number }): V {
+  async getOrCompute(key: K, compute: () => Promise<{ value: V, size: number }>): Promise<V> {
     if (this._map.has(key)) {
       const result = this._map.get(key)!;
       // reinserting makes this the least recently used entry
@@ -34,7 +34,7 @@ export class LRUCache<K, V> {
       return result.value;
     }
 
-    const result = compute();
+    const result = await compute();
 
     while (this._map.size && this._size + result.size > this._maxSize) {
       const [firstKey, firstValue] = this._map.entries().next().value!;

packages/playwright-core/src/utils/isomorphic/trace/snapshotRenderer.ts

@@ -46,9 +46,11 @@ export class SnapshotRenderer {
   private _snapshot: FrameSnapshot;
   private _callId: string;
   private _screencastFrames: PageEntry['screencastFrames'];
+  private _readSha1: (sha1: string) => Promise<Blob | undefined>;
 
-  constructor(htmlCache: LRUCache<SnapshotRenderer, string>, resources: ResourceSnapshot[], snapshots: FrameSnapshot[], screencastFrames: PageEntry['screencastFrames'], index: number) {
+  constructor(htmlCache: LRUCache<SnapshotRenderer, string>, readSha1: (sha1: string) => Promise<Blob | undefined>, resources: ResourceSnapshot[], snapshots: FrameSnapshot[], screencastFrames: PageEntry['screencastFrames'], index: number) {
     this._htmlCache = htmlCache;
+    this._readSha1 = readSha1;
     this._resources = resources;
     this._snapshots = snapshots;
     this._index = index;
@@ -74,9 +76,9 @@ export class SnapshotRenderer {
     return closestFrame?.sha1;
   }
 
-  render(): RenderedFrameSnapshot {
+  async render(): Promise<RenderedFrameSnapshot> {
     const result: string[] = [];
-    const visit = (n: NodeSnapshot, snapshotIndex: number, parentTag: string | undefined, parentAttrs: [string, string][] | undefined) => {
+    const visit = async (n: NodeSnapshot, snapshotIndex: number, parentTag: string | undefined, parentAttrs: [string, string][] | undefined) => {
       // Text node.
       if (typeof n === 'string') {
         // Best-effort Electron support: rewrite custom protocol in url() links in stylesheets.
@@ -92,7 +94,12 @@ export class SnapshotRenderer {
         // Node reference.
         const referenceIndex = snapshotIndex - n[0][0];
         if (referenceIndex >= 0 && referenceIndex <= snapshotIndex) {
-          const nodes = snapshotNodes(this._snapshots[referenceIndex]);
+          const refSnapshot = this._snapshots[referenceIndex];
+          let nodes: NodeSnapshot[] = (refSnapshot as any)._nodes;
+          if (!nodes) {
+            nodes = snapshotNodes(await this._ensureHtml(refSnapshot));
+            (refSnapshot as any)._nodes = nodes;
+          }
           const nodeIndex = n[0][1];
           if (nodeIndex >= 0 && nodeIndex < nodes.length)
             return visit(nodes[nodeIndex], referenceIndex, parentTag, parentAttrs);
@@ -134,7 +141,7 @@ export class SnapshotRenderer {
         }
         result.push('>');
         for (const child of children)
-          visit(child, snapshotIndex, nodeName, attrs);
+          await visit(child, snapshotIndex, nodeName, attrs);
         if (!autoClosing.has(nodeName))
           result.push('</', nodeName, '>');
         return;
@@ -145,8 +152,8 @@ export class SnapshotRenderer {
     };
 
     const snapshot = this._snapshot;
-    const html = this._htmlCache.getOrCompute(this, () => {
-      visit(snapshot.html, this._index, undefined, undefined);
+    const html = await this._htmlCache.getOrCompute(this, async () => {
+      await visit(await this._ensureHtml(snapshot), this._index, undefined, undefined);
       const prefix = snapshot.doctype ? `<!DOCTYPE ${snapshot.doctype}>` : '';
       const html = prefix + [
         // Hide the document in order to prevent flickering. We will unhide once script has processed shadow.
@@ -159,6 +166,19 @@ export class SnapshotRenderer {
     return { html, pageId: snapshot.pageId, frameId: snapshot.frameId, index: this._index };
   }
 
+  private async _ensureHtml(snapshot: FrameSnapshot): Promise<NodeSnapshot> {
+    if (!snapshot.html) {
+      try {
+        const blob = await this._readSha1(snapshot.sha1!);
+        const text = await blob!.text();
+        snapshot.html = JSON.parse(text);
+      } catch {
+        snapshot.html = ['html'];
+      }
+    }
+    return snapshot.html!;
+  }
+
   resourceByUrl(url: string, method: string): ResourceSnapshot | undefined {
     const snapshot = this._snapshot;
     let sameFrameResource: ResourceSnapshot | undefined;
@@ -193,12 +213,12 @@ export class SnapshotRenderer {
     let result = sameFrameResource ?? otherFrameResource;
     if (result && method.toUpperCase() === 'GET') {
       // Patch override if necessary.
-      let override = snapshot.resourceOverrides.find(o => o.url === url);
+      let override = snapshot.resourceOverrides?.find(o => o.url === url);
       if (override?.ref) {
         // "ref" means use the same content as "ref" snapshots ago.
         const index = this._index - override.ref;
         if (index >= 0 && index < this._snapshots.length)
-          override = this._snapshots[index].resourceOverrides.find(o => o.url === url);
+          override = this._snapshots[index].resourceOverrides?.find(o => o.url === url);
       }
       if (override?.sha1) {
         result = {
@@ -220,23 +240,20 @@ export class SnapshotRenderer {
 
 const autoClosing = new Set(['AREA', 'BASE', 'BR', 'COL', 'COMMAND', 'EMBED', 'HR', 'IMG', 'INPUT', 'KEYGEN', 'LINK', 'MENUITEM', 'META', 'PARAM', 'SOURCE', 'TRACK', 'WBR']);
 
-function snapshotNodes(snapshot: FrameSnapshot): NodeSnapshot[] {
-  if (!(snapshot as any)._nodes) {
-    const nodes: NodeSnapshot[] = [];
-    const visit = (n: NodeSnapshot) => {
-      if (typeof n === 'string') {
-        nodes.push(n);
-      } else if (isNodeNameAttributesChildNodesSnapshot(n)) {
-        const [,, ...children] = n;
-        for (const child of children)
-          visit(child);
-        nodes.push(n);
-      }
-    };
-    visit(snapshot.html);
-    (snapshot as any)._nodes = nodes;
-  }
-  return (snapshot as any)._nodes;
+function snapshotNodes(html: NodeSnapshot): NodeSnapshot[] {
+  const nodes: NodeSnapshot[] = [];
+  const visit = (n: NodeSnapshot) => {
+    if (typeof n === 'string') {
+      nodes.push(n);
+    } else if (isNodeNameAttributesChildNodesSnapshot(n)) {
+      const [,, ...children] = n;
+      for (const child of children)
+        visit(child);
+      nodes.push(n);
+    }
+  };
+  visit(html);
+  return nodes;
 }
 
 type ViewportSize = { width: number, height: number };

packages/playwright-core/src/utils/isomorphic/trace/snapshotServer.ts

@@ -29,12 +29,12 @@ export class SnapshotServer {
     this._resourceLoader = resourceLoader;
   }
 
-  serveSnapshot(pageOrFrameId: string, searchParams: URLSearchParams, snapshotUrl: string): Response {
+  async serveSnapshot(pageOrFrameId: string, searchParams: URLSearchParams, snapshotUrl: string): Promise<Response> {
     const snapshot = this._snapshot(pageOrFrameId, searchParams);
     if (!snapshot)
       return new Response(null, { status: 404 });
 
-    const renderedSnapshot = snapshot.render();
+    const renderedSnapshot = await snapshot.render();
     this._snapshotIds.set(snapshotUrl, snapshot);
     return new Response(renderedSnapshot.html, { status: 200, headers: { 'Content-Type': 'text/html; charset=utf-8' } });
   }

packages/playwright-core/src/utils/isomorphic/trace/snapshotStorage.ts

@@ -29,14 +29,19 @@ export class SnapshotStorage {
   private _cache = new LRUCache<SnapshotRenderer, string>(100_000_000);  // 100MB per each trace
   private _contextToResources = new Map<string, ResourceSnapshot[]>();
   private _resourceUrlsWithOverrides = new Set<string>();
+  private _readSha1: (sha1: string) => Promise<Blob | undefined>;
+
+  constructor(readSha1: (sha1: string) => Promise<Blob | undefined>) {
+    this._readSha1 = readSha1;
+  }
 
   addResource(contextId: string, resource: ResourceSnapshot): void {
     resource.request.url = rewriteURLForCustomProtocol(resource.request.url);
     this._ensureResourcesForContext(contextId).push(resource);
   }
 
   addFrameSnapshot(contextId: string, snapshot: FrameSnapshot, screencastFrames: PageEntry['screencastFrames']) {
-    for (const override of snapshot.resourceOverrides)
+    for (const override of snapshot.resourceOverrides ?? [])
       override.url = rewriteURLForCustomProtocol(override.url);
     let frameSnapshots = this._frameSnapshots.get(snapshot.frameId);
     if (!frameSnapshots) {
@@ -50,7 +55,7 @@ export class SnapshotStorage {
     }
     frameSnapshots.raw.push(snapshot);
     const resources = this._ensureResourcesForContext(contextId);
-    const renderer = new SnapshotRenderer(this._cache, resources, frameSnapshots.raw, screencastFrames, frameSnapshots.raw.length - 1);
+    const renderer = new SnapshotRenderer(this._cache, this._readSha1, resources, frameSnapshots.raw, screencastFrames, frameSnapshots.raw.length - 1);
     frameSnapshots.renderers.push(renderer);
     return renderer;
   }
@@ -72,7 +77,7 @@ export class SnapshotStorage {
     // while serving snapshots with different override values.
     for (const frameSnapshots of this._frameSnapshots.values()) {
       for (const snapshot of frameSnapshots.raw) {
-        for (const override of snapshot.resourceOverrides)
+        for (const override of snapshot.resourceOverrides ?? [])
           this._resourceUrlsWithOverrides.add(override.url);
       }
     }

packages/playwright-core/src/utils/isomorphic/trace/traceLoader.ts

@@ -53,7 +53,7 @@ export class TraceLoader {
     if (!ordinals.length)
       throw new Error('Cannot find .trace file');
 
-    this._snapshotStorage = new SnapshotStorage();
+    this._snapshotStorage = new SnapshotStorage(sha1 => this._backend.readBlob('resources/' + sha1));
 
     // 3 * ordinals progress increments below.
     const total = ordinals.length * 3;

packages/trace-viewer/src/sw/main.ts

@@ -165,7 +165,7 @@ async function doFetch(event: FetchEvent): Promise<Response> {
       if (errorResponse)
         return errorResponse;
       const pageOrFrameId = relativePath.substring('/snapshot/'.length);
-      const response = loadedTrace!.snapshotServer.serveSnapshot(pageOrFrameId, url.searchParams, url.href);
+      const response = await loadedTrace!.snapshotServer.serveSnapshot(pageOrFrameId, url.searchParams, url.href);
       if (isDeployedAsHttps)
         response.headers.set('Content-Security-Policy', 'upgrade-insecure-requests');
       return response;

packages/trace/src/snapshot.ts

@@ -47,8 +47,9 @@ export type FrameSnapshot = {
   wallTime?: number,
   collectionTime: number,
   doctype?: string,
-  html: NodeSnapshot,
-  resourceOverrides: ResourceOverride[],
+  html?: NodeSnapshot,
+  sha1?: string,
+  resourceOverrides?: ResourceOverride[],
   viewport: { width: number, height: number },
   isMainFrame: boolean,
 };