feat(har): include WebSocket in .har

add an entry for each `WebSocket` when generating a `.har`

also include data for each sent/received frame in a custom property `_webSocketMessages` (for more info see <https://developer.chrome.com/blog/new-in-devtools-76#websocket>)

both Chrome and WebKit already capture the `wallTime` for the initial request and a `timestamp` for each subsequent message (i.e. diff the `timestamp` relative to the initial `timestamp` and add the `wallTime` in order to determine the current `wallTime`)

unfortunately Firefox does not have this so fall back to `Date.now() / 1000`

Author: dcrousso

packages/playwright-core/src/server/chromium/crNetworkManager.ts

@@ -76,10 +76,10 @@ export class CRNetworkManager {
     if (this._page) {
       sessionInfo.eventListeners.push(...[
         eventsHelper.addEventListener(session, 'Network.webSocketCreated', e => this._page!.frameManager.onWebSocketCreated(e.requestId, e.url)),
-        eventsHelper.addEventListener(session, 'Network.webSocketWillSendHandshakeRequest', e => this._page!.frameManager.onWebSocketRequest(e.requestId)),
-        eventsHelper.addEventListener(session, 'Network.webSocketHandshakeResponseReceived', e => this._page!.frameManager.onWebSocketResponse(e.requestId, e.response.status, e.response.statusText)),
-        eventsHelper.addEventListener(session, 'Network.webSocketFrameSent', e => e.response.payloadData && this._page!.frameManager.onWebSocketFrameSent(e.requestId, e.response.opcode, e.response.payloadData)),
-        eventsHelper.addEventListener(session, 'Network.webSocketFrameReceived', e => e.response.payloadData && this._page!.frameManager.webSocketFrameReceived(e.requestId, e.response.opcode, e.response.payloadData)),
+        eventsHelper.addEventListener(session, 'Network.webSocketWillSendHandshakeRequest', e => this._page!.frameManager.onWebSocketRequest(e.requestId, e.wallTime, e.timestamp, headersObjectToArray(e.request.headers, '\n'))),
+        eventsHelper.addEventListener(session, 'Network.webSocketHandshakeResponseReceived', e => this._page!.frameManager.onWebSocketResponse(e.requestId, e.response.status, e.response.statusText, headersObjectToArray(e.response.headers, '\n'))),
+        eventsHelper.addEventListener(session, 'Network.webSocketFrameSent', e => e.response.payloadData && this._page!.frameManager.onWebSocketFrameSent(e.requestId, e.response.opcode, e.response.payloadData, e.timestamp)),
+        eventsHelper.addEventListener(session, 'Network.webSocketFrameReceived', e => e.response.payloadData && this._page!.frameManager.webSocketFrameReceived(e.requestId, e.response.opcode, e.response.payloadData, e.timestamp)),
         eventsHelper.addEventListener(session, 'Network.webSocketClosed', e => this._page!.frameManager.webSocketClosed(e.requestId)),
         eventsHelper.addEventListener(session, 'Network.webSocketFrameError', e => this._page!.frameManager.webSocketError(e.requestId, e.errorMessage)),
       ]);

packages/playwright-core/src/server/frames.ts

@@ -403,30 +403,44 @@ export class FrameManager {
     this._webSockets.set(requestId, ws);
   }
 
-  onWebSocketRequest(requestId: string) {
+  onWebSocketRequest(requestId: string, wallTime?: number, timestamp?: number, headers?: types.HeadersArray) {
     const ws = this._webSockets.get(requestId);
-    if (ws && ws.markAsNotified())
+    if (!ws)
+      return;
+
+    // Firefox does not provide any timing info for WebSocket activity.
+    // Remove this check if/when support is added.
+    if (wallTime !== undefined && timestamp !== undefined)
+      ws.setTimestampBaseline(wallTime, timestamp);
+
+    if (ws.markAsNotified())
       this._page.emit(Page.Events.WebSocket, ws);
+
+    // Firefox does not provide any headers for WebSocket activity.
+    // Remove this check if/when support is added.
+    if (headers)
+      ws.requestSent(headers);
   }
 
-  onWebSocketResponse(requestId: string, status: number, statusText: string) {
+  onWebSocketResponse(requestId: string, status: number, statusText: string, headers: types.HeadersArray) {
     const ws = this._webSockets.get(requestId);
-    if (status < 400)
+    if (!ws)
       return;
-    if (ws)
+    ws.responseReceived(status, statusText, headers);
+    if (status >= 400)
       ws.error(`${statusText}: ${status}`);
   }
 
-  onWebSocketFrameSent(requestId: string, opcode: number, data: string) {
+  onWebSocketFrameSent(requestId: string, opcode: number, data: string, timestamp?: number) {
     const ws = this._webSockets.get(requestId);
     if (ws)
-      ws.frameSent(opcode, data);
+      ws.frameSent(opcode, data, timestamp);
   }
 
-  webSocketFrameReceived(requestId: string, opcode: number, data: string) {
+  webSocketFrameReceived(requestId: string, opcode: number, data: string, timestamp?: number) {
     const ws = this._webSockets.get(requestId);
     if (ws)
-      ws.frameReceived(opcode, data);
+      ws.frameReceived(opcode, data, timestamp);
   }
 
   webSocketClosed(requestId: string) {

packages/playwright-core/src/server/har/harTracer.ts

@@ -30,9 +30,10 @@ import { helper } from '../helper';
 import * as network from '../network';
 import { nullProgress } from '../progress';
 
+import { Page } from '../page';
+
 import type { RegisteredListener } from '@utils/eventsHelper';
 import type { APIRequestEvent, APIRequestFinishedEvent } from '../fetch';
-import type { Page } from '../page';
 import type { Worker } from '../page';
 import type { HeadersArray, LifecycleEvent } from '../types';
 import type * as har from '@trace/har';
@@ -102,7 +103,10 @@ export class HarTracer {
     ];
     if (this._context instanceof BrowserContext) {
       this._eventListeners.push(
-          eventsHelper.addEventListener(this._context, BrowserContext.Events.Page, (page: Page) => this._createPageEntryIfNeeded(page)),
+          eventsHelper.addEventListener(this._context, BrowserContext.Events.Page, (page: Page) => {
+            this._addPageEventListeners(page);
+            this._createPageEntryIfNeeded(page);
+          }),
           eventsHelper.addEventListener(this._context, BrowserContext.Events.Request, (request: network.Request) => this._onRequest(request)),
           eventsHelper.addEventListener(this._context, BrowserContext.Events.RequestFinished, ({ request, response }) => this._onRequestFinished(request, response).catch(() => {})),
           eventsHelper.addEventListener(this._context, BrowserContext.Events.RequestFailed, request => this._onRequestFailed(request)),
@@ -111,11 +115,21 @@ export class HarTracer {
           eventsHelper.addEventListener(this._context, BrowserContext.Events.RequestFulfilled, request => this._onRequestFulfilled(request)),
           eventsHelper.addEventListener(this._context, BrowserContext.Events.RequestContinued, request => this._onRequestContinued(request)),
       );
-      for (const page of this._context.pages())
+      for (const page of this._context.pages()) {
+        this._addPageEventListeners(page);
         this._createPageEntryIfNeeded(page);
+      }
     }
   }
 
+  private _addPageEventListeners(page: Page) {
+    if (this._page && page !== this._page)
+      return;
+    this._eventListeners.push(
+        eventsHelper.addEventListener(page, Page.Events.WebSocket, (webSocket: network.WebSocket) => this._onWebSocket(page, webSocket)),
+    );
+  }
+
   private _shouldIncludeEntryWithUrl(urlString: string) {
     return !this._options.urlFilter || urlMatches(this._baseURL, urlString, this._options.urlFilter);
   }
@@ -418,6 +432,48 @@ export class HarTracer {
       harEntry._wasContinued = true;
   }
 
+  private _onWebSocket(page: Page, webSocket: network.WebSocket) {
+    if (!this._shouldIncludeEntryWithUrl(webSocket.url()))
+      return;
+    const url = network.parseURL(webSocket.url());
+    if (!url)
+      return;
+
+    const pageEntry = this._createPageEntryIfNeeded(page);
+    const harEntry = createHarEntry(pageEntry?.id, 'GET', url, page.mainFrame().guid, this._options);
+    harEntry._resourceType = 'websocket';
+    harEntry._webSocketMessages = [];
+
+    // Fallbacks in case we never receive a response.
+    harEntry.response.status = 101;
+    harEntry.response.statusText = 'Switching Protocols';
+
+    if (this._started)
+      this._delegate.onEntryStarted(harEntry);
+
+    eventsHelper.addEventListener(webSocket, network.WebSocket.Events.Request, ({ headers }: { headers: HeadersArray }) => {
+      this._recordRequestHeadersAndCookies(harEntry, headers);
+    });
+    eventsHelper.addEventListener(webSocket, network.WebSocket.Events.Response, ({ status, statusText, headers }: { status: number, statusText: string, headers: HeadersArray }) => {
+      harEntry.response.status = status;
+      harEntry.response.statusText = statusText;
+      this._recordResponseHeaders(harEntry, headers);
+    });
+    eventsHelper.addEventListener(webSocket, network.WebSocket.Events.FrameSent, ({ opcode, data, timestamp }: { opcode: number, data: string, timestamp: number }) => {
+      harEntry._webSocketMessages!.push({ type: 'send', time: timestamp, opcode, data });
+    });
+    eventsHelper.addEventListener(webSocket, network.WebSocket.Events.FrameReceived, ({ opcode, data, timestamp }: { opcode: number, data: string, timestamp: number }) => {
+      harEntry._webSocketMessages!.push({ type: 'receive', time: timestamp, opcode, data });
+    });
+    eventsHelper.addEventListener(webSocket, network.WebSocket.Events.SocketError, (errorMessage: string) => {
+      harEntry.response._failureText = errorMessage;
+    });
+    eventsHelper.addEventListener(webSocket, network.WebSocket.Events.Close, () => {
+      if (this._started)
+        this._delegate.onEntryFinished(harEntry);
+    });
+  }
+
   private _storeResponseContent(buffer: Buffer | undefined, content: har.Content, resourceType: string) {
     if (!buffer) {
       content.size = 0;

packages/playwright-core/src/server/network.ts

@@ -725,12 +725,20 @@ export class Response extends SdkObject {
 export class WebSocket extends SdkObject {
   private _url: string;
   private _notified = false;
+  private _requestWallTime: number | undefined;
+  private _requestTimestamp: number | undefined;
+  private _status: number | undefined;
+  private _statusText: string | undefined;
+  private _requestHeaders: HeadersArray | undefined;
+  private _responseHeaders: HeadersArray | undefined;
 
   static Events = {
     Close: 'close',
     SocketError: 'socketerror',
     FrameReceived: 'framereceived',
     FrameSent: 'framesent',
+    Request: 'request',
+    Response: 'response',
   };
 
   constructor(parent: SdkObject, url: string) {
@@ -748,16 +756,43 @@ export class WebSocket extends SdkObject {
     return true;
   }
 
+  setTimestampBaseline(wallTime: number, timestamp: number) {
+    this._requestWallTime = wallTime;
+    this._requestTimestamp = timestamp;
+  }
+
+  private _toWallTime(timestamp: number | undefined): number {
+    if (timestamp !== undefined && this._requestWallTime !== undefined && this._requestTimestamp !== undefined)
+      return this._requestWallTime + (timestamp - this._requestTimestamp);
+    return Date.now() / 1000;
+  }
+
   url(): string {
     return this._url;
   }
 
-  frameSent(opcode: number, data: string) {
-    this.emit(WebSocket.Events.FrameSent, { opcode, data });
+  requestSent(headers: HeadersArray) {
+    this.emit(WebSocket.Events.Request, { headers });
   }
 
-  frameReceived(opcode: number, data: string) {
-    this.emit(WebSocket.Events.FrameReceived, { opcode, data });
+  responseReceived(status: number, statusText: string, headers: HeadersArray) {
+    this.emit(WebSocket.Events.Response, { status, statusText, headers });
+  }
+
+  frameSent(opcode: number, data: string, timestamp?: number) {
+    this.emit(WebSocket.Events.FrameSent, {
+      opcode,
+      data,
+      timestamp: this._toWallTime(timestamp),
+    });
+  }
+
+  frameReceived(opcode: number, data: string, timestamp?: number) {
+    this.emit(WebSocket.Events.FrameReceived, {
+      opcode,
+      data,
+      timestamp: this._toWallTime(timestamp),
+    });
   }
 
   error(errorMessage: string) {

packages/playwright-core/src/server/webkit/webview/wvPage.ts

@@ -18,7 +18,7 @@
 import { PNG } from 'pngjs';
 import jpegjs from 'jpeg-js';
 import { assert } from '@isomorphic/assert';
-import { headersArrayToObject } from '@isomorphic/headers';
+import { headersArrayToObject, headersObjectToArray } from '@isomorphic/headers';
 import { splitErrorMessage } from '@isomorphic/stackTrace';
 import { debugLogger } from '@utils/debugLogger';
 import { eventsHelper } from '@utils/eventsHelper';
@@ -300,6 +300,8 @@ export class WVPage implements PageDelegate {
   }
 
   private _buildSessionListeners(session: WVSession): RegisteredListener[] {
+    const setCookieSeparator = process.platform === 'darwin' ? ',' : 'playwright-set-cookie-separator';
+
     return [
       eventsHelper.addEventListener(session, 'Page.frameNavigated', event => this._onFrameNavigated(event.frame, false)),
       eventsHelper.addEventListener(session, 'Page.navigatedWithinDocument', event => this._onFrameNavigatedWithinDocument(event.frameId, event.url)),
@@ -316,10 +318,10 @@ export class WVPage implements PageDelegate {
       eventsHelper.addEventListener(session, 'Network.loadingFinished', e => this._onLoadingFinished(e)),
       eventsHelper.addEventListener(session, 'Network.loadingFailed', e => this._onLoadingFailed(session, e)),
       eventsHelper.addEventListener(session, 'Network.webSocketCreated', e => this._page.frameManager.onWebSocketCreated(e.requestId, e.url)),
-      eventsHelper.addEventListener(session, 'Network.webSocketWillSendHandshakeRequest', e => this._page.frameManager.onWebSocketRequest(e.requestId)),
-      eventsHelper.addEventListener(session, 'Network.webSocketHandshakeResponseReceived', e => this._page.frameManager.onWebSocketResponse(e.requestId, e.response.status, e.response.statusText)),
-      eventsHelper.addEventListener(session, 'Network.webSocketFrameSent', e => e.response.payloadData && this._page.frameManager.onWebSocketFrameSent(e.requestId, e.response.opcode, e.response.payloadData)),
-      eventsHelper.addEventListener(session, 'Network.webSocketFrameReceived', e => e.response.payloadData && this._page.frameManager.webSocketFrameReceived(e.requestId, e.response.opcode, e.response.payloadData)),
+      eventsHelper.addEventListener(session, 'Network.webSocketWillSendHandshakeRequest', e => this._page.frameManager.onWebSocketRequest(e.requestId, e.walltime, e.timestamp, headersObjectToArray(e.request.headers))),
+      eventsHelper.addEventListener(session, 'Network.webSocketHandshakeResponseReceived', e => this._page.frameManager.onWebSocketResponse(e.requestId, e.response.status, e.response.statusText, headersObjectToArray(e.response.headers, ',', setCookieSeparator))),
+      eventsHelper.addEventListener(session, 'Network.webSocketFrameSent', e => e.response.payloadData && this._page.frameManager.onWebSocketFrameSent(e.requestId, e.response.opcode, e.response.payloadData, e.timestamp)),
+      eventsHelper.addEventListener(session, 'Network.webSocketFrameReceived', e => e.response.payloadData && this._page.frameManager.webSocketFrameReceived(e.requestId, e.response.opcode, e.response.payloadData, e.timestamp)),
       eventsHelper.addEventListener(session, 'Network.webSocketClosed', e => this._page.frameManager.webSocketClosed(e.requestId)),
       eventsHelper.addEventListener(session, 'Network.webSocketFrameError', e => this._page.frameManager.webSocketError(e.requestId, e.errorMessage)),
     ];

packages/playwright-core/src/server/webkit/wkPage.ts

@@ -17,7 +17,7 @@
 
 import { PNG } from 'pngjs';
 import jpegjs from 'jpeg-js';
-import { headersArrayToObject } from '@isomorphic/headers';
+import { headersArrayToObject, headersObjectToArray } from '@isomorphic/headers';
 import { splitErrorMessage } from '@isomorphic/stackTrace';
 import { eventsHelper } from '@utils/eventsHelper';
 import { hostPlatform } from '@utils/hostPlatform';
@@ -376,6 +376,8 @@ export class WKPage implements PageDelegate {
   }
 
   private _addSessionListeners() {
+    const setCookieSeparator = process.platform === 'darwin' ? ',' : 'playwright-set-cookie-separator';
+
     this._sessionListeners = [
       eventsHelper.addEventListener(this._session, 'Page.frameNavigated', event => this._onFrameNavigated(event.frame, false)),
       eventsHelper.addEventListener(this._session, 'Page.navigatedWithinDocument', event => this._onFrameNavigatedWithinDocument(event.frameId, event.url)),
@@ -397,10 +399,10 @@ export class WKPage implements PageDelegate {
       eventsHelper.addEventListener(this._session, 'Network.loadingFinished', e => this._onLoadingFinished(e)),
       eventsHelper.addEventListener(this._session, 'Network.loadingFailed', e => this._onLoadingFailed(this._session, e)),
       eventsHelper.addEventListener(this._session, 'Network.webSocketCreated', e => this._page.frameManager.onWebSocketCreated(e.requestId, e.url)),
-      eventsHelper.addEventListener(this._session, 'Network.webSocketWillSendHandshakeRequest', e => this._page.frameManager.onWebSocketRequest(e.requestId)),
-      eventsHelper.addEventListener(this._session, 'Network.webSocketHandshakeResponseReceived', e => this._page.frameManager.onWebSocketResponse(e.requestId, e.response.status, e.response.statusText)),
-      eventsHelper.addEventListener(this._session, 'Network.webSocketFrameSent', e => e.response.payloadData && this._page.frameManager.onWebSocketFrameSent(e.requestId, e.response.opcode, e.response.payloadData)),
-      eventsHelper.addEventListener(this._session, 'Network.webSocketFrameReceived', e => e.response.payloadData && this._page.frameManager.webSocketFrameReceived(e.requestId, e.response.opcode, e.response.payloadData)),
+      eventsHelper.addEventListener(this._session, 'Network.webSocketWillSendHandshakeRequest', e => this._page.frameManager.onWebSocketRequest(e.requestId, e.walltime, e.timestamp, headersObjectToArray(e.request.headers))),
+      eventsHelper.addEventListener(this._session, 'Network.webSocketHandshakeResponseReceived', e => this._page.frameManager.onWebSocketResponse(e.requestId, e.response.status, e.response.statusText, headersObjectToArray(e.response.headers, ',', setCookieSeparator))),
+      eventsHelper.addEventListener(this._session, 'Network.webSocketFrameSent', e => e.response.payloadData && this._page.frameManager.onWebSocketFrameSent(e.requestId, e.response.opcode, e.response.payloadData, e.timestamp)),
+      eventsHelper.addEventListener(this._session, 'Network.webSocketFrameReceived', e => e.response.payloadData && this._page.frameManager.webSocketFrameReceived(e.requestId, e.response.opcode, e.response.payloadData, e.timestamp)),
       eventsHelper.addEventListener(this._session, 'Network.webSocketClosed', e => this._page.frameManager.webSocketClosed(e.requestId)),
       eventsHelper.addEventListener(this._session, 'Network.webSocketFrameError', e => this._page.frameManager.webSocketError(e.requestId, e.errorMessage)),
     ];

packages/trace/src/har.ts

@@ -72,6 +72,15 @@ export type Entry = {
   _wasFulfilled?: boolean;
   _wasContinued?: boolean;
   _apiRequest?: boolean;
+  _resourceType?: string;
+  _webSocketMessages?: WebSocketMessage[];
+};
+
+export type WebSocketMessage = {
+  type: 'send' | 'receive';
+  time: number;
+  opcode: number;
+  data: string;
 };
 
 export type Request = {