Skip to content

Commit 1ac9a57

Browse files
dcroussodcrousso
authored
feat(har): include WebSocket in .har (#41015)
also include data for each sent/received frame in a custom property `_webSocketMessages` (for more info see <https://developer.chrome.com/blog/new-in-devtools-76#websocket>) both Chrome and WebKit already capture the `wallTime` for the initial request and a `timestamp` for each subsequent message (i.e. diff the `timestamp` relative to the initial `timestamp` and add the `wallTime` in order to determine the current `wallTime`) Firefox exposes a walltime `timestamp` for each message directly fixes <#30315>
1 parent 899c7a5 commit 1ac9a57

14 files changed

Lines changed: 634 additions & 59 deletions

File tree

‎packages/playwright-core/src/server/chromium/crNetworkManager.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -76,10 +76,10 @@ export class CRNetworkManager {
7676
if (this._page) {
7777
sessionInfo.eventListeners.push(...[
7878
eventsHelper.addEventListener(session, 'Network.webSocketCreated', e => this._page!.frameManager.onWebSocketCreated(e.requestId, e.url)),
79-
eventsHelper.addEventListener(session, 'Network.webSocketWillSendHandshakeRequest', e => this._page!.frameManager.onWebSocketRequest(e.requestId)),
80-
eventsHelper.addEventListener(session, 'Network.webSocketHandshakeResponseReceived', e => this._page!.frameManager.onWebSocketResponse(e.requestId, e.response.status, e.response.statusText)),
81-
eventsHelper.addEventListener(session, 'Network.webSocketFrameSent', e => e.response.payloadData && this._page!.frameManager.onWebSocketFrameSent(e.requestId, e.response.opcode, e.response.payloadData)),
82-
eventsHelper.addEventListener(session, 'Network.webSocketFrameReceived', e => e.response.payloadData && this._page!.frameManager.webSocketFrameReceived(e.requestId, e.response.opcode, e.response.payloadData)),
79+
eventsHelper.addEventListener(session, 'Network.webSocketWillSendHandshakeRequest', e => this._page!.frameManager.onWebSocketRequest(e.requestId, headersObjectToArray(e.request.headers, '\n'), e.wallTime, e.timestamp)),
80+
eventsHelper.addEventListener(session, 'Network.webSocketHandshakeResponseReceived', e => this._page!.frameManager.onWebSocketResponse(e.requestId, e.response.status, e.response.statusText, headersObjectToArray(e.response.headers, '\n'))),
81+
eventsHelper.addEventListener(session, 'Network.webSocketFrameSent', e => e.response.payloadData && this._page!.frameManager.onWebSocketFrameSent(e.requestId, e.response.opcode, e.response.payloadData, e.timestamp)),
82+
eventsHelper.addEventListener(session, 'Network.webSocketFrameReceived', e => e.response.payloadData && this._page!.frameManager.webSocketFrameReceived(e.requestId, e.response.opcode, e.response.payloadData, e.timestamp)),
8383
eventsHelper.addEventListener(session, 'Network.webSocketClosed', e => this._page!.frameManager.webSocketClosed(e.requestId)),
8484
eventsHelper.addEventListener(session, 'Network.webSocketFrameError', e => this._page!.frameManager.webSocketError(e.requestId, e.errorMessage)),
8585
]);

‎packages/playwright-core/src/server/firefox/ffNetworkManager.ts‎

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,20 +19,21 @@ import { eventsHelper } from '@utils/eventsHelper';
1919
import * as network from '../network';
2020

2121
import type { FFSession } from './ffConnection';
22+
import type { FFPage } from './ffPage';
2223
import type { HeadersArray } from '../../server/types';
2324
import type { RegisteredListener } from '@utils/eventsHelper';
2425
import type * as frames from '../frames';
25-
import type { Page } from '../page';
2626
import type * as types from '../types';
2727
import type { Protocol } from './protocol';
2828

2929
export class FFNetworkManager {
3030
private _session: FFSession;
3131
private _requests: Map<string, InterceptableRequest>;
32-
private _page: Page;
32+
private _page: FFPage;
3333
private _eventListeners: RegisteredListener[];
34+
private _webSocketRequestIds = new Set<string>();
3435

35-
constructor(session: FFSession, page: Page) {
36+
constructor(session: FFSession, page: FFPage) {
3637
this._session = session;
3738

3839
this._requests = new Map();
@@ -59,26 +60,36 @@ export class FFNetworkManager {
5960

6061
_onRequestWillBeSent(event: Protocol.Network.requestWillBeSentPayload) {
6162
const redirectedFrom = event.redirectedFrom ? (this._requests.get(event.redirectedFrom) || null) : null;
62-
const frame = redirectedFrom ? redirectedFrom.request.frame() : (event.frameId ? this._page.frameManager.frame(event.frameId) : null);
63+
const frame = redirectedFrom ? redirectedFrom.request.frame() : (event.frameId ? this._page._page.frameManager.frame(event.frameId) : null);
6364
if (!frame)
6465
return;
6566
// Align with Chromium and WebKit and not expose preflight OPTIONS requests to the client.
6667
if (event.method === 'OPTIONS' && !event.isIntercepted)
6768
return;
6869
if (redirectedFrom)
6970
this._requests.delete(redirectedFrom._id);
71+
// Align with Chromium and WebKit by having WebSocket be handled separately from other network activity.
72+
if (event.cause === 'TYPE_WEBSOCKET') {
73+
this._webSocketRequestIds.add(event.requestId);
74+
this._page._onWebSocketRequestWillBeSent(event.requestId, event.url, event.headers);
75+
return;
76+
}
7077
const request = new InterceptableRequest(frame, redirectedFrom, event);
7178
let route;
7279
if (event.isIntercepted)
7380
route = new FFRouteImpl(this._session, request);
7481
this._requests.set(request._id, request);
75-
this._page.frameManager.requestStarted(request.request, route);
82+
this._page._page.frameManager.requestStarted(request.request, route);
7683
}
7784

7885
_onResponseReceived(event: Protocol.Network.responseReceivedPayload) {
7986
const request = this._requests.get(event.requestId);
80-
if (!request)
87+
if (!request) {
88+
// Align with Chromium and WebKit by having WebSocket be handled separately from other network activity.
89+
if (this._webSocketRequestIds.has(event.requestId))
90+
this._page._onWebSocketResponseReceived(event.requestId, event.status, event.statusText, event.headers);
8191
return;
92+
}
8293
const getResponseBody = async () => {
8394
const response = await this._session.send('Network.getResponseBody', {
8495
requestId: request._id
@@ -124,13 +135,19 @@ export class FFNetworkManager {
124135
response.setRawResponseHeaders(null);
125136
// Headers size are not available in Firefox.
126137
response.setResponseHeadersSize(null);
127-
this._page.frameManager.requestReceivedResponse(response);
138+
this._page._page.frameManager.requestReceivedResponse(response);
128139
}
129140

130141
_onRequestFinished(event: Protocol.Network.requestFinishedPayload) {
131142
const request = this._requests.get(event.requestId);
132-
if (!request)
143+
if (!request) {
144+
// Align with Chromium and WebKit by having WebSocket be handled separately from other network activity.
145+
if (this._webSocketRequestIds.has(event.requestId)) {
146+
this._webSocketRequestIds.delete(event.requestId);
147+
this._page._onWebSocketRequestFinished(event.requestId);
148+
}
133149
return;
150+
}
134151
const response = request.request._existingResponse()!;
135152
response.setTransferSize(event.transferSize);
136153
response.setEncodedBodySize(event.encodedBodySize);
@@ -145,7 +162,7 @@ export class FFNetworkManager {
145162
response._requestFinished(responseEndTime);
146163
}
147164
response._setHttpVersion(event.protocolVersion ?? null);
148-
this._page.frameManager.reportRequestFinished(request.request, response);
165+
this._page._page.frameManager.reportRequestFinished(request.request, response);
149166
}
150167

151168
_onRequestFailed(event: Protocol.Network.requestFailedPayload) {
@@ -161,7 +178,7 @@ export class FFNetworkManager {
161178
response._setHttpVersion(null);
162179
}
163180
request.request._setFailureText(event.errorCode);
164-
this._page.frameManager.requestFailed(request.request, event.errorCode === 'NS_BINDING_ABORTED');
181+
this._page._page.frameManager.requestFailed(request.request, event.errorCode === 'NS_BINDING_ABORTED');
165182
}
166183
}
167184

‎packages/playwright-core/src/server/firefox/ffPage.ts‎

Lines changed: 53 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,12 @@
1515
* limitations under the License.
1616
*/
1717

18+
import { assert } from '@isomorphic/assert';
1819
import { splitErrorMessage } from '@isomorphic/stackTrace';
1920
import { eventsHelper } from '@utils/eventsHelper';
2021
import * as dialog from '../dialog';
2122
import * as dom from '../dom';
23+
import * as network from '../network';
2224
import { InitScript } from '../page';
2325
import { Page, Worker } from '../page';
2426
import { FFSession } from './ffConnection';
@@ -53,6 +55,8 @@ export class FFPage implements PageDelegate {
5355
private _eventListeners: RegisteredListener[];
5456
private _workers = new Map<string, { frameId: string, session: FFSession }>();
5557
private _initScripts: { initScript: InitScript, worldName?: string }[] = [];
58+
private _webSocketRequests = new Map<string, { url: string, headers: types.HeadersArray }>();
59+
private _webSocketResponses = new Map<string, { status: number, statusText: string, headers: types.HeadersArray }>();
5660

5761
constructor(session: FFSession, browserContext: FFBrowserContext, opener: FFPage | null) {
5862
this._session = session;
@@ -64,7 +68,7 @@ export class FFPage implements PageDelegate {
6468
this._browserContext = browserContext;
6569
this._page = new Page(this, browserContext);
6670
this.rawMouse.setPage(this._page);
67-
this._networkManager = new FFNetworkManager(session, this._page);
71+
this._networkManager = new FFNetworkManager(session, this);
6872
this._page.on(Page.Events.FrameDetached, frame => this._removeContextsForFrame(frame));
6973
// TODO: remove Page.willOpenNewWindowAsynchronously from the protocol.
7074
this._eventListeners = [
@@ -90,6 +94,7 @@ export class FFPage implements PageDelegate {
9094
eventsHelper.addEventListener(this._session, 'Page.crashed', this._onCrashed.bind(this)),
9195

9296
eventsHelper.addEventListener(this._session, 'Page.webSocketCreated', this._onWebSocketCreated.bind(this)),
97+
eventsHelper.addEventListener(this._session, 'Page.webSocketOpened', this._onWebSocketOpened.bind(this)),
9398
eventsHelper.addEventListener(this._session, 'Page.webSocketClosed', this._onWebSocketClosed.bind(this)),
9499
eventsHelper.addEventListener(this._session, 'Page.webSocketFrameReceived', this._onWebSocketFrameReceived.bind(this)),
95100
eventsHelper.addEventListener(this._session, 'Page.webSocketFrameSent', this._onWebSocketFrameSent.bind(this)),
@@ -119,7 +124,51 @@ export class FFPage implements PageDelegate {
119124

120125
_onWebSocketCreated(event: Protocol.Page.webSocketCreatedPayload) {
121126
this._page.frameManager.onWebSocketCreated(webSocketId(event.frameId, event.wsid), event.requestURL);
122-
this._page.frameManager.onWebSocketRequest(webSocketId(event.frameId, event.wsid));
127+
}
128+
129+
_onWebSocketRequestWillBeSent(requestId: string, url: string, headers: types.HeadersArray) {
130+
this._webSocketRequests.set(requestId, { url, headers });
131+
}
132+
133+
_onWebSocketResponseReceived(requestId: string, status: number, statusText: string, headers: types.HeadersArray) {
134+
this._webSocketResponses.set(requestId, { status, statusText, headers });
135+
}
136+
137+
_onWebSocketRequestFinished(requestId: string) {
138+
const response = this._webSocketResponses.get(requestId);
139+
assert(response);
140+
// If the request does not succeed then the WebSocket will never open, so pretend that it did.
141+
if (response.status >= 400) {
142+
const request = this._webSocketRequests.get(requestId);
143+
assert(request);
144+
145+
this._webSocketRequests.delete(requestId);
146+
this._webSocketResponses.delete(requestId);
147+
148+
const url = network.parseURL(request.url);
149+
assert(url);
150+
url.protocol = url.protocol === 'https' ? 'wss' : 'ws';
151+
152+
this._page.frameManager.onWebSocketCreated(requestId, url.toString());
153+
this._page.frameManager.onWebSocketRequest(requestId, request.headers);
154+
this._page.frameManager.onWebSocketResponse(requestId, response.status, response.statusText, response.headers);
155+
this._page.frameManager.webSocketClosed(requestId);
156+
return;
157+
}
158+
}
159+
160+
_onWebSocketOpened(event: Protocol.Page.webSocketOpenedPayload) {
161+
const request = this._webSocketRequests.get(event.requestId);
162+
assert(request);
163+
164+
const response = this._webSocketResponses.get(event.requestId);
165+
assert(response);
166+
167+
this._webSocketRequests.delete(event.requestId);
168+
this._webSocketResponses.delete(event.requestId);
169+
170+
this._page.frameManager.onWebSocketRequest(webSocketId(event.frameId, event.wsid), request.headers);
171+
this._page.frameManager.onWebSocketResponse(webSocketId(event.frameId, event.wsid), response.status, response.statusText, response.headers);
123172
}
124173

125174
_onWebSocketClosed(event: Protocol.Page.webSocketClosedPayload) {
@@ -129,11 +178,11 @@ export class FFPage implements PageDelegate {
129178
}
130179

131180
_onWebSocketFrameReceived(event: Protocol.Page.webSocketFrameReceivedPayload) {
132-
this._page.frameManager.webSocketFrameReceived(webSocketId(event.frameId, event.wsid), event.opcode, event.data);
181+
this._page.frameManager.webSocketFrameReceived(webSocketId(event.frameId, event.wsid), event.opcode, event.data, event.timestamp);
133182
}
134183

135184
_onWebSocketFrameSent(event: Protocol.Page.webSocketFrameSentPayload) {
136-
this._page.frameManager.onWebSocketFrameSent(webSocketId(event.frameId, event.wsid), event.opcode, event.data);
185+
this._page.frameManager.onWebSocketFrameSent(webSocketId(event.frameId, event.wsid), event.opcode, event.data, event.timestamp);
137186
}
138187

139188
_onExecutionContextCreated(payload: Protocol.Runtime.executionContextCreatedPayload) {

‎packages/playwright-core/src/server/frames.ts‎

Lines changed: 24 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -403,43 +403,56 @@ export class FrameManager {
403403
this._webSockets.set(requestId, ws);
404404
}
405405

406-
onWebSocketRequest(requestId: string) {
406+
onWebSocketRequest(requestId: string, headers: types.HeadersArray, wallTime?: number, timestamp?: number) {
407407
const ws = this._webSockets.get(requestId);
408-
if (ws && ws.markAsNotified())
408+
if (!ws)
409+
return;
410+
411+
if (ws.markAsNotified())
409412
this._page.emit(Page.Events.WebSocket, ws);
413+
414+
ws.requestSent(headers, wallTime, timestamp);
410415
}
411416

412-
onWebSocketResponse(requestId: string, status: number, statusText: string) {
417+
onWebSocketResponse(requestId: string, status: number, statusText: string, headers: types.HeadersArray) {
413418
const ws = this._webSockets.get(requestId);
414-
if (status < 400)
419+
if (!ws)
415420
return;
416-
if (ws)
421+
422+
ws.responseReceived(status, statusText, headers);
423+
if (status >= 400)
417424
ws.error(`${statusText}: ${status}`);
418425
}
419426

420-
onWebSocketFrameSent(requestId: string, opcode: number, data: string) {
427+
onWebSocketFrameSent(requestId: string, opcode: number, data: string, timestamp: number) {
421428
const ws = this._webSockets.get(requestId);
422429
if (ws)
423-
ws.frameSent(opcode, data);
430+
ws.frameSent(opcode, data, timestamp);
424431
}
425432

426-
webSocketFrameReceived(requestId: string, opcode: number, data: string) {
433+
webSocketFrameReceived(requestId: string, opcode: number, data: string, timestamp: number) {
427434
const ws = this._webSockets.get(requestId);
428435
if (ws)
429-
ws.frameReceived(opcode, data);
436+
ws.frameReceived(opcode, data, timestamp);
430437
}
431438

432439
webSocketClosed(requestId: string) {
433440
const ws = this._webSockets.get(requestId);
434-
if (ws)
441+
if (ws) {
442+
if (ws.markAsNotified())
443+
this._page.emit(Page.Events.WebSocket, ws);
435444
ws.closed();
445+
}
436446
this._webSockets.delete(requestId);
437447
}
438448

439449
webSocketError(requestId: string, errorMessage: string): void {
440450
const ws = this._webSockets.get(requestId);
441-
if (ws)
451+
if (ws) {
452+
if (ws.markAsNotified())
453+
this._page.emit(Page.Events.WebSocket, ws);
442454
ws.error(errorMessage);
455+
}
443456
}
444457

445458
private _fireInternalFrameNavigation(frame: Frame, event: NavigationEvent) {

‎packages/playwright-core/src/server/har/harTracer.ts‎

Lines changed: 60 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,10 @@ import { helper } from '../helper';
3030
import * as network from '../network';
3131
import { nullProgress } from '../progress';
3232

33+
import { Page } from '../page';
34+
3335
import type { RegisteredListener } from '@utils/eventsHelper';
3436
import type { APIRequestEvent, APIRequestFinishedEvent } from '../fetch';
35-
import type { Page } from '../page';
3637
import type { Worker } from '../page';
3738
import type { HeadersArray, LifecycleEvent } from '../types';
3839
import type * as har from '@trace/har';
@@ -102,7 +103,10 @@ export class HarTracer {
102103
];
103104
if (this._context instanceof BrowserContext) {
104105
this._eventListeners.push(
105-
eventsHelper.addEventListener(this._context, BrowserContext.Events.Page, (page: Page) => this._createPageEntryIfNeeded(page)),
106+
eventsHelper.addEventListener(this._context, BrowserContext.Events.Page, (page: Page) => {
107+
this._addPageEventListeners(page);
108+
this._createPageEntryIfNeeded(page);
109+
}),
106110
eventsHelper.addEventListener(this._context, BrowserContext.Events.Request, (request: network.Request) => this._onRequest(request)),
107111
eventsHelper.addEventListener(this._context, BrowserContext.Events.RequestFinished, ({ request, response }) => this._onRequestFinished(request, response).catch(() => {})),
108112
eventsHelper.addEventListener(this._context, BrowserContext.Events.RequestFailed, request => this._onRequestFailed(request)),
@@ -111,11 +115,21 @@ export class HarTracer {
111115
eventsHelper.addEventListener(this._context, BrowserContext.Events.RequestFulfilled, request => this._onRequestFulfilled(request)),
112116
eventsHelper.addEventListener(this._context, BrowserContext.Events.RequestContinued, request => this._onRequestContinued(request)),
113117
);
114-
for (const page of this._context.pages())
118+
for (const page of this._context.pages()) {
119+
this._addPageEventListeners(page);
115120
this._createPageEntryIfNeeded(page);
121+
}
116122
}
117123
}
118124

125+
private _addPageEventListeners(page: Page) {
126+
if (this._page && page !== this._page)
127+
return;
128+
this._eventListeners.push(
129+
eventsHelper.addEventListener(page, Page.Events.WebSocket, (webSocket: network.WebSocket) => this._onWebSocket(page, webSocket)),
130+
);
131+
}
132+
119133
private _shouldIncludeEntryWithUrl(urlString: string) {
120134
return !this._options.urlFilter || urlMatches(this._baseURL, urlString, this._options.urlFilter);
121135
}
@@ -418,6 +432,49 @@ export class HarTracer {
418432
harEntry._wasContinued = true;
419433
}
420434

435+
private _onWebSocket(page: Page, webSocket: network.WebSocket) {
436+
if (!this._shouldIncludeEntryWithUrl(webSocket.url()))
437+
return;
438+
const url = network.parseURL(webSocket.url());
439+
if (!url)
440+
return;
441+
442+
const pageEntry = this._createPageEntryIfNeeded(page);
443+
const harEntry = createHarEntry(pageEntry?.id, 'GET', url, page.mainFrame().guid, this._options);
444+
harEntry._resourceType = 'websocket';
445+
harEntry._webSocketMessages = [];
446+
447+
const eventListeners = [
448+
eventsHelper.addEventListener(webSocket, network.WebSocket.Events.Request, ({ headers }: { headers: HeadersArray }) => {
449+
this._recordRequestHeadersAndCookies(harEntry, headers);
450+
}),
451+
eventsHelper.addEventListener(webSocket, network.WebSocket.Events.Response, ({ status, statusText, headers }: { status: number, statusText: string, headers: HeadersArray }) => {
452+
harEntry.response.status = status;
453+
harEntry.response.statusText = statusText;
454+
this._recordResponseHeaders(harEntry, headers);
455+
}),
456+
eventsHelper.addEventListener(webSocket, network.WebSocket.Events.FrameSent, ({ opcode, data, timestamp }: { opcode: number, data: string, timestamp: number }) => {
457+
harEntry._webSocketMessages!.push({ type: 'send', time: timestamp, opcode, data });
458+
}),
459+
eventsHelper.addEventListener(webSocket, network.WebSocket.Events.FrameReceived, ({ opcode, data, timestamp }: { opcode: number, data: string, timestamp: number }) => {
460+
harEntry._webSocketMessages!.push({ type: 'receive', time: timestamp, opcode, data });
461+
}),
462+
eventsHelper.addEventListener(webSocket, network.WebSocket.Events.SocketError, (errorMessage: string) => {
463+
harEntry.response._failureText = errorMessage;
464+
}),
465+
eventsHelper.addEventListener(webSocket, network.WebSocket.Events.Close, () => {
466+
eventsHelper.removeEventListeners(eventListeners);
467+
468+
if (this._started)
469+
this._delegate.onEntryFinished(harEntry);
470+
}),
471+
];
472+
this._eventListeners.push(...eventListeners);
473+
474+
if (this._started)
475+
this._delegate.onEntryStarted(harEntry);
476+
}
477+
421478
private _storeResponseContent(buffer: Buffer | undefined, content: har.Content, resourceType: string) {
422479
if (!buffer) {
423480
content.size = 0;

0 commit comments

Comments
 (0)