@@ -16,7 +16,7 @@ export default class Queue {
16
16
private processing : boolean = false ;
17
17
private processingSnapshot : string = '' ;
18
18
private ctx : Context ;
19
-
19
+
20
20
constructor ( ctx : Context ) {
21
21
this . ctx = ctx ;
22
22
}
@@ -28,7 +28,7 @@ export default class Queue {
28
28
this . processNext ( ) ;
29
29
}
30
30
}
31
-
31
+
32
32
private async processNext ( ) : Promise < void > {
33
33
if ( ! this . isEmpty ( ) ) {
34
34
const snapshot = this . snapshots . shift ( ) ;
@@ -37,10 +37,10 @@ export default class Queue {
37
37
let { processedSnapshot, warnings } = await processSnapshot ( snapshot , this . ctx ) ;
38
38
await this . ctx . client . uploadSnapshot ( this . ctx , processedSnapshot ) ;
39
39
this . ctx . totalSnapshots ++ ;
40
- this . processedSnapshots . push ( { name : snapshot . name , warnings} ) ;
40
+ this . processedSnapshots . push ( { name : snapshot . name , warnings } ) ;
41
41
} catch ( error : any ) {
42
42
this . ctx . log . debug ( `snapshot failed; ${ error } ` ) ;
43
- this . processedSnapshots . push ( { name : snapshot . name , error : error . message } ) ;
43
+ this . processedSnapshots . push ( { name : snapshot . name , error : error . message } ) ;
44
44
}
45
45
// Close open browser contexts and pages
46
46
if ( this . ctx . browser ) {
@@ -77,8 +77,8 @@ export default class Queue {
77
77
}
78
78
79
79
async function processSnapshot ( snapshot : Snapshot , ctx : Context ) : Promise < Record < string , any > > {
80
- updateLogContext ( { task : 'discovery' } ) ;
81
- ctx . log . debug ( `Processing snapshot ${ snapshot . name } ` ) ;
80
+ updateLogContext ( { task : 'discovery' } ) ;
81
+ ctx . log . debug ( `Processing snapshot ${ snapshot . name } ${ snapshot . url } ` ) ;
82
82
83
83
let launchOptions : Record < string , any > = { headless : true }
84
84
let contextOptions : Record < string , any > = {
@@ -92,35 +92,44 @@ async function processSnapshot(snapshot: Snapshot, ctx: Context): Promise<Record
92
92
}
93
93
const context = await ctx . browser . newContext ( contextOptions ) ;
94
94
ctx . log . debug ( `Browser context created with options ${ JSON . stringify ( contextOptions ) } ` ) ;
95
-
96
95
// Setting the cookies in playwright context
97
- const domainName = new URL ( snapshot . url ) . hostname ;
98
- ctx . log . debug ( 'Setting cookies in context for domain:' , domainName ) ;
99
-
100
-
101
- const cookieArray = snapshot . dom . cookies . split ( '; ' ) . map ( cookie => {
102
- if ( ! cookie ) return null ;
103
- const [ name , value ] = cookie . split ( '=' ) ;
104
- if ( ! name || ! value ) return null ;
105
-
106
- return {
107
- name : name . trim ( ) ,
108
- value : value . trim ( ) ,
109
- domain : domainName ,
110
- path : '/'
111
- } ;
112
- } ) . filter ( Boolean ) ;
113
-
114
-
115
- if ( cookieArray && Array . isArray ( cookieArray ) && cookieArray . length > 0 ) {
116
- await context . addCookies ( cookieArray ) ;
117
- ctx . log . debug ( 'Cookies added' ) ;
118
- } else {
119
- ctx . log . debug ( 'No valid cookies to add' ) ;
96
+ if ( snapshot . dom . cookies ) {
97
+ const domainName = new URL ( snapshot . url ) . hostname ;
98
+ ctx . log . debug ( `Setting cookies for domain: ${ domainName } ` ) ;
99
+
100
+ const cookieArray = snapshot . dom . cookies . split ( '; ' ) . map ( cookie => {
101
+ if ( ! cookie ) return null ;
102
+ const [ name , value ] = cookie . split ( '=' ) ;
103
+ if ( ! name || ! value ) return null ;
104
+
105
+ return {
106
+ name : name . trim ( ) ,
107
+ value : value . trim ( ) ,
108
+ domain : domainName ,
109
+ path : '/'
110
+ } ;
111
+ } ) . filter ( Boolean ) ;
112
+
113
+ if ( cookieArray . length > 0 ) {
114
+ await context . addCookies ( cookieArray ) ;
115
+ } else {
116
+ ctx . log . debug ( 'No valid cookies to add' ) ;
117
+ }
120
118
}
121
-
122
119
const page = await context . newPage ( ) ;
120
+
121
+ // populate cache with already captured resources
123
122
let cache : Record < string , any > = { } ;
123
+ if ( snapshot . dom . resources . length ) {
124
+ for ( let resource of snapshot . dom . resources ) {
125
+ // convert text/css content to base64
126
+ let body = resource . mimetype == 'text/css' ? Buffer . from ( resource . content ) . toString ( 'base64' ) : resource . content ;
127
+ cache [ resource . url ] = {
128
+ body : body ,
129
+ type : resource . mimetype
130
+ }
131
+ }
132
+ }
124
133
125
134
// Use route to intercept network requests and discover resources
126
135
await page . route ( '**/*' , async ( route , request ) => {
@@ -138,7 +147,7 @@ async function processSnapshot(snapshot: Snapshot, ctx: Context): Promise<Record
138
147
ctx . config . allowedHostnames . push ( new URL ( snapshot . url ) . hostname ) ;
139
148
if ( ctx . config . enableJavaScript ) ALLOWED_RESOURCES . push ( 'script' ) ;
140
149
if ( ctx . config . basicAuthorization ) {
141
- ctx . log . debug ( `Adding basic authorization to the headers for root url` ) ;
150
+ ctx . log . debug ( `Adding basic authorization to the headers for root url` ) ;
142
151
let token = Buffer . from ( `${ ctx . config . basicAuthorization . username } :${ ctx . config . basicAuthorization . password } ` ) . toString ( 'base64' ) ;
143
152
requestOptions . headers = {
144
153
...request . headers ( ) ,
@@ -151,9 +160,15 @@ async function processSnapshot(snapshot: Snapshot, ctx: Context): Promise<Record
151
160
if ( requestUrl === snapshot . url ) {
152
161
response = {
153
162
status : ( ) => 200 ,
154
- headers : ( ) => ( { 'content-type' : 'text/html' } )
163
+ headers : ( ) => ( { 'content-type' : 'text/html' } )
164
+ }
165
+ body = snapshot . dom . html ;
166
+ } else if ( cache [ requestUrl ] ) {
167
+ response = {
168
+ status : ( ) => 200 ,
169
+ headers : ( ) => ( { 'content-type' : cache [ requestUrl ] . mimetype } )
155
170
}
156
- body = snapshot . dom . html
171
+ body = cache [ requestUrl ] . body ;
157
172
} else {
158
173
response = await page . request . fetch ( request , requestOptions ) ;
159
174
body = await response . body ( ) ;
@@ -162,7 +177,7 @@ async function processSnapshot(snapshot: Snapshot, ctx: Context): Promise<Record
162
177
// handle response
163
178
if ( ! body ) {
164
179
ctx . log . debug ( `Handling request ${ requestUrl } \n - skipping no response` ) ;
165
- } else if ( ! body . length ) {
180
+ } else if ( ! body . length ) {
166
181
ctx . log . debug ( `Handling request ${ requestUrl } \n - skipping empty response` ) ;
167
182
} else if ( requestUrl === snapshot . url ) {
168
183
ctx . log . debug ( `Handling request ${ requestUrl } \n - skipping root resource` ) ;
@@ -204,7 +219,7 @@ async function processSnapshot(snapshot: Snapshot, ctx: Context): Promise<Record
204
219
let ignoreOrSelectBoxes : string ;
205
220
if ( options && Object . keys ( options ) . length ) {
206
221
ctx . log . debug ( `Snapshot options: ${ JSON . stringify ( options ) } ` ) ;
207
-
222
+
208
223
const isNotAllEmpty = ( obj : Record < string , Array < string > > ) : boolean => {
209
224
for ( let key in obj ) if ( obj [ key ] ?. length ) return true ;
210
225
return false ;
@@ -240,7 +255,7 @@ async function processSnapshot(snapshot: Snapshot, ctx: Context): Promise<Record
240
255
selectors . push ( ...value ) ;
241
256
break ;
242
257
}
243
- }
258
+ }
244
259
}
245
260
}
246
261
@@ -261,14 +276,14 @@ async function processSnapshot(snapshot: Snapshot, ctx: Context): Promise<Record
261
276
// Update `previousDeviceType` to the current device type for comparison in the next iteration
262
277
previousDeviceType = device ;
263
278
264
- await page . setViewportSize ( { width : viewport . width , height : viewport . height || MIN_VIEWPORT_HEIGHT } ) ;
265
- ctx . log . debug ( `Page resized to ${ viewport . width } x${ viewport . height || MIN_VIEWPORT_HEIGHT } ` ) ;
266
-
279
+ await page . setViewportSize ( { width : viewport . width , height : viewport . height || MIN_VIEWPORT_HEIGHT } ) ;
280
+ ctx . log . debug ( `Page resized to ${ viewport . width } x${ viewport . height || MIN_VIEWPORT_HEIGHT } ` ) ;
281
+
267
282
// navigate to snapshot url once
268
283
if ( ! navigated ) {
269
284
try {
270
285
// domcontentloaded event is more reliable than load event
271
- await page . goto ( snapshot . url , { waitUntil : "domcontentloaded" } ) ;
286
+ await page . goto ( snapshot . url , { waitUntil : "domcontentloaded" } ) ;
272
287
// adding extra timeout since domcontentloaded event is fired pretty quickly
273
288
await new Promise ( r => setTimeout ( r , 1250 ) ) ;
274
289
if ( ctx . config . waitForTimeout ) await page . waitForTimeout ( ctx . config . waitForTimeout ) ;
@@ -278,7 +293,7 @@ async function processSnapshot(snapshot: Snapshot, ctx: Context): Promise<Record
278
293
ctx . log . debug ( `Navigation to discovery page failed; ${ error } ` )
279
294
throw new Error ( error . message )
280
295
}
281
-
296
+
282
297
}
283
298
if ( ctx . config . cliEnableJavaScript && fullPage ) await page . evaluate ( scrollToBottomAndBackToTop , { frequency : 100 , timing : ctx . config . scrollTime } ) ;
284
299
@@ -289,8 +304,6 @@ async function processSnapshot(snapshot: Snapshot, ctx: Context): Promise<Record
289
304
ctx . log . debug ( `Network idle failed due to ${ error } ` ) ;
290
305
}
291
306
292
- await new Promise ( r => setTimeout ( r , 1000 ) ) ;
293
-
294
307
// snapshot options
295
308
if ( processedOptions . element ) {
296
309
let l = await page . locator ( processedOptions . element ) . all ( )
@@ -323,18 +336,6 @@ async function processSnapshot(snapshot: Snapshot, ctx: Context): Promise<Record
323
336
}
324
337
}
325
338
326
- // add dom resources to cache
327
- if ( snapshot . dom . resources . length ) {
328
- for ( let resource of snapshot . dom . resources ) {
329
- // convert text/css content to base64
330
- let body = resource . mimetype == 'text/css' ? Buffer . from ( resource . content ) . toString ( 'base64' ) : resource . content ;
331
- cache [ resource . url ] = {
332
- body : body ,
333
- type : resource . mimetype
334
- }
335
- }
336
- }
337
-
338
339
return {
339
340
processedSnapshot : {
340
341
name : snapshot . name ,
0 commit comments