Skip to content

Commit 9311c7e

Browse files
macieklamberskimacieklamberski
committed
test: Add testing suite for custom fetch implementation
1 parent e466bfa commit 9311c7e

File tree

5 files changed

+191
-28
lines changed

5 files changed

+191
-28
lines changed

‎src/types/index.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,12 @@ export type RequestOptions = {
1010
timeout?: number
1111
}
1212

13-
export type CustomFetchFn<TCustomResponse> = (
13+
export type CustomFetchFn<TCustomResponse = unknown> = (
1414
url: string,
1515
options: RequestOptions,
1616
) => Promise<TCustomResponse>
1717

18-
export type CustomConfig<TCustomResponse> = {
18+
export type CustomConfig<TCustomResponse = unknown> = {
1919
fn: CustomFetchFn<TCustomResponse>
2020
}
2121

@@ -26,7 +26,7 @@ export type ScrapeStrategy = {
2626
useProxy?: boolean
2727
}
2828

29-
export type ScrapeConfig<TCustomResponse> = {
29+
export type ScrapeConfig<TCustomResponse = unknown> = {
3030
options?: ScrapeOptions<TCustomResponse>
3131
browser?: BrowserConfig
3232
custom?: CustomConfig<TCustomResponse>

‎src/types/result.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,12 @@ export type ScrapeResultBrowser = {
1414
cleanup: () => Promise<void>
1515
}
1616

17-
export type ScrapeResultCustom<TCustomResponse> = {
17+
export type ScrapeResultCustom<TCustomResponse = unknown> = {
1818
mechanism: 'custom'
1919
response: TCustomResponse
2020
}
2121

22-
export type ScrapeResult<TCustomResponse> =
22+
export type ScrapeResult<TCustomResponse = unknown> =
2323
| ScrapeResultFetch
2424
| ScrapeResultBrowser
2525
| ScrapeResultCustom<TCustomResponse>

‎src/types/validate.ts‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
import type { Response as PlaywrightResponse } from 'playwright'
22

3-
export type ValidateResponse<TCustomResponse> = (
4-
context:
5-
| { mechanism: 'fetch'; response: Response }
6-
| { mechanism: 'browser'; response: PlaywrightResponse }
7-
| { mechanism: 'custom'; response: TCustomResponse },
3+
export type ValidateResponseContext<TCustomResponse = unknown> =
4+
| { mechanism: 'fetch'; response: Response }
5+
| { mechanism: 'browser'; response: PlaywrightResponse }
6+
| { mechanism: 'custom'; response: TCustomResponse }
7+
8+
export type ValidateResponse<TCustomResponse = unknown> = (
9+
context: ValidateResponseContext<TCustomResponse>,
810
) => boolean

‎src/utils/strategy.test.ts‎

Lines changed: 174 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import { describe, expect, mock, test } from 'bun:test'
2-
import { calculateRetryDelay, getRandomFrom, withRetry } from './strategy.js'
2+
import type { RequestOptions, ScrapeConfig } from '../types/index.js'
3+
import type { ValidateResponseContext } from '../types/validate.js'
4+
import { calculateRetryDelay, executeCustomRequest, getRandomFrom, withRetry } from './strategy.js'
35

46
describe('calculateRetryDelay', () => {
57
describe('exponential backoff', () => {
@@ -284,24 +286,183 @@ describe('executeBrowserRequest', () => {
284286

285287
describe('executeCustomRequest', () => {
286288
describe('custom fetch function', () => {
287-
// TODO: should throw error when custom fetch not provided
288-
// TODO: should execute custom fetch function
289-
// TODO: should pass url to custom fetch
290-
// TODO: should pass options to custom fetch
291-
// TODO: should return custom response
289+
test('should throw error when custom fetch not provided', async () => {
290+
const config: ScrapeConfig = {}
291+
const options: RequestOptions = {}
292+
293+
const resultFn = () => executeCustomRequest('https://example.com', config, options)
294+
295+
expect(resultFn()).rejects.toThrow('Custom fetch function not provided')
296+
})
297+
298+
test('should execute custom fetch function', async () => {
299+
const mockFn = mock(async () => ({ data: 'test' }))
300+
const config: ScrapeConfig = { custom: { fn: mockFn } }
301+
const options: RequestOptions = {}
302+
303+
await executeCustomRequest('https://example.com', config, options)
304+
305+
expect(mockFn).toHaveBeenCalledTimes(1)
306+
})
307+
308+
test('should pass url to custom fetch', async () => {
309+
let capturedUrl: string | undefined
310+
const config: ScrapeConfig = {
311+
custom: {
312+
fn: async (url) => {
313+
capturedUrl = url
314+
return { data: 'test' }
315+
},
316+
},
317+
}
318+
const options: RequestOptions = {}
319+
320+
await executeCustomRequest('https://example.com/test', config, options)
321+
322+
expect(capturedUrl).toBe('https://example.com/test')
323+
})
324+
325+
test('should pass options to custom fetch', async () => {
326+
let capturedOptions: RequestOptions | undefined
327+
const config: ScrapeConfig = {
328+
custom: {
329+
fn: async (_url, options) => {
330+
capturedOptions = options
331+
return { data: 'test' }
332+
},
333+
},
334+
}
335+
const options: RequestOptions = {
336+
headers: { 'X-Test': 'value' },
337+
timeout: 5000,
338+
proxy: 'http://proxy.com:8080',
339+
}
340+
341+
await executeCustomRequest('https://example.com', config, options)
342+
343+
expect(capturedOptions).toEqual({
344+
headers: { 'X-Test': 'value' },
345+
timeout: 5000,
346+
proxy: 'http://proxy.com:8080',
347+
})
348+
})
349+
350+
test('should return custom response', async () => {
351+
const customResponse = { data: 'test', count: 42 }
352+
const config: ScrapeConfig = {
353+
custom: { fn: async () => customResponse },
354+
}
355+
const options: RequestOptions = {}
356+
357+
const result = await executeCustomRequest('https://example.com', config, options)
358+
359+
expect(result.mechanism).toBe('custom')
360+
expect(result.response).toEqual(customResponse)
361+
})
292362
})
293363

294364
describe('validation', () => {
295-
// TODO: should validate response when validator provided
296-
// TODO: should pass mechanism and response to validator
297-
// TODO: should throw error when validation fails
298-
// TODO: should skip validation when validator not provided
365+
test('should validate response when validator provided', async () => {
366+
const mockValidator = mock(() => true)
367+
const config: ScrapeConfig = {
368+
custom: { fn: async () => ({ status: 'ok' }) },
369+
options: { validateResponse: mockValidator },
370+
}
371+
const options: RequestOptions = {}
372+
373+
await executeCustomRequest('https://example.com', config, options)
374+
375+
expect(mockValidator).toHaveBeenCalledTimes(1)
376+
})
377+
378+
test('should pass mechanism and response to validator', async () => {
379+
let capturedContext: ValidateResponseContext | undefined
380+
const customResponse = { status: 'ok' }
381+
const config: ScrapeConfig = {
382+
custom: { fn: async () => customResponse },
383+
options: {
384+
validateResponse: (context) => {
385+
capturedContext = context
386+
return true
387+
},
388+
},
389+
}
390+
const options: RequestOptions = {}
391+
392+
await executeCustomRequest('https://example.com', config, options)
393+
394+
expect(capturedContext?.mechanism).toBe('custom')
395+
expect(capturedContext?.response).toEqual(customResponse)
396+
})
397+
398+
test('should throw error when validation fails', async () => {
399+
const config: ScrapeConfig = {
400+
custom: { fn: async () => ({ status: 'error' }) },
401+
options: {
402+
validateResponse: () => false,
403+
},
404+
}
405+
const options: RequestOptions = {}
406+
407+
const resultFn = () => executeCustomRequest('https://example.com', config, options)
408+
409+
expect(resultFn()).rejects.toThrow('Response validation failed')
410+
})
411+
412+
test('should skip validation when validator not provided', async () => {
413+
const config: ScrapeConfig = {
414+
custom: { fn: async () => ({ data: 'test' }) },
415+
}
416+
const options: RequestOptions = {}
417+
418+
const result = await executeCustomRequest('https://example.com', config, options)
419+
420+
expect(result.mechanism).toBe('custom')
421+
expect(result.response).toEqual({ data: 'test' })
422+
})
299423
})
300424

301425
describe('error handling', () => {
302-
// TODO: should throw error when response is null
303-
// TODO: should propagate custom fetch errors
304-
// TODO: should handle validation errors
426+
test('should throw error when response is null', async () => {
427+
const config: ScrapeConfig = {
428+
custom: { fn: async () => null },
429+
}
430+
const options: RequestOptions = {}
431+
const resultFn = () => executeCustomRequest('https://example.com', config, options)
432+
433+
expect(resultFn()).rejects.toThrow('No response received')
434+
})
435+
436+
test('should propagate custom fetch errors', async () => {
437+
const config: ScrapeConfig = {
438+
custom: {
439+
fn: async () => {
440+
throw new Error('Custom fetch failed')
441+
},
442+
},
443+
}
444+
const options: RequestOptions = {}
445+
446+
const resultFn = () => executeCustomRequest('https://example.com', config, options)
447+
448+
expect(resultFn()).rejects.toThrow('Custom fetch failed')
449+
})
450+
451+
test('should handle validation errors', async () => {
452+
const config: ScrapeConfig = {
453+
custom: { fn: async () => ({ data: 'test' }) },
454+
options: {
455+
validateResponse: () => {
456+
throw new Error('Validation error')
457+
},
458+
},
459+
}
460+
const options: RequestOptions = {}
461+
462+
const resultFn = () => executeCustomRequest('https://example.com', config, options)
463+
464+
expect(resultFn()).rejects.toThrow('Validation error')
465+
})
305466
})
306467
})
307468

‎src/utils/strategy.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ export const withRetry = async <T>(fn: () => Promise<T>, retryConfig?: RetryConf
6969
throw lastError
7070
}
7171

72-
const executeFetchRequest = async <TCustomResponse>(
72+
const executeFetchRequest = async <TCustomResponse = unknown>(
7373
url: string,
7474
config: ScrapeConfig<TCustomResponse>,
7575
options: RequestOptions,
@@ -109,7 +109,7 @@ const executeFetchRequest = async <TCustomResponse>(
109109
}
110110
}
111111

112-
const executeBrowserRequest = async <TCustomResponse>(
112+
const executeBrowserRequest = async <TCustomResponse = unknown>(
113113
url: string,
114114
config: ScrapeConfig<TCustomResponse>,
115115
options: RequestOptions,
@@ -149,7 +149,7 @@ const executeBrowserRequest = async <TCustomResponse>(
149149
}
150150
}
151151

152-
const executeCustomRequest = async <TCustomResponse>(
152+
export const executeCustomRequest = async <TCustomResponse = unknown>(
153153
url: string,
154154
config: ScrapeConfig<TCustomResponse>,
155155
options: RequestOptions,
@@ -173,7 +173,7 @@ const executeCustomRequest = async <TCustomResponse>(
173173
return { mechanism: 'custom', response }
174174
}
175175

176-
const executeRequest = async <TCustomResponse>(
176+
const executeRequest = async <TCustomResponse = unknown>(
177177
url: string,
178178
config: ScrapeConfig<TCustomResponse>,
179179
strategy: ScrapeStrategy,
@@ -190,7 +190,7 @@ const executeRequest = async <TCustomResponse>(
190190
return await executeCustomRequest(url, config, options)
191191
}
192192

193-
export const executeStrategy = async <TCustomResponse>(
193+
export const executeStrategy = async <TCustomResponse = unknown>(
194194
url: string,
195195
config: ScrapeConfig<TCustomResponse>,
196196
strategy: ScrapeStrategy,

0 commit comments

Comments
 (0)