Skip to content

Commit 4b70a48

Browse files
committed
fix(core): align typed router with v3 PR review polish
- default handler keeps `request.userData` loosely typed (it is a fallback for any request, including labels not in the route map) - split factory/`Router.create` into explicit overloads (route map vs legacy flat userData) for backwards compatibility, keeping the schema overload - drop the exported `RouteMap` alias (referenced as prose in docs instead)
1 parent 1110f07 commit 4b70a48

13 files changed

Lines changed: 138 additions & 65 deletions

File tree

packages/basic-crawler/src/internals/basic-crawler.ts

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2349,9 +2349,12 @@ export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
23492349
*/
23502350
export function createBasicRouter<
23512351
Context extends BasicCrawlingContext = BasicCrawlingContext,
2352-
UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,
2353-
Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>,
2352+
Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,
23542353
>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
2354+
export function createBasicRouter<
2355+
Context extends BasicCrawlingContext = BasicCrawlingContext,
2356+
UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,
2357+
>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
23552358
export function createBasicRouter<
23562359
Context extends BasicCrawlingContext = BasicCrawlingContext,
23572360
const Schemas extends RouteSchemas = RouteSchemas,

packages/cheerio-crawler/src/internals/cheerio-crawler.ts

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -366,9 +366,12 @@ export async function cheerioCrawlerEnqueueLinks(
366366
*/
367367
export function createCheerioRouter<
368368
Context extends CheerioCrawlingContext = CheerioCrawlingContext,
369-
UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,
370-
Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>,
369+
Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,
371370
>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
371+
export function createCheerioRouter<
372+
Context extends CheerioCrawlingContext = CheerioCrawlingContext,
373+
UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,
374+
>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
372375
export function createCheerioRouter<
373376
Context extends CheerioCrawlingContext = CheerioCrawlingContext,
374377
const Schemas extends RouteSchemas = RouteSchemas,

packages/core/src/errors.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ export class CriticalError extends NonRetryableError {}
1717
export class MissingRouteError extends CriticalError {}
1818

1919
/**
20-
* Thrown when a request's `userData` does not match the {@apilink RouteMap} schema registered for its label.
20+
* Thrown when a request's `userData` does not match the {@apilink RouteSchemas|Standard Schema} registered for its label.
2121
*
2222
* As the `userData` does not change between attempts, this error is non-retryable.
2323
*/

packages/core/src/router.ts

Lines changed: 39 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -9,18 +9,21 @@ import type { Awaitable } from './typedefs.js';
99
const defaultRoute = Symbol('default-route');
1010

1111
/**
12-
* A map of request labels to the shape of `request.userData` expected for that label. Pass it as the
13-
* `Routes` type argument of {@apilink Router} (or a `createXRouter` factory) to get per-label typing of
14-
* `request.userData` and autocomplete/validation of labels in {@apilink Router.addHandler}.
15-
*
16-
* ```ts
17-
* interface MyRoutes {
18-
* PRODUCT: { sku: string; price: number };
19-
* CATEGORY: { categoryId: string };
20-
* }
21-
* ```
12+
* The crawling context received by a route handler, with `request.userData` narrowed to `UserData`.
13+
*/
14+
export type RouterHandlerContext<Context, UserData extends Dictionary> = Omit<Context, 'request'> & {
15+
request: LoadedRequest<Request<UserData>>;
16+
};
17+
18+
/**
19+
* The set of labels accepted by {@apilink Router.addHandler}. When the router declares a concrete
20+
* route map (e.g. `{ PRODUCT: ...; CATEGORY: ... }`), only those labels (plus symbols) are
21+
* allowed — unknown labels become a compile-time error. When the map is left open (the default
22+
* `Record<string, ...>`), any string or symbol label is accepted, preserving the original behaviour.
2223
*/
23-
export type RouteMap = Record<string, Dictionary>;
24+
export type RouterLabel<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes
25+
? string | symbol
26+
: (keyof Routes & string) | symbol;
2427

2528
/**
2629
* A map of request labels to a [Standard Schema](https://standardschema.dev) (Zod, Valibot, ArkType, …)
@@ -31,32 +34,15 @@ export type RouteMap = Record<string, Dictionary>;
3134
export type RouteSchemas = Record<string, StandardSchemaV1>;
3235

3336
/**
34-
* Derives a {@apilink RouteMap} (label → `userData` type) from a {@apilink RouteSchemas} map by inferring
35-
* each schema's output type. Outputs that are not object-shaped fall back to a plain {@apilink Dictionary}.
37+
* Derives a route map (label → `userData` type) from a {@apilink RouteSchemas} map by inferring each
38+
* schema's output type. Outputs that are not object-shaped fall back to a plain {@apilink Dictionary}.
3639
*/
3740
export type RoutesFromSchemas<Schemas extends RouteSchemas> = {
3841
[Label in keyof Schemas]: StandardSchemaV1.InferOutput<Schemas[Label]> extends Dictionary
3942
? StandardSchemaV1.InferOutput<Schemas[Label]>
4043
: Dictionary;
4144
};
4245

43-
/**
44-
* The crawling context received by a route handler, with `request.userData` narrowed to `UserData`.
45-
*/
46-
export type RouterHandlerContext<Context, UserData extends Dictionary> = Omit<Context, 'request'> & {
47-
request: LoadedRequest<Request<UserData>>;
48-
};
49-
50-
/**
51-
* The set of labels accepted by {@apilink Router.addHandler}. When the router declares a concrete
52-
* {@apilink RouteMap} (e.g. `{ PRODUCT: ...; CATEGORY: ... }`), only those labels (plus symbols) are
53-
* allowed — unknown labels become a compile-time error. When the map is left open (the default
54-
* `Record<string, ...>`), any string or symbol label is accepted, preserving the original behaviour.
55-
*/
56-
export type RouterLabel<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes
57-
? string | symbol
58-
: (keyof Routes & string) | symbol;
59-
6046
export interface RouterHandler<
6147
Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext,
6248
Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,
@@ -137,9 +123,9 @@ export type RouterRoutes<Context, Routes extends Record<keyof Routes, Dictionary
137123
*
138124
* ## Typed labels
139125
*
140-
* To get `request.userData` typed per label, declare a {@apilink RouteMap} and pass it as the second
141-
* type argument. The label passed to {@apilink Router.addHandler} then drives the type of
142-
* `request.userData`, and unknown labels are rejected at compile time:
126+
* To get `request.userData` typed per label, declare a route map and pass it as the second type
127+
* argument. The label passed to {@apilink Router.addHandler} then drives the type of `request.userData`,
128+
* and unknown labels are rejected at compile time:
143129
*
144130
* ```ts
145131
* import { createCheerioRouter, CheerioCrawlingContext } from 'crawlee';
@@ -194,7 +180,7 @@ export class Router<
194180
protected constructor() {}
195181

196182
/**
197-
* Registers new route handler for given label. When the router declares a {@apilink RouteMap}, the
183+
* Registers new route handler for given label. When the router declares a route map, the
198184
* `label` is restricted to the declared labels and `request.userData` is typed accordingly.
199185
*/
200186
addHandler<Label extends keyof Routes & string>(
@@ -203,8 +189,9 @@ export class Router<
203189
): void;
204190

205191
/**
206-
* Registers new route handler for given label, with an explicit `request.userData` type. Use this
207-
* overload to type a handler whose label is not part of the router's {@apilink RouteMap}.
192+
* Registers new route handler for given label, explicitly typing `request.userData` via the
193+
* `UserData` type argument. Useful when the router has no declared route map (the open default)
194+
* and you want to type a single handler, or to register a handler under a `symbol` label.
208195
*/
209196
addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(
210197
label: RouterLabel<Routes>,
@@ -217,10 +204,11 @@ export class Router<
217204
}
218205

219206
/**
220-
* Registers default route handler. By default `request.userData` is typed as the union of all
221-
* `userData` shapes declared in the router's {@apilink RouteMap}.
207+
* Registers default route handler. As a fallback it can receive any request (including labels not
208+
* declared in the route map), so `request.userData` defaults to the context's `userData` type
209+
* (loosely typed by default). Pass an explicit `UserData` type argument to narrow it.
222210
*/
223-
addDefaultHandler<UserData extends Dictionary = Routes[keyof Routes]>(
211+
addDefaultHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(
224212
handler: (ctx: RouterHandlerContext<Context, UserData>) => Awaitable<void>,
225213
) {
226214
this.validate(defaultRoute);
@@ -332,12 +320,21 @@ export class Router<
332320
* });
333321
* ```
334322
*/
323+
// The handler overloads keep the second type argument backwards compatible. When it is a route map
324+
// (every value is a `Dictionary`) the first overload applies and labels are typed per route. Otherwise
325+
// it fails the `Record<keyof Routes, Dictionary>` constraint and falls through to the second overload,
326+
// where it is treated as the legacy flat `userData` shape shared by all handlers. The third overload
327+
// accepts a Standard Schema per label, inferring the route map and validating `userData` at runtime.
335328
static create<
336329
Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext,
337-
UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,
338-
Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>,
330+
Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,
339331
>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
340332

333+
static create<
334+
Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext,
335+
UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,
336+
>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
337+
341338
static create<
342339
Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext,
343340
const Schemas extends RouteSchemas = RouteSchemas,
@@ -357,7 +354,7 @@ export class Router<
357354

358355
for (const [label, value] of Object.entries(routesOrSchemas ?? {})) {
359356
if (typeof value === 'function') {
360-
router.addHandler(label as keyof Context & string, value as (ctx: any) => Awaitable<void>);
357+
router.addHandler(label, value as (ctx: any) => Awaitable<void>);
361358
} else {
362359
router.schemas.set(label, value);
363360
}

packages/http-crawler/src/internals/file-download.ts

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -262,9 +262,12 @@ function trackBodyConsumption(response: Response): { response: ResponseWithUrl;
262262
*/
263263
export function createFileRouter<
264264
Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext,
265-
UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,
266-
Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>,
265+
Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,
267266
>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
267+
export function createFileRouter<
268+
Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext,
269+
UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,
270+
>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
268271
export function createFileRouter<
269272
Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext,
270273
const Schemas extends RouteSchemas = RouteSchemas,

packages/http-crawler/src/internals/http-crawler.ts

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -843,9 +843,12 @@ interface RequestFunctionOptions {
843843
*/
844844
export function createHttpRouter<
845845
Context extends HttpCrawlingContext = HttpCrawlingContext,
846-
UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,
847-
Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>,
846+
Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,
848847
>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
848+
export function createHttpRouter<
849+
Context extends HttpCrawlingContext = HttpCrawlingContext,
850+
UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,
851+
>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
849852
export function createHttpRouter<
850853
Context extends HttpCrawlingContext = HttpCrawlingContext,
851854
const Schemas extends RouteSchemas = RouteSchemas,

packages/jsdom-crawler/src/internals/jsdom-crawler.ts

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -497,9 +497,12 @@ function extractUrlsFromWindow(window: DOMWindow, selector: string, baseUrl: str
497497
*/
498498
export function createJSDOMRouter<
499499
Context extends JSDOMCrawlingContext = JSDOMCrawlingContext,
500-
UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,
501-
Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>,
500+
Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,
502501
>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
502+
export function createJSDOMRouter<
503+
Context extends JSDOMCrawlingContext = JSDOMCrawlingContext,
504+
UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,
505+
>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
503506
export function createJSDOMRouter<
504507
Context extends JSDOMCrawlingContext = JSDOMCrawlingContext,
505508
const Schemas extends RouteSchemas = RouteSchemas,

packages/linkedom-crawler/src/internals/linkedom-crawler.ts

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -387,9 +387,12 @@ function extractUrlsFromWindow(window: Window, selector: string, baseUrl: string
387387
*/
388388
export function createLinkeDOMRouter<
389389
Context extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext,
390-
UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,
391-
Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>,
390+
Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,
392391
>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
392+
export function createLinkeDOMRouter<
393+
Context extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext,
394+
UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,
395+
>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
393396
export function createLinkeDOMRouter<
394397
Context extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext,
395398
const Schemas extends RouteSchemas = RouteSchemas,

packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -784,9 +784,12 @@ export class AdaptivePlaywrightCrawler<
784784

785785
export function createAdaptivePlaywrightRouter<
786786
Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext,
787-
UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,
788-
Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>,
787+
Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,
789788
>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
789+
export function createAdaptivePlaywrightRouter<
790+
Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext,
791+
UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,
792+
>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
790793
export function createAdaptivePlaywrightRouter<
791794
Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext,
792795
const Schemas extends RouteSchemas = RouteSchemas,

packages/playwright-crawler/src/internals/playwright-crawler.ts

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -349,9 +349,12 @@ export function handleCloudflareChallengeHook(options?: HandleCloudflareChalleng
349349
*/
350350
export function createPlaywrightRouter<
351351
Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext,
352-
UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,
353-
Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>,
352+
Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,
354353
>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
354+
export function createPlaywrightRouter<
355+
Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext,
356+
UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,
357+
>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
355358
export function createPlaywrightRouter<
356359
Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext,
357360
const Schemas extends RouteSchemas = RouteSchemas,

0 commit comments

Comments
 (0)