feat: propagate router route map to crawler & context request methods

When a typed router is used, the route map now also types the request inputs:

- handler context: `ctx.addRequests` and `ctx.enqueueLinks` require the
  `userData` shape matching the request's `label` (and reject unknown labels);
  this is driven by the router, so it works for every crawler type.
- crawler instance: `Routes` is inferred from the `requestHandler` option and
  used to type `crawler.addRequests`/`crawler.run` for the HTTP-based crawlers
  (Basic/Http/Cheerio/JSDOM/LinkeDOM).

Unlabeled requests keep loose `userData` (they hit the default handler). All
typing is backwards compatible via the open-map default.

Note: crawler-instance `addRequests` typing is not yet wired for the browser
crawlers (Playwright/Puppeteer/Stagehand) — their `requestHandler` is redefined
in BrowserCrawlerOptions which breaks generic inference through the hierarchy;
their handler-context methods are still fully typed via the router.

Author: B4nan

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -28,6 +28,7 @@ import type {
     StatisticsOptions,
     StatisticState,
     StorageIdentifier,
+    TypedRequestsLike,
 } from '@crawlee/core';
 import {
     AutoscaledPool,
@@ -112,7 +113,7 @@ export type ErrorHandler<
 
 export interface StatusMessageCallbackParams<
     Context extends CrawlingContext = BasicCrawlingContext,
-    Crawler extends BasicCrawler<any> = BasicCrawler<Context>,
+    Crawler extends BasicCrawler<any, any, any, any> = BasicCrawler<Context>,
 > {
     state: StatisticState;
     crawler: Crawler;
@@ -122,7 +123,7 @@ export interface StatusMessageCallbackParams<
 
 export type StatusMessageCallback<
     Context extends CrawlingContext = BasicCrawlingContext,
-    Crawler extends BasicCrawler<any> = BasicCrawler<Context>,
+    Crawler extends BasicCrawler<any, any, any, any> = BasicCrawler<Context>,
 > = (params: StatusMessageCallbackParams<Context, Crawler>) => Awaitable<void>;
 
 export type RequireContextPipeline<
@@ -136,6 +137,7 @@ export interface BasicCrawlerOptions<
     Context extends CrawlingContext = CrawlingContext,
     ContextExtension = Dictionary<never>,
     ExtendedContext extends Context = Context & ContextExtension,
+    Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,
 > {
     /**
      * User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
@@ -154,7 +156,7 @@ export interface BasicCrawlerOptions<
      * The exceptions are logged to the request using the
      * {@apilink Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
      */
-    requestHandler?: RequestHandler<ExtendedContext>;
+    requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
 
     /**
      * Allows the user to extend the crawling context passed to the request handler with custom functionality.
@@ -514,6 +516,7 @@ export class BasicCrawler<
     Context extends CrawlingContext = CrawlingContext,
     ContextExtension = Dictionary<never>,
     ExtendedContext extends Context = Context & ContextExtension,
+    Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,
 > {
     protected static readonly CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';
 
@@ -582,7 +585,10 @@ export class BasicCrawler<
      * Default {@apilink Router} instance that will be used if we don't specify any {@apilink BasicCrawlerOptions.requestHandler|`requestHandler`}.
      * See {@apilink Router.addHandler|`router.addHandler()`} and {@apilink Router.addDefaultHandler|`router.addDefaultHandler()`}.
      */
-    readonly router: RouterHandler<Context> = Router.create<Context>();
+    readonly router: RouterHandler<Context, Routes> = Router.create<Context>() as unknown as RouterHandler<
+        Context,
+        Routes
+    >;
 
     private _basicContextPipeline?: ContextPipeline<{ request: Request }, CrawlingContext>;
 
@@ -707,7 +713,7 @@ export class BasicCrawler<
      * All `BasicCrawler` parameters are passed via an options object.
      */
     constructor(
-        options: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> &
+        options: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes> &
             RequireContextPipeline<CrawlingContext, Context> = {} as any, // cast because the constructor logic handles missing `contextPipelineBuilder` - the type is just for DX
     ) {
         ow(options, 'BasicCrawlerOptions', ow.object.exactShape(BasicCrawler.optionsShape));
@@ -1265,7 +1271,7 @@ export class BasicCrawler<
      * @param [requests] The requests to add.
      * @param [options] Options for the request queue.
      */
-    async run(requests?: RequestsLike, options?: CrawlerRunOptions): Promise<FinalStatistics> {
+    async run(requests?: TypedRequestsLike<Routes>, options?: CrawlerRunOptions): Promise<FinalStatistics> {
         if (this.running) {
             throw new Error(
                 'This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.',
@@ -1536,7 +1542,7 @@ export class BasicCrawler<
      * @param options Options for the request queue
      */
     async addRequests(
-        requests: ReadonlyDeep<RequestsLike>,
+        requests: ReadonlyDeep<TypedRequestsLike<Routes>>,
         options: CrawlerAddRequestsOptions = {},
     ): Promise<CrawlerAddRequestsResult> {
         await this.getRequestManager();

packages/cheerio-crawler/src/internals/cheerio-crawler.ts

@@ -37,7 +37,8 @@ export interface CheerioCrawlerOptions<
     ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension,
     UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
     JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-> extends HttpCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {}
+    Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>,
+> extends HttpCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext, Routes> {}
 
 export type CheerioHook<
     UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
@@ -182,11 +183,15 @@ export type CheerioRequestHandler<
 export class CheerioCrawler<
     ContextExtension = Dictionary<never>,
     ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension,
-> extends HttpCrawler<CheerioCrawlingContext, ContextExtension, ExtendedContext> {
+    Routes extends Record<keyof Routes, Dictionary> = Record<
+        string,
+        GetUserDataFromRequest<CheerioCrawlingContext['request']>
+    >,
+> extends HttpCrawler<CheerioCrawlingContext, ContextExtension, ExtendedContext, Routes> {
     /**
      * All `CheerioCrawler` parameters are passed via an options object.
      */
-    constructor(options?: CheerioCrawlerOptions<ContextExtension, ExtendedContext>) {
+    constructor(options?: CheerioCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes>) {
         const { contextPipelineBuilder, ...rest } = options ?? {};
 
         super({

packages/core/src/crawlers/crawler_commons.ts

@@ -4,7 +4,7 @@ import type { ReadonlyDeep, SetRequired } from 'type-fest';
 import type { Configuration } from '../configuration.js';
 import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links.js';
 import type { CrawleeLogger } from '../log.js';
-import type { Request, Source } from '../request.js';
+import type { Request, RequestOptions, Source } from '../request.js';
 import type { Dataset } from '../storages/dataset.js';
 import { KeyValueStore, type RecordOptions } from '../storages/key_value_store.js';
 import type { RequestQueueOperationOptions } from '../storages/request_queue.js';
@@ -13,6 +13,73 @@ import type { StorageIdentifier } from '../storages/storage_instance_manager.js'
 /** @internal */
 export type IsAny<T> = 0 extends 1 & T ? true : false;
 
+/**
+ * A request input (URL string, request-options object, or {@apilink Request}) whose `userData` is typed
+ * according to its `label`, based on a router's route map.
+ *
+ * When the route map is open (the default `Record<string, ...>`), this is just the regular loose
+ * {@apilink Source} input. When the map declares concrete labels, providing a `label` requires the matching
+ * `userData` shape and rejects labels not present in the map; unlabeled requests keep loose `userData`.
+ */
+export type LabeledSource<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes
+    ? string | Source
+    :
+          | string
+          | Request
+          | ({ requestsFromUrl?: string; regex?: RegExp } & (
+                | {
+                      [Label in keyof Routes & string]: Omit<Partial<RequestOptions<Routes[Label]>>, 'label'> & {
+                          label: Label;
+                      };
+                  }[keyof Routes & string]
+                | (Omit<Partial<RequestOptions>, 'label'> & { label?: undefined })
+            ));
+
+/**
+ * The iterable/array of {@apilink LabeledSource} inputs accepted by the label-aware `addRequests`/`run`
+ * methods of a crawler bound to a typed router.
+ */
+export type TypedRequestsLike<Routes extends Record<keyof Routes, Dictionary>> =
+    | AsyncIterable<LabeledSource<Routes>>
+    | Iterable<LabeledSource<Routes>>
+    | LabeledSource<Routes>[];
+
+/**
+ * The label-aware `addRequests` method signature exposed on a request handler's context when the crawler is
+ * bound to a typed router. Mirrors {@apilink RestrictedCrawlingContext.addRequests} with typed sources.
+ */
+export type TypedContextAddRequests<Routes extends Record<keyof Routes, Dictionary>> = (
+    requestsLike: ReadonlyDeep<LabeledSource<Routes>[]>,
+    options?: ReadonlyDeep<RequestQueueOperationOptions>,
+) => Promise<void>;
+
+/**
+ * An `enqueueLinks`-options object with its `label`/`userData` retyped according to a router's route map: a
+ * declared `label` requires the matching `userData` shape (unknown labels are rejected), while unlabeled
+ * calls keep loose `userData`. Returns the options unchanged when the route map is open (the default).
+ */
+type TypedEnqueueLinksOptions<Options, Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes
+    ? Options
+    : Omit<Options, 'label' | 'userData'> &
+          (
+              | { [Label in keyof Routes & string]: { label: Label; userData?: Routes[Label] } }[keyof Routes & string]
+              | { label?: undefined; userData?: Dictionary }
+          );
+
+/**
+ * Transforms a context's existing `enqueueLinks` method so that the `label`/`userData` in its options follow
+ * the router's route map, while preserving everything else about the signature (argument optionality and
+ * return type, which differ between crawler types).
+ */
+export type TypedContextEnqueueLinks<
+    EnqueueLinks,
+    Routes extends Record<keyof Routes, Dictionary>,
+> = EnqueueLinks extends (options?: infer Options) => infer Result
+    ? (options?: TypedEnqueueLinksOptions<Options, Routes>) => Result
+    : EnqueueLinks extends (options: infer Options) => infer Result
+      ? (options: TypedEnqueueLinksOptions<Options, Routes>) => Result
+      : EnqueueLinks;
+
 /** @internal */
 export type WithRequired<T, K extends keyof T> = T & { [P in K]-?: T[P] };
 

packages/core/src/router.ts

@@ -1,19 +1,34 @@
 import type { Dictionary } from '@crawlee/types';
 import type { StandardSchemaV1 } from '@standard-schema/spec';
 
-import type { CrawlingContext, LoadedRequest, RestrictedCrawlingContext } from './crawlers/crawler_commons.js';
+import type {
+    CrawlingContext,
+    LoadedRequest,
+    RestrictedCrawlingContext,
+    TypedContextAddRequests,
+    TypedContextEnqueueLinks,
+} from './crawlers/crawler_commons.js';
 import { MissingRouteError, RequestValidationError } from './errors.js';
 import type { Request } from './request.js';
 import type { Awaitable } from './typedefs.js';
 
 const defaultRoute = Symbol('default-route');
 
 /**
- * The crawling context received by a route handler, with `request.userData` narrowed to `UserData`.
+ * The crawling context received by a route handler, with `request.userData` narrowed to `UserData`, and
+ * `addRequests`/`enqueueLinks` typed according to the router's route map (`Routes`) so that enqueuing a
+ * request under a declared label requires the matching `userData` shape.
  */
-export type RouterHandlerContext<Context, UserData extends Dictionary> = Omit<Context, 'request'> & {
+export type RouterHandlerContext<
+    Context,
+    UserData extends Dictionary,
+    Routes extends Record<keyof Routes, Dictionary>,
+> = Omit<Context, 'request' | 'addRequests' | 'enqueueLinks'> & {
     request: LoadedRequest<Request<UserData>>;
-};
+    addRequests: TypedContextAddRequests<Routes>;
+} & (Context extends { enqueueLinks: infer EnqueueLinks }
+        ? { enqueueLinks: TypedContextEnqueueLinks<EnqueueLinks, Routes> }
+        : {});
 
 /**
  * The set of labels accepted by {@apilink Router.addHandler}. When the router declares a concrete
@@ -185,7 +200,7 @@ export class Router<
      */
     addHandler<Label extends keyof Routes & string>(
         label: Label,
-        handler: (ctx: RouterHandlerContext<Context, Routes[Label]>) => Awaitable<void>,
+        handler: (ctx: RouterHandlerContext<Context, Routes[Label], Routes>) => Awaitable<void>,
     ): void;
 
     /**
@@ -195,7 +210,7 @@ export class Router<
      */
     addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(
         label: RouterLabel<Routes>,
-        handler: (ctx: RouterHandlerContext<Context, UserData>) => Awaitable<void>,
+        handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>,
     ): void;
 
     addHandler(label: string | symbol, handler: (ctx: any) => Awaitable<void>): void {
@@ -209,7 +224,7 @@ export class Router<
      * (loosely typed by default). Pass an explicit `UserData` type argument to narrow it.
      */
     addDefaultHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(
-        handler: (ctx: RouterHandlerContext<Context, UserData>) => Awaitable<void>,
+        handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>,
     ) {
         this.validate(defaultRoute);
         this.routes.set(defaultRoute, handler);

packages/http-crawler/src/internals/http-crawler.ts

@@ -64,7 +64,8 @@ export interface HttpCrawlerOptions<
     Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext,
     ContextExtension = Dictionary<never>,
     ExtendedContext extends Context = Context & ContextExtension,
-> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> {
+    Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,
+> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes> {
     /**
      * Timeout in which the HTTP request to the resource needs to finish, given in seconds.
      */
@@ -315,7 +316,8 @@ export class HttpCrawler<
     Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext,
     ContextExtension = Dictionary<never>,
     ExtendedContext extends Context = Context & ContextExtension,
-> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {
+    Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,
+> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes> {
     protected preNavigationHooks: InternalHttpHook<CrawlingContext>[];
     protected postNavigationHooks: ((
         crawlingContext: CrawlingContextWithResponse,

packages/jsdom-crawler/src/internals/jsdom-crawler.ts

@@ -41,7 +41,8 @@ export interface JSDOMCrawlerOptions<
     ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension,
     UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
     JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {
+    Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>,
+> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext, Routes> {
     /**
      * Download and run scripts.
      */
@@ -186,7 +187,11 @@ const resources = new ResourceLoader({
 export class JSDOMCrawler<
     ContextExtension = Dictionary<never>,
     ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension,
-> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext> {
+    Routes extends Record<keyof Routes, Dictionary> = Record<
+        string,
+        GetUserDataFromRequest<JSDOMCrawlingContext['request']>
+    >,
+> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext, Routes> {
     protected static override optionsShape = {
         ...HttpCrawler.optionsShape,
         runScripts: ow.optional.boolean,
@@ -197,7 +202,7 @@ export class JSDOMCrawler<
     protected hideInternalConsole: boolean;
     protected virtualConsole: VirtualConsole | null = null;
 
-    constructor(options: JSDOMCrawlerOptions<ContextExtension, ExtendedContext> = {}) {
+    constructor(options: JSDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes> = {}) {
         const { runScripts = false, hideInternalConsole = false, contextPipelineBuilder, ...httpOptions } = options;
 
         super({

packages/linkedom-crawler/src/internals/linkedom-crawler.ts

@@ -37,7 +37,8 @@ export interface LinkeDOMCrawlerOptions<
     ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension,
     UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
     JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
-> extends HttpCrawlerOptions<LinkeDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {}
+    Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>,
+> extends HttpCrawlerOptions<LinkeDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext, Routes> {}
 
 export interface LinkeDOMCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptions, 'urls' | 'requestManager'> {}
 
@@ -169,10 +170,14 @@ export type LinkeDOMRequestHandler<
 export class LinkeDOMCrawler<
     ContextExtension = Dictionary<never>,
     ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension,
-> extends HttpCrawler<LinkeDOMCrawlingContext, ContextExtension, ExtendedContext> {
+    Routes extends Record<keyof Routes, Dictionary> = Record<
+        string,
+        GetUserDataFromRequest<LinkeDOMCrawlingContext['request']>
+    >,
+> extends HttpCrawler<LinkeDOMCrawlingContext, ContextExtension, ExtendedContext, Routes> {
     private static parser = new DOMParser();
 
-    constructor(options: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext>) {
+    constructor(options: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes>) {
         const { contextPipelineBuilder, ...rest } = options;
 
         super({