feat: schema-validated router labels via standard-schema

Introduce per-label typing of `request.userData` for the router, in two
layers:

- A `label -> userData` map can be passed as the router's `Routes` type
  argument, typing `request.userData` per label and rejecting unknown
  labels at compile time. Backwards compatible (default is an open map).
- A per-label Standard Schema map (Zod, Valibot, ArkType, …) passed to
  `Router.create`/`createXRouter` both infers the `userData` types and
  validates them at runtime before the handler runs, replacing
  `request.userData` with the parsed value. Invalid requests throw a new
  non-retryable `RequestValidationError`.

Adds the types-only `@standard-schema/spec` dependency to `@crawlee/core`.

Relates to #3082

Author: B4nan

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -21,6 +21,8 @@ import type {
     RequestTransform,
     RouterHandler,
     RouterRoutes,
+    RouteSchemas,
+    RoutesFromSchemas,
     SkippedRequestCallback,
     Source,
     StatisticsOptions,
@@ -2348,6 +2350,12 @@ export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
 export function createBasicRouter<
     Context extends BasicCrawlingContext = BasicCrawlingContext,
     UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,
->(routes?: RouterRoutes<Context, UserData>) {
-    return Router.create<Context>(routes);
+    Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>,
+>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+export function createBasicRouter<
+    Context extends BasicCrawlingContext = BasicCrawlingContext,
+    const Schemas extends RouteSchemas = RouteSchemas,
+>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
+export function createBasicRouter(routesOrSchemas?: any): any {
+    return Router.create(routesOrSchemas);
 }

packages/cheerio-crawler/src/internals/cheerio-crawler.ts

@@ -8,7 +8,10 @@ import type {
     InternalHttpHook,
     IRequestManager,
     RequestHandler,
+    RouterHandler,
     RouterRoutes,
+    RouteSchemas,
+    RoutesFromSchemas,
     SkippedRequestCallback,
 } from '@crawlee/http';
 import {
@@ -364,6 +367,12 @@ export async function cheerioCrawlerEnqueueLinks(
 export function createCheerioRouter<
     Context extends CheerioCrawlingContext = CheerioCrawlingContext,
     UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,
->(routes?: RouterRoutes<Context, UserData>) {
-    return Router.create<Context>(routes);
+    Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>,
+>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+export function createCheerioRouter<
+    Context extends CheerioCrawlingContext = CheerioCrawlingContext,
+    const Schemas extends RouteSchemas = RouteSchemas,
+>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
+export function createCheerioRouter(routesOrSchemas?: any): any {
+    return Router.create(routesOrSchemas);
 }

packages/core/package.json

@@ -57,6 +57,7 @@
         "@crawlee/types": "workspace:*",
         "@crawlee/utils": "workspace:*",
         "@sapphire/async-queue": "^1.5.5",
+        "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
         "csv-stringify": "^6.5.2",
         "json5": "^2.2.3",

packages/core/src/errors.ts

@@ -16,6 +16,32 @@ export class CriticalError extends NonRetryableError {}
  */
 export class MissingRouteError extends CriticalError {}
 
+/**
+ * Thrown when a request's `userData` does not match the {@apilink RouteMap} schema registered for its label.
+ *
+ * As the `userData` does not change between attempts, this error is non-retryable.
+ */
+export class RequestValidationError extends NonRetryableError {
+    constructor(
+        readonly label: string | symbol,
+        readonly issues: readonly {
+            readonly message: string;
+            readonly path?: readonly (PropertyKey | { key: PropertyKey })[];
+        }[],
+    ) {
+        const details = issues
+            .map((issue) => {
+                const path = (issue.path ?? [])
+                    .map((segment) => (typeof segment === 'object' ? segment.key : segment))
+                    .join('.');
+                return `- ${path ? `${path}: ` : ''}${issue.message}`;
+            })
+            .join('\n');
+
+        super(`Request userData for label '${String(label)}' failed schema validation:\n${details}`);
+    }
+}
+
 /**
  * Errors of `RetryRequestError` type will always be retried by the crawler.
  *

packages/core/src/router.ts

@@ -1,22 +1,73 @@
 import type { Dictionary } from '@crawlee/types';
+import type { StandardSchemaV1 } from '@standard-schema/spec';
 
 import type { CrawlingContext, LoadedRequest, RestrictedCrawlingContext } from './crawlers/crawler_commons.js';
-import { MissingRouteError } from './errors.js';
+import { MissingRouteError, RequestValidationError } from './errors.js';
 import type { Request } from './request.js';
 import type { Awaitable } from './typedefs.js';
 
 const defaultRoute = Symbol('default-route');
 
+/**
+ * A map of request labels to the shape of `request.userData` expected for that label. Pass it as the
+ * `Routes` type argument of {@apilink Router} (or a `createXRouter` factory) to get per-label typing of
+ * `request.userData` and autocomplete/validation of labels in {@apilink Router.addHandler}.
+ *
+ * ```ts
+ * interface MyRoutes {
+ *     PRODUCT: { sku: string; price: number };
+ *     CATEGORY: { categoryId: string };
+ * }
+ * ```
+ */
+export type RouteMap = Record<string, Dictionary>;
+
+/**
+ * A map of request labels to a [Standard Schema](https://standardschema.dev) (Zod, Valibot, ArkType, …)
+ * validating that label's `request.userData`. Pass it to {@apilink Router.create} or a `createXRouter`
+ * factory to derive the per-label `request.userData` types *and* validate them at runtime before the
+ * matching handler runs.
+ */
+export type RouteSchemas = Record<string, StandardSchemaV1>;
+
+/**
+ * Derives a {@apilink RouteMap} (label → `userData` type) from a {@apilink RouteSchemas} map by inferring
+ * each schema's output type. Outputs that are not object-shaped fall back to a plain {@apilink Dictionary}.
+ */
+export type RoutesFromSchemas<Schemas extends RouteSchemas> = {
+    [Label in keyof Schemas]: StandardSchemaV1.InferOutput<Schemas[Label]> extends Dictionary
+        ? StandardSchemaV1.InferOutput<Schemas[Label]>
+        : Dictionary;
+};
+
+/**
+ * The crawling context received by a route handler, with `request.userData` narrowed to `UserData`.
+ */
+export type RouterHandlerContext<Context, UserData extends Dictionary> = Omit<Context, 'request'> & {
+    request: LoadedRequest<Request<UserData>>;
+};
+
+/**
+ * The set of labels accepted by {@apilink Router.addHandler}. When the router declares a concrete
+ * {@apilink RouteMap} (e.g. `{ PRODUCT: ...; CATEGORY: ... }`), only those labels (plus symbols) are
+ * allowed — unknown labels become a compile-time error. When the map is left open (the default
+ * `Record<string, ...>`), any string or symbol label is accepted, preserving the original behaviour.
+ */
+export type RouterLabel<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes
+    ? string | symbol
+    : (keyof Routes & string) | symbol;
+
 export interface RouterHandler<
     Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext,
-> extends Router<Context> {
+    Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,
+> extends Router<Context, Routes> {
     (ctx: Context): Awaitable<void>;
 }
 
 export type GetUserDataFromRequest<T> = T extends Request<infer Y> ? Y : never;
 
-export type RouterRoutes<Context, UserData extends Dictionary> = {
-    [label in string | symbol]: (ctx: Omit<Context, 'request'> & { request: Request<UserData> }) => Awaitable<void>;
+export type RouterRoutes<Context, Routes extends Record<keyof Routes, Dictionary>> = {
+    [Label in keyof Routes]: (ctx: Omit<Context, 'request'> & { request: Request<Routes[Label]> }) => Awaitable<void>;
 };
 
 /**
@@ -83,9 +134,57 @@ export type RouterRoutes<Context, UserData extends Dictionary> = {
  *    ctx.log.info('...');
  * });
  * ```
+ *
+ * ## Typed labels
+ *
+ * To get `request.userData` typed per label, declare a {@apilink RouteMap} and pass it as the second
+ * type argument. The label passed to {@apilink Router.addHandler} then drives the type of
+ * `request.userData`, and unknown labels are rejected at compile time:
+ *
+ * ```ts
+ * import { createCheerioRouter, CheerioCrawlingContext } from 'crawlee';
+ *
+ * interface Routes {
+ *     PRODUCT: { sku: string; price: number };
+ *     CATEGORY: { categoryId: string };
+ * }
+ *
+ * const router = createCheerioRouter<CheerioCrawlingContext, Routes>();
+ *
+ * router.addHandler('PRODUCT', async ({ request }) => {
+ *     request.userData.sku;   // string
+ *     request.userData.price; // number
+ * });
+ *
+ * router.addHandler('TYPO', async () => {}); // compile error: not a known label
+ * ```
+ *
+ * ## Schema-validated labels
+ *
+ * Passing a [Standard Schema](https://standardschema.dev) per label both infers the `request.userData`
+ * types *and* validates them at runtime before the handler runs (replacing `request.userData` with the
+ * parsed value). A failing request throws a {@apilink RequestValidationError}.
+ *
+ * ```ts
+ * import { z } from 'zod';
+ * import { createCheerioRouter } from 'crawlee';
+ *
+ * const router = createCheerioRouter({
+ *     PRODUCT: z.object({ sku: z.string(), price: z.number() }),
+ *     CATEGORY: z.object({ categoryId: z.string() }),
+ * });
+ *
+ * router.addHandler('PRODUCT', async ({ request }) => {
+ *     request.userData.price; // number, inferred from the schema and validated at runtime
+ * });
+ * ```
  */
-export class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'>> {
+export class Router<
+    Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'>,
+    Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>,
+> {
     private readonly routes: Map<string | symbol, (ctx: any) => Awaitable<void>> = new Map();
+    private readonly schemas: Map<string | symbol, StandardSchemaV1> = new Map();
     private readonly middlewares: ((ctx: Context) => Awaitable<void>)[] = [];
 
     /**
@@ -95,26 +194,52 @@ export class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLink
     protected constructor() {}
 
     /**
-     * Registers new route handler for given label.
+     * Registers new route handler for given label. When the router declares a {@apilink RouteMap}, the
+     * `label` is restricted to the declared labels and `request.userData` is typed accordingly.
+     */
+    addHandler<Label extends keyof Routes & string>(
+        label: Label,
+        handler: (ctx: RouterHandlerContext<Context, Routes[Label]>) => Awaitable<void>,
+    ): void;
+
+    /**
+     * Registers new route handler for given label, with an explicit `request.userData` type. Use this
+     * overload to type a handler whose label is not part of the router's {@apilink RouteMap}.
      */
     addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(
-        label: string | symbol,
-        handler: (ctx: Omit<Context, 'request'> & { request: LoadedRequest<Request<UserData>> }) => Awaitable<void>,
-    ) {
+        label: RouterLabel<Routes>,
+        handler: (ctx: RouterHandlerContext<Context, UserData>) => Awaitable<void>,
+    ): void;
+
+    addHandler(label: string | symbol, handler: (ctx: any) => Awaitable<void>): void {
         this.validate(label);
         this.routes.set(label, handler);
     }
 
     /**
-     * Registers default route handler.
+     * Registers default route handler. By default `request.userData` is typed as the union of all
+     * `userData` shapes declared in the router's {@apilink RouteMap}.
      */
-    addDefaultHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(
-        handler: (ctx: Omit<Context, 'request'> & { request: LoadedRequest<Request<UserData>> }) => Awaitable<void>,
+    addDefaultHandler<UserData extends Dictionary = Routes[keyof Routes]>(
+        handler: (ctx: RouterHandlerContext<Context, UserData>) => Awaitable<void>,
     ) {
         this.validate(defaultRoute);
         this.routes.set(defaultRoute, handler);
     }
 
+    /**
+     * Registers {@apilink RouteSchemas|Standard Schema} validators for the given labels. Before a matching
+     * route handler runs, `request.userData` is validated against the label's schema and replaced with the
+     * parsed value; a failing request throws a {@apilink RequestValidationError}.
+     */
+    addSchemas(schemas: Partial<Record<keyof Routes & string, StandardSchemaV1>>) {
+        for (const [label, schema] of Object.entries(schemas)) {
+            if (schema) {
+                this.schemas.set(label, schema as StandardSchemaV1);
+            }
+        }
+    }
+
     /**
      * Registers a middleware that will be fired before the matching route handler.
      * Multiple middlewares can be registered, they will be fired in the same order.
@@ -142,6 +267,27 @@ export class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLink
         );
     }
 
+    /**
+     * Validates `request.userData` against the schema registered for its label (if any), replacing it with
+     * the parsed value. Throws a {@apilink RequestValidationError} when validation fails.
+     */
+    private async validateRequest(context: Context) {
+        const { label } = context.request;
+        const schema = label != null ? this.schemas.get(label) : undefined;
+
+        if (!schema) {
+            return;
+        }
+
+        const result = await schema['~standard'].validate(context.request.userData);
+
+        if (result.issues) {
+            throw new RequestValidationError(label!, result.issues);
+        }
+
+        context.request.userData = result.value as Dictionary;
+    }
+
     /**
      * Throws when the label already exists in our registry.
      */
@@ -174,27 +320,55 @@ export class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLink
      * });
      * await crawler.run();
      * ```
+     *
+     * Passing a {@apilink RouteSchemas|Standard Schema} per label instead of handlers infers the
+     * `request.userData` types and validates them at runtime:
+     *
+     * ```ts
+     * import { z } from 'zod';
+     *
+     * const router = Router.create({
+     *     PRODUCT: z.object({ sku: z.string() }),
+     * });
+     * ```
      */
     static create<
         Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext,
         UserData extends Dictionary = GetUserDataFromRequest<Context['request']>,
-    >(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context> {
-        const router = new Router<Context>();
+        Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>,
+    >(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+
+    static create<
+        Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext,
+        const Schemas extends RouteSchemas = RouteSchemas,
+    >(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
+
+    static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext>(
+        routesOrSchemas?: Record<string, ((ctx: any) => Awaitable<void>) | StandardSchemaV1>,
+    ): RouterHandler<Context, any> {
+        const router = new Router<Context, any>();
         const obj = Object.create(Function.prototype);
 
         obj.addHandler = router.addHandler.bind(router);
         obj.addDefaultHandler = router.addDefaultHandler.bind(router);
+        obj.addSchemas = router.addSchemas.bind(router);
         obj.getHandler = router.getHandler.bind(router);
         obj.use = router.use.bind(router);
 
-        for (const [label, handler] of Object.entries(routes ?? {})) {
-            router.addHandler(label, handler);
+        for (const [label, value] of Object.entries(routesOrSchemas ?? {})) {
+            if (typeof value === 'function') {
+                router.addHandler(label as keyof Context & string, value as (ctx: any) => Awaitable<void>);
+            } else {
+                router.schemas.set(label, value);
+            }
         }
 
         const func = async function (context: Context) {
             const { url, loadedUrl, label } = context.request;
             context.log.debug('Page opened.', { label, url: loadedUrl ?? url });
 
+            await router.validateRequest(context);
+
             for (const middleware of router.middlewares) {
                 await middleware(context);
             }
@@ -204,6 +378,6 @@ export class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLink
 
         Object.setPrototypeOf(func, obj);
 
-        return func as unknown as RouterHandler<Context>;
+        return func as unknown as RouterHandler<Context, any>;
     }
 }