perf!: import only cheerio/slim in cheerio/http crawlers and utils

packages/cheerio-crawler/src/internals/cheerio-parser.ts

@@ -1,7 +1,13 @@
 import type { DOMParser, InternalHttpCrawlingContext } from '@crawlee/http';
 import { extractUrlsFromCheerio } from '@crawlee/utils/internal';
 import type { CheerioAPI, CheerioOptions } from 'cheerio';
-import * as cheerio from 'cheerio';
+// We parse with htmlparser2, not with cheerio's default parse5: parse5 is a strict HTML5 parser with no
+// XML mode, so it mangles the XML/RSS/Atom feeds this parser also serves (`<link>` is a void element in
+// HTML, CDATA is not recognised, self-closing unknown tags swallow their siblings). It is also stricter
+// than htmlparser2 on broken markup, which is the norm when scraping.
+// The slim entrypoint is the htmlparser2-only build, so it additionally keeps `parse5` and - because
+// cheerio declares `undici` for its unused `fromURL()` helper - ~1 MB of `undici` out of the module graph.
+import * as cheerio from 'cheerio/slim';
 import { parseDocument } from 'htmlparser2';
 
 export interface CheerioParseResult {

packages/http-crawler/src/internals/dom-crawler.ts

@@ -277,7 +277,16 @@ export class DOMCrawler<
                 });
             },
             async parseWithCheerio(selector?: string, _timeoutMs = 5_000) {
-                const $ = (await parser.toCheerio?.(context)) ?? (await import('cheerio')).load(context.body);
+                // Parsers that do not build a cheerio tree themselves fall back to the full cheerio
+                // entrypoint (parse5). That is deliberate for the DOM-backed parsers - jsdom and linkedom
+                // are spec-compliant, so a spec-compliant HTML parser gives a `$` that agrees with them.
+                // `xmlMode` still has to follow the response: `<link>` is a void element in HTML, so
+                // without it every `<link>` in an XML feed reads back empty.
+                const $ =
+                    (await parser.toCheerio?.(context)) ??
+                    (await import('cheerio')).load(context.body, {
+                        xmlMode: context.contentType.type.includes('xml'),
+                    });
 
                 if (selector && $(selector).get().length === 0) {
                     throw new Error(`Selector '${selector}' not found.`);

packages/http-crawler/src/internals/http-crawler.ts

@@ -611,17 +611,23 @@ export class HttpCrawler<
         const response = parsed.response!;
         const contentType = parsed.contentType!;
 
+        // The slim entrypoint always parses with htmlparser2, which is what `CheerioCrawler` uses as
+        // well - parse5 is an HTML5 parser with no XML mode, so it mangles the XML feeds that arrive
+        // here just as often as HTML does. It also keeps `undici` and `parse5` out of the module graph.
+        const loadBody = async () => {
+            const { load } = await import('cheerio/slim');
+
+            return load(parsed.body!.toString(), { xmlMode: contentType.type.includes('xml') });
+        };
         const waitForSelector = async (selector: string, _timeoutMs?: number) => {
-            const cheerio = await import('cheerio');
-            const $ = cheerio.load(parsed.body!.toString());
+            const $ = await loadBody();
 
             if ($(selector).get().length === 0) {
                 throw new Error(`Selector '${selector}' not found.`);
             }
         };
         const parseWithCheerio = async (selector?: string, timeoutMs?: number) => {
-            const cheerio = await import('cheerio');
-            const $ = cheerio.load(parsed.body!.toString());
+            const $ = await loadBody();
 
             if (selector) {
                 await (crawlingContext as InternalHttpCrawlingContext).waitForSelector(selector, timeoutMs);

packages/playwright-crawler/src/internals/utils/playwright-utils.ts

@@ -588,6 +588,9 @@ export async function parseWithCheerio(
         ? null
         : ((await page.evaluate(`(${expandShadowRoots.toString()})(document)`)) as string);
     const pageContent = html || (await page.content());
+    // Full cheerio (parse5) on purpose: `pageContent` is the browser's own serialization of its DOM,
+    // and only a spec-compliant HTML5 parser reproduces the tree the browser had - so selectors copied
+    // out of devtools keep working. See `CheerioCrawler` for why the HTTP crawlers use htmlparser2.
     const { load } = await import('cheerio');
     const $ = load(pageContent);
 

packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts

@@ -263,6 +263,9 @@ export async function parseWithCheerio(
         : ((await page.evaluate(`(${expandShadowRoots.toString()})(document)`)) as string);
     const pageContent = html || (await page.content());
 
+    // Full cheerio (parse5) on purpose: `pageContent` is the browser's own serialization of its DOM,
+    // and only a spec-compliant HTML5 parser reproduces the tree the browser had - so selectors copied
+    // out of devtools keep working. See `CheerioCrawler` for why the HTTP crawlers use htmlparser2.
     const { load } = await import('cheerio');
     return load(pageContent);
 }

packages/utils/src/internals/cheerio.ts

@@ -39,7 +39,7 @@ const BLOCK_TAGS_REGEX =
  * @return Plain text
  */
 export async function htmlToText(htmlOrCheerioElement: string | CheerioAPI): Promise<string> {
-    const { load } = await import('cheerio');
+    const { load } = await import('cheerio/slim');
 
     if (!htmlOrCheerioElement) return '';
 
@@ -53,8 +53,12 @@ export async function htmlToText(htmlOrCheerioElement: string | CheerioAPI): Pro
             if (elem.type === 'text') {
                 // Compress spaces, unless we're inside <pre> element
                 let compr;
-                if (elem.parent?.tagName === 'pre') compr = elem.data;
-                else compr = elem.data.replace(/\s+/g, ' ');
+                if (elem.parent?.tagName === 'pre') {
+                    // A single newline right after `<pre>` is markup, not content - the HTML spec has
+                    // the parser drop it. htmlparser2 keeps it, so strip it here instead; that way the
+                    // output does not depend on which parser produced the tree.
+                    compr = elem.parent.children?.[0] === elem ? elem.data.replace(/^\r?\n/, '') : elem.data;
+                } else compr = elem.data.replace(/\s+/g, ' ');
                 // If text is empty or ends with a whitespace, don't add the leading whitespace
                 if (compr.startsWith(' ') && /(^|\s)$/.test(text)) compr = compr.substring(1);
                 text += compr;

packages/utils/src/internals/extract-microdata.ts

@@ -35,7 +35,7 @@ export async function extractMicrodata(raw: string): Promise<MicrodataItem[]>;
 export async function extractMicrodata($: CheerioAPI): Promise<MicrodataItem[]>;
 export async function extractMicrodata(htmlOrCheerioElement: string | CheerioAPI): Promise<MicrodataItem[]> {
     // Dynamic so that importing `@crawlee/utils` does not pull in cheerio - see #3836.
-    const { load } = await import('cheerio');
+    const { load } = await import('cheerio/slim');
     const $ = typeof htmlOrCheerioElement === 'string' ? load(htmlOrCheerioElement) : htmlOrCheerioElement;
     const context: ExtractionContext = { $ };
 

packages/utils/src/internals/open_graph_parser.ts

@@ -403,7 +403,7 @@ export async function parseOpenGraph(
     additionalProperties?: OpenGraphProperty[],
 ): Promise<Dictionary<OpenGraphResult>>;
 export async function parseOpenGraph(item: CheerioAPI | string, additionalProperties?: OpenGraphProperty[]) {
-    const { load } = await import('cheerio');
+    const { load } = await import('cheerio/slim');
     const $ = typeof item === 'string' ? load(item) : item;
 
     return [...(additionalProperties || []), ...OPEN_GRAPH_PROPERTIES].reduce((acc, curr) => {

packages/utils/src/internals/social.ts

@@ -663,7 +663,7 @@ export async function parseHandlesFromHtml(
     html: string,
     data: Record<string, unknown> | null = null,
 ): Promise<SocialHandles> {
-    const cheerio = await import('cheerio');
+    const cheerio = await import('cheerio/slim');
 
     const result: SocialHandles = {
         emails: [],

test/core/crawlers/http_crawler.test.ts

@@ -31,6 +31,16 @@ router.set('/noext', (req, res) => {
     res.end(`<html><head><title>Example Domain</title></head></html>`);
 });
 
+router.set('/feed.xml', (req, res) => {
+    res.setHeader('content-type', 'application/rss+xml; charset=utf-8');
+    res.end(
+        `<?xml version="1.0" encoding="UTF-8"?><rss version="2.0"><channel>` +
+            `<item><title>Post one</title><link>https://example.com/one</link></item>` +
+            `<item><title>Post two</title><link>https://example.com/two</link></item>` +
+            `</channel></rss>`,
+    );
+});
+
 router.set('/invalidContentType', (req, res) => {
     res.setHeader('content-type', 'crazy-stuff; charset=utf-8');
     res.end(`<html><head><title>Example Domain</title></head></html>`);
@@ -222,6 +232,29 @@ test('parseWithCheerio works', async () => {
     expect(results).toStrictEqual(['Example Domain']);
 });
 
+// `<link>` is a void element in HTML, so an HTML parser drops its content - the XML feeds that
+// `HttpCrawler` also serves have to be parsed in xml mode to survive.
+test('parseWithCheerio parses XML responses as XML', async () => {
+    const results: string[][] = [];
+
+    const crawler = new HttpCrawler({
+        maxRequestRetries: 0,
+        additionalMimeTypes: ['application/rss+xml'],
+        requestHandler: async ({ parseWithCheerio }) => {
+            const $ = await parseWithCheerio();
+            results.push(
+                $('item > link')
+                    .map((_i, el) => $(el).text())
+                    .get(),
+            );
+        },
+    });
+
+    await crawler.run([`${url}/feed.xml`]);
+
+    expect(results).toStrictEqual([['https://example.com/one', 'https://example.com/two']]);
+});
+
 test('should parse content type from header', async () => {
     const results: { type: string; encoding: BufferEncoding }[] = [];