diff --git a/packages/cheerio-crawler/src/internals/cheerio-parser.ts b/packages/cheerio-crawler/src/internals/cheerio-parser.ts index 24bac90a10ef..04ce3d0e1679 100644 --- a/packages/cheerio-crawler/src/internals/cheerio-parser.ts +++ b/packages/cheerio-crawler/src/internals/cheerio-parser.ts @@ -1,7 +1,13 @@ import type { DOMParser, InternalHttpCrawlingContext } from '@crawlee/http'; import { extractUrlsFromCheerio } from '@crawlee/utils/internal'; import type { CheerioAPI, CheerioOptions } from 'cheerio'; -import * as cheerio from 'cheerio'; +// We parse with htmlparser2, not with cheerio's default parse5: parse5 is a strict HTML5 parser with no +// XML mode, so it mangles the XML/RSS/Atom feeds this parser also serves (`` is a void element in +// HTML, CDATA is not recognised, self-closing unknown tags swallow their siblings). It is also stricter +// than htmlparser2 on broken markup, which is the norm when scraping. +// The slim entrypoint is the htmlparser2-only build, so it additionally keeps `parse5` and - because +// cheerio declares `undici` for its unused `fromURL()` helper - ~1 MB of `undici` out of the module graph. +import * as cheerio from 'cheerio/slim'; import { parseDocument } from 'htmlparser2'; export interface CheerioParseResult { diff --git a/packages/http-crawler/src/internals/dom-crawler.ts b/packages/http-crawler/src/internals/dom-crawler.ts index 274479bb020e..871c59bfc762 100644 --- a/packages/http-crawler/src/internals/dom-crawler.ts +++ b/packages/http-crawler/src/internals/dom-crawler.ts @@ -277,7 +277,16 @@ export class DOMCrawler< }); }, async parseWithCheerio(selector?: string, _timeoutMs = 5_000) { - const $ = (await parser.toCheerio?.(context)) ?? (await import('cheerio')).load(context.body); + // Parsers that do not build a cheerio tree themselves fall back to the full cheerio + // entrypoint (parse5). That is deliberate for the DOM-backed parsers - jsdom and linkedom + // are spec-compliant, so a spec-compliant HTML parser gives a `$` that agrees with them. + // `xmlMode` still has to follow the response: `` is a void element in HTML, so + // without it every `` in an XML feed reads back empty. + const $ = + (await parser.toCheerio?.(context)) ?? + (await import('cheerio')).load(context.body, { + xmlMode: context.contentType.type.includes('xml'), + }); if (selector && $(selector).get().length === 0) { throw new Error(`Selector '${selector}' not found.`); diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts index 5d0cb7e47777..7bead7e89603 100644 --- a/packages/http-crawler/src/internals/http-crawler.ts +++ b/packages/http-crawler/src/internals/http-crawler.ts @@ -611,17 +611,23 @@ export class HttpCrawler< const response = parsed.response!; const contentType = parsed.contentType!; + // The slim entrypoint always parses with htmlparser2, which is what `CheerioCrawler` uses as + // well - parse5 is an HTML5 parser with no XML mode, so it mangles the XML feeds that arrive + // here just as often as HTML does. It also keeps `undici` and `parse5` out of the module graph. + const loadBody = async () => { + const { load } = await import('cheerio/slim'); + + return load(parsed.body!.toString(), { xmlMode: contentType.type.includes('xml') }); + }; const waitForSelector = async (selector: string, _timeoutMs?: number) => { - const cheerio = await import('cheerio'); - const $ = cheerio.load(parsed.body!.toString()); + const $ = await loadBody(); if ($(selector).get().length === 0) { throw new Error(`Selector '${selector}' not found.`); } }; const parseWithCheerio = async (selector?: string, timeoutMs?: number) => { - const cheerio = await import('cheerio'); - const $ = cheerio.load(parsed.body!.toString()); + const $ = await loadBody(); if (selector) { await (crawlingContext as InternalHttpCrawlingContext).waitForSelector(selector, timeoutMs); diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts index 8e95548c262f..817401862eda 100644 --- a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts +++ b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts @@ -588,6 +588,9 @@ export async function parseWithCheerio( ? null : ((await page.evaluate(`(${expandShadowRoots.toString()})(document)`)) as string); const pageContent = html || (await page.content()); + // Full cheerio (parse5) on purpose: `pageContent` is the browser's own serialization of its DOM, + // and only a spec-compliant HTML5 parser reproduces the tree the browser had - so selectors copied + // out of devtools keep working. See `CheerioCrawler` for why the HTTP crawlers use htmlparser2. const { load } = await import('cheerio'); const $ = load(pageContent); diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts index dfca18387a11..215ba1cc1ff7 100644 --- a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts +++ b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts @@ -263,6 +263,9 @@ export async function parseWithCheerio( : ((await page.evaluate(`(${expandShadowRoots.toString()})(document)`)) as string); const pageContent = html || (await page.content()); + // Full cheerio (parse5) on purpose: `pageContent` is the browser's own serialization of its DOM, + // and only a spec-compliant HTML5 parser reproduces the tree the browser had - so selectors copied + // out of devtools keep working. See `CheerioCrawler` for why the HTTP crawlers use htmlparser2. const { load } = await import('cheerio'); return load(pageContent); } diff --git a/packages/utils/src/internals/cheerio.ts b/packages/utils/src/internals/cheerio.ts index 20f74819c20a..7d33910e9a39 100644 --- a/packages/utils/src/internals/cheerio.ts +++ b/packages/utils/src/internals/cheerio.ts @@ -39,7 +39,7 @@ const BLOCK_TAGS_REGEX = * @return Plain text */ export async function htmlToText(htmlOrCheerioElement: string | CheerioAPI): Promise { - const { load } = await import('cheerio'); + const { load } = await import('cheerio/slim'); if (!htmlOrCheerioElement) return ''; @@ -53,8 +53,12 @@ export async function htmlToText(htmlOrCheerioElement: string | CheerioAPI): Pro if (elem.type === 'text') { // Compress spaces, unless we're inside
 element
                 let compr;
-                if (elem.parent?.tagName === 'pre') compr = elem.data;
-                else compr = elem.data.replace(/\s+/g, ' ');
+                if (elem.parent?.tagName === 'pre') {
+                    // A single newline right after `
` is markup, not content - the HTML spec has
+                    // the parser drop it. htmlparser2 keeps it, so strip it here instead; that way the
+                    // output does not depend on which parser produced the tree.
+                    compr = elem.parent.children?.[0] === elem ? elem.data.replace(/^\r?\n/, '') : elem.data;
+                } else compr = elem.data.replace(/\s+/g, ' ');
                 // If text is empty or ends with a whitespace, don't add the leading whitespace
                 if (compr.startsWith(' ') && /(^|\s)$/.test(text)) compr = compr.substring(1);
                 text += compr;
diff --git a/packages/utils/src/internals/extract-microdata.ts b/packages/utils/src/internals/extract-microdata.ts
index a9b222196104..ab2b2f4c367d 100644
--- a/packages/utils/src/internals/extract-microdata.ts
+++ b/packages/utils/src/internals/extract-microdata.ts
@@ -35,7 +35,7 @@ export async function extractMicrodata(raw: string): Promise;
 export async function extractMicrodata($: CheerioAPI): Promise;
 export async function extractMicrodata(htmlOrCheerioElement: string | CheerioAPI): Promise {
     // Dynamic so that importing `@crawlee/utils` does not pull in cheerio - see #3836.
-    const { load } = await import('cheerio');
+    const { load } = await import('cheerio/slim');
     const $ = typeof htmlOrCheerioElement === 'string' ? load(htmlOrCheerioElement) : htmlOrCheerioElement;
     const context: ExtractionContext = { $ };
 
diff --git a/packages/utils/src/internals/open_graph_parser.ts b/packages/utils/src/internals/open_graph_parser.ts
index 3290f6b793da..1f5d15b1f7a9 100644
--- a/packages/utils/src/internals/open_graph_parser.ts
+++ b/packages/utils/src/internals/open_graph_parser.ts
@@ -403,7 +403,7 @@ export async function parseOpenGraph(
     additionalProperties?: OpenGraphProperty[],
 ): Promise>;
 export async function parseOpenGraph(item: CheerioAPI | string, additionalProperties?: OpenGraphProperty[]) {
-    const { load } = await import('cheerio');
+    const { load } = await import('cheerio/slim');
     const $ = typeof item === 'string' ? load(item) : item;
 
     return [...(additionalProperties || []), ...OPEN_GRAPH_PROPERTIES].reduce((acc, curr) => {
diff --git a/packages/utils/src/internals/social.ts b/packages/utils/src/internals/social.ts
index 9977d6504d6c..9293934dcd85 100644
--- a/packages/utils/src/internals/social.ts
+++ b/packages/utils/src/internals/social.ts
@@ -663,7 +663,7 @@ export async function parseHandlesFromHtml(
     html: string,
     data: Record | null = null,
 ): Promise {
-    const cheerio = await import('cheerio');
+    const cheerio = await import('cheerio/slim');
 
     const result: SocialHandles = {
         emails: [],
diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.ts
index 729d59989f1d..7eb5c01faa57 100644
--- a/test/core/crawlers/http_crawler.test.ts
+++ b/test/core/crawlers/http_crawler.test.ts
@@ -31,6 +31,16 @@ router.set('/noext', (req, res) => {
     res.end(`Example Domain`);
 });
 
+router.set('/feed.xml', (req, res) => {
+    res.setHeader('content-type', 'application/rss+xml; charset=utf-8');
+    res.end(
+        `` +
+            `Post onehttps://example.com/one` +
+            `Post twohttps://example.com/two` +
+            ``,
+    );
+});
+
 router.set('/invalidContentType', (req, res) => {
     res.setHeader('content-type', 'crazy-stuff; charset=utf-8');
     res.end(`Example Domain`);
@@ -222,6 +232,29 @@ test('parseWithCheerio works', async () => {
     expect(results).toStrictEqual(['Example Domain']);
 });
 
+// `` is a void element in HTML, so an HTML parser drops its content - the XML feeds that
+// `HttpCrawler` also serves have to be parsed in xml mode to survive.
+test('parseWithCheerio parses XML responses as XML', async () => {
+    const results: string[][] = [];
+
+    const crawler = new HttpCrawler({
+        maxRequestRetries: 0,
+        additionalMimeTypes: ['application/rss+xml'],
+        requestHandler: async ({ parseWithCheerio }) => {
+            const $ = await parseWithCheerio();
+            results.push(
+                $('item > link')
+                    .map((_i, el) => $(el).text())
+                    .get(),
+            );
+        },
+    });
+
+    await crawler.run([`${url}/feed.xml`]);
+
+    expect(results).toStrictEqual([['https://example.com/one', 'https://example.com/two']]);
+});
+
 test('should parse content type from header', async () => {
     const results: { type: string; encoding: BufferEncoding }[] = [];