diff --git a/packages/cheerio-crawler/src/internals/cheerio-parser.ts b/packages/cheerio-crawler/src/internals/cheerio-parser.ts
index 24bac90a10ef..04ce3d0e1679 100644
--- a/packages/cheerio-crawler/src/internals/cheerio-parser.ts
+++ b/packages/cheerio-crawler/src/internals/cheerio-parser.ts
@@ -1,7 +1,13 @@
import type { DOMParser, InternalHttpCrawlingContext } from '@crawlee/http';
import { extractUrlsFromCheerio } from '@crawlee/utils/internal';
import type { CheerioAPI, CheerioOptions } from 'cheerio';
-import * as cheerio from 'cheerio';
+// We parse with htmlparser2, not with cheerio's default parse5: parse5 is a strict HTML5 parser with no
+// XML mode, so it mangles the XML/RSS/Atom feeds this parser also serves (`` is a void element in
+// HTML, CDATA is not recognised, self-closing unknown tags swallow their siblings). It is also stricter
+// than htmlparser2 on broken markup, which is the norm when scraping.
+// The slim entrypoint is the htmlparser2-only build, so it additionally keeps `parse5` and - because
+// cheerio declares `undici` for its unused `fromURL()` helper - ~1 MB of `undici` out of the module graph.
+import * as cheerio from 'cheerio/slim';
import { parseDocument } from 'htmlparser2';
export interface CheerioParseResult {
diff --git a/packages/http-crawler/src/internals/dom-crawler.ts b/packages/http-crawler/src/internals/dom-crawler.ts
index 274479bb020e..871c59bfc762 100644
--- a/packages/http-crawler/src/internals/dom-crawler.ts
+++ b/packages/http-crawler/src/internals/dom-crawler.ts
@@ -277,7 +277,16 @@ export class DOMCrawler<
});
},
async parseWithCheerio(selector?: string, _timeoutMs = 5_000) {
- const $ = (await parser.toCheerio?.(context)) ?? (await import('cheerio')).load(context.body);
+ // Parsers that do not build a cheerio tree themselves fall back to the full cheerio
+ // entrypoint (parse5). That is deliberate for the DOM-backed parsers - jsdom and linkedom
+ // are spec-compliant, so a spec-compliant HTML parser gives a `$` that agrees with them.
+ // `xmlMode` still has to follow the response: `` is a void element in HTML, so
+ // without it every `` in an XML feed reads back empty.
+ const $ =
+ (await parser.toCheerio?.(context)) ??
+ (await import('cheerio')).load(context.body, {
+ xmlMode: context.contentType.type.includes('xml'),
+ });
if (selector && $(selector).get().length === 0) {
throw new Error(`Selector '${selector}' not found.`);
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index 5d0cb7e47777..7bead7e89603 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -611,17 +611,23 @@ export class HttpCrawler<
const response = parsed.response!;
const contentType = parsed.contentType!;
+ // The slim entrypoint always parses with htmlparser2, which is what `CheerioCrawler` uses as
+ // well - parse5 is an HTML5 parser with no XML mode, so it mangles the XML feeds that arrive
+ // here just as often as HTML does. It also keeps `undici` and `parse5` out of the module graph.
+ const loadBody = async () => {
+ const { load } = await import('cheerio/slim');
+
+ return load(parsed.body!.toString(), { xmlMode: contentType.type.includes('xml') });
+ };
const waitForSelector = async (selector: string, _timeoutMs?: number) => {
- const cheerio = await import('cheerio');
- const $ = cheerio.load(parsed.body!.toString());
+ const $ = await loadBody();
if ($(selector).get().length === 0) {
throw new Error(`Selector '${selector}' not found.`);
}
};
const parseWithCheerio = async (selector?: string, timeoutMs?: number) => {
- const cheerio = await import('cheerio');
- const $ = cheerio.load(parsed.body!.toString());
+ const $ = await loadBody();
if (selector) {
await (crawlingContext as InternalHttpCrawlingContext).waitForSelector(selector, timeoutMs);
diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts
index 8e95548c262f..817401862eda 100644
--- a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts
+++ b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts
@@ -588,6 +588,9 @@ export async function parseWithCheerio(
? null
: ((await page.evaluate(`(${expandShadowRoots.toString()})(document)`)) as string);
const pageContent = html || (await page.content());
+ // Full cheerio (parse5) on purpose: `pageContent` is the browser's own serialization of its DOM,
+ // and only a spec-compliant HTML5 parser reproduces the tree the browser had - so selectors copied
+ // out of devtools keep working. See `CheerioCrawler` for why the HTTP crawlers use htmlparser2.
const { load } = await import('cheerio');
const $ = load(pageContent);
diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts
index dfca18387a11..215ba1cc1ff7 100644
--- a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts
+++ b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts
@@ -263,6 +263,9 @@ export async function parseWithCheerio(
: ((await page.evaluate(`(${expandShadowRoots.toString()})(document)`)) as string);
const pageContent = html || (await page.content());
+ // Full cheerio (parse5) on purpose: `pageContent` is the browser's own serialization of its DOM,
+ // and only a spec-compliant HTML5 parser reproduces the tree the browser had - so selectors copied
+ // out of devtools keep working. See `CheerioCrawler` for why the HTTP crawlers use htmlparser2.
const { load } = await import('cheerio');
return load(pageContent);
}
diff --git a/packages/utils/src/internals/cheerio.ts b/packages/utils/src/internals/cheerio.ts
index 20f74819c20a..7d33910e9a39 100644
--- a/packages/utils/src/internals/cheerio.ts
+++ b/packages/utils/src/internals/cheerio.ts
@@ -39,7 +39,7 @@ const BLOCK_TAGS_REGEX =
* @return Plain text
*/
export async function htmlToText(htmlOrCheerioElement: string | CheerioAPI): Promise {
- const { load } = await import('cheerio');
+ const { load } = await import('cheerio/slim');
if (!htmlOrCheerioElement) return '';
@@ -53,8 +53,12 @@ export async function htmlToText(htmlOrCheerioElement: string | CheerioAPI): Pro
if (elem.type === 'text') {
// Compress spaces, unless we're inside
element
let compr;
- if (elem.parent?.tagName === 'pre') compr = elem.data;
- else compr = elem.data.replace(/\s+/g, ' ');
+ if (elem.parent?.tagName === 'pre') {
+ // A single newline right after `
` is markup, not content - the HTML spec has
+ // the parser drop it. htmlparser2 keeps it, so strip it here instead; that way the
+ // output does not depend on which parser produced the tree.
+ compr = elem.parent.children?.[0] === elem ? elem.data.replace(/^\r?\n/, '') : elem.data;
+ } else compr = elem.data.replace(/\s+/g, ' ');
// If text is empty or ends with a whitespace, don't add the leading whitespace
if (compr.startsWith(' ') && /(^|\s)$/.test(text)) compr = compr.substring(1);
text += compr;
diff --git a/packages/utils/src/internals/extract-microdata.ts b/packages/utils/src/internals/extract-microdata.ts
index a9b222196104..ab2b2f4c367d 100644
--- a/packages/utils/src/internals/extract-microdata.ts
+++ b/packages/utils/src/internals/extract-microdata.ts
@@ -35,7 +35,7 @@ export async function extractMicrodata(raw: string): Promise;
export async function extractMicrodata($: CheerioAPI): Promise;
export async function extractMicrodata(htmlOrCheerioElement: string | CheerioAPI): Promise {
// Dynamic so that importing `@crawlee/utils` does not pull in cheerio - see #3836.
- const { load } = await import('cheerio');
+ const { load } = await import('cheerio/slim');
const $ = typeof htmlOrCheerioElement === 'string' ? load(htmlOrCheerioElement) : htmlOrCheerioElement;
const context: ExtractionContext = { $ };
diff --git a/packages/utils/src/internals/open_graph_parser.ts b/packages/utils/src/internals/open_graph_parser.ts
index 3290f6b793da..1f5d15b1f7a9 100644
--- a/packages/utils/src/internals/open_graph_parser.ts
+++ b/packages/utils/src/internals/open_graph_parser.ts
@@ -403,7 +403,7 @@ export async function parseOpenGraph(
additionalProperties?: OpenGraphProperty[],
): Promise>;
export async function parseOpenGraph(item: CheerioAPI | string, additionalProperties?: OpenGraphProperty[]) {
- const { load } = await import('cheerio');
+ const { load } = await import('cheerio/slim');
const $ = typeof item === 'string' ? load(item) : item;
return [...(additionalProperties || []), ...OPEN_GRAPH_PROPERTIES].reduce((acc, curr) => {
diff --git a/packages/utils/src/internals/social.ts b/packages/utils/src/internals/social.ts
index 9977d6504d6c..9293934dcd85 100644
--- a/packages/utils/src/internals/social.ts
+++ b/packages/utils/src/internals/social.ts
@@ -663,7 +663,7 @@ export async function parseHandlesFromHtml(
html: string,
data: Record | null = null,
): Promise {
- const cheerio = await import('cheerio');
+ const cheerio = await import('cheerio/slim');
const result: SocialHandles = {
emails: [],
diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.ts
index 729d59989f1d..7eb5c01faa57 100644
--- a/test/core/crawlers/http_crawler.test.ts
+++ b/test/core/crawlers/http_crawler.test.ts
@@ -31,6 +31,16 @@ router.set('/noext', (req, res) => {
res.end(`Example Domain`);
});
+router.set('/feed.xml', (req, res) => {
+ res.setHeader('content-type', 'application/rss+xml; charset=utf-8');
+ res.end(
+ `` +
+ `Post onehttps://example.com/one` +
+ `Post twohttps://example.com/two` +
+ ``,
+ );
+});
+
router.set('/invalidContentType', (req, res) => {
res.setHeader('content-type', 'crazy-stuff; charset=utf-8');
res.end(`Example Domain`);
@@ -222,6 +232,29 @@ test('parseWithCheerio works', async () => {
expect(results).toStrictEqual(['Example Domain']);
});
+// `` is a void element in HTML, so an HTML parser drops its content - the XML feeds that
+// `HttpCrawler` also serves have to be parsed in xml mode to survive.
+test('parseWithCheerio parses XML responses as XML', async () => {
+ const results: string[][] = [];
+
+ const crawler = new HttpCrawler({
+ maxRequestRetries: 0,
+ additionalMimeTypes: ['application/rss+xml'],
+ requestHandler: async ({ parseWithCheerio }) => {
+ const $ = await parseWithCheerio();
+ results.push(
+ $('item > link')
+ .map((_i, el) => $(el).text())
+ .get(),
+ );
+ },
+ });
+
+ await crawler.run([`${url}/feed.xml`]);
+
+ expect(results).toStrictEqual([['https://example.com/one', 'https://example.com/two']]);
+});
+
test('should parse content type from header', async () => {
const results: { type: string; encoding: BufferEncoding }[] = [];