fix: shrink HTML with cheerio

Author: Patai5

code/package-lock.json

@@ -13,6 +13,7 @@
         "ajv-formats": "^2.1.1",
         "apify": "^3.1.16",
         "apify-client": "^2.9.3",
+        "cheerio": "^1.0.0",
         "crawlee": "^3.8.1",
         "gpt-3-encoder": "^1.1.4",
         "joplin-turndown-plugin-gfm": "^1.0.12",

code/package.json

@@ -1,50 +1,31 @@
+import { load } from 'cheerio';
 import { encode } from 'gpt-3-encoder';
-import { Page } from 'playwright';
 
 import { htmlToMarkdownProcessor } from './markdown.js';
 
 const JSON_REGEX = /\{(?:[^{}]|())*\}/;
 
 /**
  * Shrinks HTML by removing css targeted elements and extra spaces
- * @param html
  */
 export const shrinkHtml = async (
     html: string,
-    page: Page,
     options: { removeLinkUrls: boolean; removeElementsCssSelector?: string },
 ) => {
     const { removeElementsCssSelector, removeLinkUrls } = options;
 
-    const stripped = await page.evaluate(
-        // eslint-disable-next-line @typescript-eslint/no-shadow
-        ([unstripped, removeSelector, removeLinkUrls]) => {
-            const doc = new DOMParser().parseFromString(unstripped, 'text/html');
-            if (removeSelector) {
-                const elements = doc.querySelectorAll(removeSelector);
-                for (const element of elements) {
-                    // there have been some cases when the page's own scripts cause errors and running this line
-                    // causes them to reemerge, so what in try/cartch
-                    try {
-                        element.remove();
-                    } catch (err) {
-                        /* ignore */
-                    }
-                }
-            }
+    const $ = load(html);
 
-            if (removeLinkUrls) {
-                const linkEls = doc.querySelectorAll('a');
-                for (const linkEl of linkEls) {
-                    linkEl.removeAttribute('href');
-                }
-            }
+    if (removeElementsCssSelector) {
+        $(removeElementsCssSelector).map((_, el) => $(el).remove());
+    }
+    if (removeLinkUrls) {
+        $('a').map((_, el) => $(el).removeAttr('href'));
+    }
 
-            return doc.documentElement.outerHTML;
-        },
-        [html, removeElementsCssSelector, removeLinkUrls] as const,
-    );
-    return stripped.replace(/\s{2,}/g, ' ') // remove extra spaces
+    const stripped = $.html();
+    return stripped
+        .replace(/\s{2,}/g, ' ') // remove extra spaces
         .replace(/>\s+</g, '><'); // remove all spaces between tags
 };
 

code/src/processors.ts

@@ -114,7 +114,7 @@ export const crawlRoute = async (context: PlaywrightCrawlingContext) => {
         originContentHtml = await page.content();
     }
 
-    const shrunkHtml = await shrinkHtml(originContentHtml, page, { removeLinkUrls, removeElementsCssSelector });
+    const shrunkHtml = await shrinkHtml(originContentHtml, { removeLinkUrls, removeElementsCssSelector });
     const originPageContent = pageFormat === PAGE_FORMAT.MARKDOWN ? htmlToMarkdown(shrunkHtml) : shrunkHtml;
 
     const instructionTokenLength = getNumberOfTextTokens(instructions);