Fix scraper dropping characters from CSS pseudo-element anti-scraping techniques

chrome-extension/content.js

@@ -67,6 +67,29 @@ chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
   }
 });
 
+// Convert a small subset of Markdown to HTML.
+// Handles bold, italic, headings, inline code, unordered lists, and code fences.
+function markdownToHtml(text) {
+  return text
+    // Strip wrapping code fences (```html ... ``` or ``` ... ```)
+    .replace(/^```(?:html)?\n?/gi, '').replace(/\n?```$/g, '')
+    // Headings
+    .replace(/^### (.*$)/gim, '<h3>$1</h3>')
+    .replace(/^## (.*$)/gim, '<h2>$1</h2>')
+    .replace(/^# (.*$)/gim, '<h1>$1</h1>')
+    // Bold and italic
+    .replace(/\*\*(.*?)\*\*/g, '<strong>$1</strong>')
+    .replace(/\*(.*?)\*/g, '<em>$1</em>')
+    // Inline code
+    .replace(/`([^`]+)`/g, '<code>$1</code>')
+    // Unordered list items (lines starting with "* " or "- ")
+    .replace(/^[*-] (.+)$/gim, '<li>$1</li>')
+    // Wrap consecutive <li> elements in a <ul>
+    .replace(/(<li>[\s\S]*?<\/li>\n?)+/g, '<ul>$&</ul>')
+    // Newlines to <br>
+    .replace(/\n/g, '<br>');
+}
+
 async function fetchSummary(additionalQuestions, selectedLanguage, prompt, summaryLength, targetElement, debugEnabled) {
   // Increase tokenLimit for Gemini-style providers. This is an approximate
   // token limit (measured in tokens) used to decide how much of the page to
@@ -227,17 +250,10 @@ async function fetchSummary(additionalQuestions, selectedLanguage, prompt, summa
               if (contentPiece) {
                 summary += contentPiece;
 
-                // Extremely simple markdown-to-HTML parser for LLMs that ignore the HTML prompt
-                let formattedSummary = summary
-                  .replace(/^```html\n?/gi, '').replace(/\n?```$/g, '') // strip markdown codeblocks if they wrap HTML
-                  .replace(/\*\*(.*?)\*\*/g, '<strong>$1</strong>')
-                  .replace(/\*(.*?)\*/g, '<em>$1</em>')
-                  .replace(/^### (.*$)/gim, '<h3>$1</h3>')
-                  .replace(/^## (.*$)/gim, '<h2>$1</h2>')
-                  .replace(/^# (.*$)/gim, '<h1>$1</h1>');
-
+                // Markdown-to-HTML conversion for live streaming preview
+                const formattedSummary = markdownToHtml(summary);
                 // Live update the UI
-                outputArea.innerHTML = `<small style="opacity:0.7">Drafting...</small><br>${formattedSummary.replace(/\\n/g, '<br>')}`;
+                outputArea.innerHTML = `<small style="opacity:0.7">Drafting...</small><br>${formattedSummary}`;
                 if (debugEnabled) updateDebugPanel(summary, finalApiUrl);
               }
             } catch (e) { /* Ignore partial JSON chunks */ }
@@ -247,17 +263,11 @@ async function fetchSummary(additionalQuestions, selectedLanguage, prompt, summa
         // Finalize UI
         streamContainer.remove();
 
-        // Final markdown-to-HTML pass (including clean spacing)
-        let finalHtml = summary
-          .replace(/^```(?:html)?\n?/gi, '').replace(/\n?```$/g, '')
-          .replace(/\*\*(.*?)\*\*/g, '<strong>$1</strong>')
-          .replace(/\*(.*?)\*/g, '<em>$1</em>')
-          .replace(/^### (.*$)/gim, '<h3>$1</h3>')
-          .replace(/^## (.*$)/gim, '<h2>$1</h2>')
-          .replace(/^# (.*$)/gim, '<h1>$1</h1>');
+        // Final markdown-to-HTML pass
+        const finalHtml = markdownToHtml(summary);
 
         const summaryContainer = document.createElement('blockquote');
-        summaryContainer.innerHTML = `<div><h2>AI Summary 🧙</h2>${finalHtml.replace(/\\n/g, '<br>')}</div>`;
+        summaryContainer.innerHTML = `<div><h2>AI Summary 🧙</h2>${finalHtml}</div>`;
         insertSummary(targetElement, summaryContainer);
 
         saveToLocalStorage(truncatedContent, summary, window.location.href, document.title, '');
@@ -314,9 +324,6 @@ function chunkText(text, chunkSize) {
 // Function to extract all relevant text content from the page
 function getAllTextContent() {
   console.log('Getting all text content');
-
-  // Create a clone of the document body so we don't mutate the actual page
-  const clone = document.body.cloneNode(true);
 
   // List of selectors for elements that usually contain non-article noise
   const noiseSelectors = [
@@ -326,45 +333,96 @@ function getAllTextContent() {
     '#comments', '.comments', '.sidebar', '#sidebar'
   ];
 
-  for (const selector of noiseSelectors) {
-    const nodes = clone.querySelectorAll(selector);
-    for (const node of nodes) {
-      node.remove();
+  const blockTags = new Set([
+    'p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
+    'li', 'article', 'section', 'blockquote', 'br', 'tr'
+  ]);
+
+  // Check whether an element (or any of its ancestors) matches a noise selector.
+  // We work on the live DOM so getComputedStyle is available for pseudo-elements.
+  // Pre-compute the set of noise container elements once for efficient ancestry checks.
+  const noiseSet = new Set();
+  for (const sel of noiseSelectors) {
+    try {
+      document.querySelectorAll(sel).forEach(el => noiseSet.add(el));
+    } catch (e) {
+      console.warn('getAllTextContent: skipping invalid noise selector', sel, e);
     }
   }
 
-  // To prevent block elements from running together when reading textContent,
-  // we can insert newlines before and after them.
-  const blockElements = ['p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li', 'article', 'section', 'blockquote'];
-  for (const tag of blockElements) {
-    const nodes = clone.querySelectorAll(tag);
-    for (const node of nodes) {
-      node.prepend(document.createTextNode('\\n\\n'));
-      node.append(document.createTextNode('\\n\\n'));
+  // Returns true if el is a known noise container root. Because extractText() never
+  // recurses into skipped subtrees, the parent context is already confirmed clean,
+  // so a direct Set lookup is sufficient (no ancestor walk needed).
+  function isNoise(el) {
+    return noiseSet.has(el);
+  }
+
+  // Extract the rendered text injected by a CSS pseudo-element (::before / ::after).
+  // Many sites use pseudo-elements to insert characters as an anti-scraping measure;
+  // textContent on a detached clone never sees this content, but getComputedStyle does.
+  function getPseudoContent(el, pseudo) {
+    try {
+      const content = window.getComputedStyle(el, pseudo).getPropertyValue('content');
+      if (!content || content === 'none' || content === 'normal') return '';
+      // Strip the outermost CSS quotes from the string value (browser always wraps in double quotes).
+      const q = content[0];
+      const stripped = (q === '"' || q === "'") && content[content.length - 1] === q
+        ? content.slice(1, -1)
+        : content;
+      // Discard icon-font characters: fonts like Font Awesome map glyphs to Unicode
+      // Private Use Area code points, which would appear as garbage symbols in output.
+      // The regex covers BMP PUA (U+E000–U+F8FF) and supplementary PUA planes
+      // (U+F0000–U+FFFFD, U+100000–U+10FFFD) via their UTF-16 surrogate pairs.
+      return stripped.replace(/[\uE000-\uF8FF]|[\uDB80-\uDBFF][\uDC00-\uDFFF]/g, '');
+    } catch (e) {
+      return '';
     }
   }
 
-  // Fallback chain: look for article, or main, or use the whole body clone
-  let mainNode = clone.querySelector('article') || 
-                 clone.querySelector('main') || 
-                 clone.querySelector('[role="main"]') || 
-                 clone;
+  // Recursively build plain text from a live DOM element, capturing both text
+  // nodes and CSS pseudo-element content that textContent would normally miss.
+  function extractText(el) {
+    let result = '';
+    result += getPseudoContent(el, '::before');
+    for (const child of el.childNodes) {
+      if (child.nodeType === Node.TEXT_NODE) {
+        result += child.textContent;
+      } else if (child.nodeType === Node.ELEMENT_NODE) {
+        if (isNoise(child)) continue;
+        const tag = child.tagName.toLowerCase();
+        if (blockTags.has(tag)) {
+          result += '\n\n' + extractText(child) + '\n\n';
+        } else {
+          result += extractText(child);
+        }
+      }
+    }
+    result += getPseudoContent(el, '::after');
+    return result;
+  }
+
+  // Fallback chain: look for article, or main, or use the whole body
+  const mainEl =
+    document.querySelector('article') ||
+    document.querySelector('main') ||
+    document.querySelector('[role="main"]') ||
+    document.body;
+
+  let text = extractText(mainEl);
 
-  let text = mainNode.textContent || '';
-
   // Clean up whitespace: replace multiple spaces with one, and multiple newlines with double newlines
   let cleanText = text
-    .replace(/[ \\t]+/g, ' ')      
-    .replace(/\\n\\s*\\n/g, '\\n\\n')  
+    .replace(/[ \t]+/g, ' ')
+    .replace(/\n\s*\n/g, '\n\n')
     .trim();
 
   // If it's suspiciously short (e.g. they put the main article outside <main>), fallback
-  if (cleanText.length < 500 && mainNode !== clone) {
-    console.log("Extracted content too short, falling back to full body container.");
-    text = clone.textContent || '';
+  if (cleanText.length < 500 && mainEl !== document.body) {
+    console.log('Extracted content too short, falling back to full body container.');
+    text = extractText(document.body);
     cleanText = text
-      .replace(/[ \\t]+/g, ' ')
-      .replace(/\\n\\s*\\n/g, '\\n\\n')
+      .replace(/[ \t]+/g, ' ')
+      .replace(/\n\s*\n/g, '\n\n')
       .trim();
   }
 
@@ -526,6 +584,9 @@ function updateDebugPanel(text, apiUrl) {
     panel.style.cssText = `position:fixed;right:10px;bottom:10px;width:350px;max-height:40vh;overflow:auto;background:#222;color:#0f0;padding:10px;z-index:10000;font-family:monospace;font-size:11px;border-radius:5px;box-shadow:0 0 10px rgba(0,0,0,0.5);`;
     document.body.appendChild(panel);
   }
-  panel.innerHTML = `<strong>Debug (${apiUrl})</strong><hr><pre style="white-space:pre-wrap">${text}</pre>`;
+  // Set content via textContent to avoid XSS from API responses or URL strings.
+  panel.innerHTML = '<strong></strong><hr><pre style="white-space:pre-wrap"></pre>';
+  panel.querySelector('strong').textContent = `Debug (${apiUrl})`;
+  panel.querySelector('pre').textContent = text;
 }