fix(scraper): check HTTP response status before extracting page content

All three scrapers (accessibility, cookie, privacy) now capture the
response object from page.goto() and return scrape_error with the HTTP
status code when the server responds with 4xx/5xx. Previously, error
pages were silently sent to Bedrock, resulting in misleading
"did not contain an accessibility statement" messages.

Also adds character-count logging after text extraction to aid
future debugging of similar issues.

Author: edwardlonsdale

src/scraper/accessibilityScraper.ts

@@ -92,29 +92,52 @@ export async function scrapeAccessibility(
       // session warm-up best-effort
     }
 
+    let response: import("playwright").Response | null = null;
     try {
-      await page.goto(effectiveUrl, {
+      response = await page.goto(effectiveUrl, {
         waitUntil: "networkidle",
         timeout: config.playwrightTimeout,
       });
     } catch {
       // proceed with whatever loaded
     }
 
+    if (response && response.status() >= 400) {
+      return insertAccessibilityResult(pool, {
+        ...base,
+        ...empty,
+        scrapeStatus: "scrape_error",
+        errorMessage: `HTTP ${response.status()} fetching accessibility statement`,
+        accessibilityStatementUrl: effectiveUrl,
+        rawBedrockResponse: null,
+      });
+    }
+
     const deeperLink = await findDeeperStatementLink(
       page,
       page.url(),
       "accessibility",
     );
     if (deeperLink) {
+      let deeperResponse: import("playwright").Response | null = null;
       try {
-        await page.goto(deeperLink.href, {
+        deeperResponse = await page.goto(deeperLink.href, {
           waitUntil: "networkidle",
           timeout: config.playwrightTimeout,
         });
       } catch {
         // proceed with whatever loaded
       }
+      if (deeperResponse && deeperResponse.status() >= 400) {
+        try {
+          await page.goto(effectiveUrl, {
+            waitUntil: "networkidle",
+            timeout: config.playwrightTimeout,
+          });
+        } catch {
+          // fall back to whatever loaded
+        }
+      }
     }
 
     const html = await page.content();
@@ -131,6 +154,9 @@ export async function scrapeAccessibility(
     }
 
     const mainText = await extractMainText(page);
+    console.log(
+      `[accessibility] ${service.name}: extracted ${mainText.length} chars`,
+    );
     const bedrockResult = await extractAccessibilityFromBedrock(
       mainText,
       config,

src/scraper/cookieScraper.ts

@@ -77,29 +77,52 @@ export async function scrapeCookies(
       }
     });
 
+    let response: import("playwright").Response | null = null;
     try {
-      await page.goto(effectiveUrl, {
+      response = await page.goto(effectiveUrl, {
         waitUntil: "networkidle",
         timeout: config.playwrightTimeout,
       });
     } catch {
       // proceed with whatever loaded
     }
 
+    if (response && response.status() >= 400) {
+      return insertCookieResult(pool, {
+        ...base,
+        ...empty,
+        scrapeStatus: "scrape_error",
+        errorMessage: `HTTP ${response.status()} fetching cookie policy`,
+        cookiePolicyUrl: effectiveUrl,
+        rawBedrockResponse: null,
+      });
+    }
+
     const deeperLink = await findDeeperStatementLink(
       page,
       page.url(),
       "cookies",
     );
     if (deeperLink) {
+      let deeperResponse: import("playwright").Response | null = null;
       try {
-        await page.goto(deeperLink.href, {
+        deeperResponse = await page.goto(deeperLink.href, {
           waitUntil: "networkidle",
           timeout: config.playwrightTimeout,
         });
       } catch {
         // proceed with whatever loaded
       }
+      if (deeperResponse && deeperResponse.status() >= 400) {
+        try {
+          await page.goto(effectiveUrl, {
+            waitUntil: "networkidle",
+            timeout: config.playwrightTimeout,
+          });
+        } catch {
+          // fall back to whatever loaded
+        }
+      }
     }
 
     const html = await page.content();
@@ -116,6 +139,9 @@ export async function scrapeCookies(
     }
 
     const fullText = await extractFullText(page);
+    console.log(
+      `[cookies] ${service.name}: extracted ${fullText.length} chars`,
+    );
     const bedrockResult = await extractCookiesFromBedrock(
       fullText,
       setCookieHeaders,

src/scraper/privacyScraper.ts

@@ -66,29 +66,52 @@ export async function scrapePrivacy(
       // session warm-up best-effort
     }
 
+    let response: import("playwright").Response | null = null;
     try {
-      await page.goto(effectiveUrl, {
+      response = await page.goto(effectiveUrl, {
         waitUntil: "networkidle",
         timeout: config.playwrightTimeout,
       });
     } catch {
       // proceed with whatever loaded
     }
 
+    if (response && response.status() >= 400) {
+      return insertPrivacyResult(pool, {
+        ...base,
+        ...empty,
+        scrapeStatus: "scrape_error",
+        errorMessage: `HTTP ${response.status()} fetching privacy notice`,
+        privacyPolicyUrl: effectiveUrl,
+        rawBedrockResponse: null,
+      });
+    }
+
     const deeperLink = await findDeeperStatementLink(
       page,
       page.url(),
       "privacy",
     );
     if (deeperLink) {
+      let deeperResponse: import("playwright").Response | null = null;
       try {
-        await page.goto(deeperLink.href, {
+        deeperResponse = await page.goto(deeperLink.href, {
           waitUntil: "networkidle",
           timeout: config.playwrightTimeout,
         });
       } catch {
         // proceed with whatever loaded
       }
+      if (deeperResponse && deeperResponse.status() >= 400) {
+        try {
+          await page.goto(effectiveUrl, {
+            waitUntil: "networkidle",
+            timeout: config.playwrightTimeout,
+          });
+        } catch {
+          // fall back to whatever loaded
+        }
+      }
     }
 
     const html = await page.content();
@@ -105,6 +128,9 @@ export async function scrapePrivacy(
     }
 
     const mainText = await extractMainText(page);
+    console.log(
+      `[privacy] ${service.name}: extracted ${mainText.length} chars`,
+    );
     const bedrockResult = await extractPrivacyFromBedrock(mainText, config);
 
     if ("error" in bedrockResult) {