Add warnings for missing or invalid URL replacements; rename function UrlWithoutAnchors to RemoveAnchorsFromUrl

Author: anemone008

apps/parser/src/helpers/url-handling.ts

@@ -13,6 +13,9 @@ export function sanitizeUrlAsFilename(
   options?: SanitizeOptions,
 ): string {
   if (!url) {
+    console.warn(
+      `Missing input url, sanitizing as default "${DEFAULT_REPLACEMENT}"`,
+    );
     return DEFAULT_REPLACEMENT;
   }
   let filenameBase = url;
@@ -59,18 +62,24 @@ export function sanitizeUrlAsFilename(
 
 function validReplacementOrDefault(candidate: string): string {
   if (!candidate) {
+    console.warn(
+      `Missing replacement character, using default "${DEFAULT_REPLACEMENT}"`,
+    );
     return DEFAULT_REPLACEMENT;
   }
   if (
     /[\/\?<>\\:\*\|"]/u.test(candidate) ||
     /[\x00-\x1f\x80-\x9f]/u.test(candidate)
   ) {
+    console.warn(
+      `Invalid replacement character: "${candidate}", using default "${DEFAULT_REPLACEMENT}"`,
+    );
     return DEFAULT_REPLACEMENT;
   }
   return candidate;
 }
 
-export const UrlWithoutAnchors = (rawUrl: string): string => {
+export const RemoveAnchorsFromUrl = (rawUrl: string): string => {
   try {
     const parsed = new URL(rawUrl);
     parsed.hash = "";
@@ -101,7 +110,7 @@ export function deriveSubPath(
     if (!relPath.startsWith("/")) relPath = "/" + relPath;
   }
   if (
-    UrlWithoutAnchors(targetUrl) === sanitizedBaseUrl ||
+    RemoveAnchorsFromUrl(targetUrl) === sanitizedBaseUrl ||
     relPath === "/" ||
     relPath === ""
   ) {

apps/parser/src/modules/config.ts

@@ -1,7 +1,7 @@
 import path from "node:path";
 import { EnvConfig } from "./types";
 import {
-  UrlWithoutAnchors,
+  RemoveAnchorsFromUrl,
   sanitizeUrlAsFilename,
 } from "../helpers/url-handling";
 import * as dotenv from "dotenv";
@@ -27,7 +27,7 @@ export function resolveEnv(): EnvConfig {
       "Missing required URL. Set URL in environment or .env file.",
     );
   }
-  const sanitizedBaseUrl = UrlWithoutAnchors(baseUrl);
+  const sanitizedBaseUrl = RemoveAnchorsFromUrl(baseUrl);
   const parsedDepth = Number.parseInt(depth ?? `${DEFAULT_DEPTH}`, 10);
   const maxDepth = Number.isNaN(parsedDepth)
     ? DEFAULT_DEPTH

apps/parser/src/modules/crawler.ts

@@ -1,8 +1,12 @@
 import { Browser } from "puppeteer";
 import { ParsedNode, ParsedMetadata } from "./types";
-import { UrlWithoutAnchors } from "../helpers/url-handling";
+import { RemoveAnchorsFromUrl } from "../helpers/url-handling";
 import { expandInteractiveSections } from "./dom-actions";
 
+const PAGE_NAVIGATION_OPTIONS = {
+  waitUntil: "networkidle2" as const,
+};
+
 export async function exploreAndParsePages(
   browser: Browser,
   node: ParsedNode,
@@ -25,7 +29,7 @@ export async function exploreAndParsePages(
   if (parsedPages.has(visitKey) || depth > maxDepth) {
     return;
   }
-  const normalizedUrl = UrlWithoutAnchors(node.url);
+  const normalizedUrl = RemoveAnchorsFromUrl(node.url);
   if (!isWithinScope(normalizedUrl, baseScope, validDomainVariants)) {
     return;
   }
@@ -43,7 +47,7 @@ export async function exploreAndParsePages(
   try {
     page = await browser.newPage();
     await page.goto(node.url, {
-      waitUntil: "networkidle2",
+      ...PAGE_NAVIGATION_OPTIONS,
       timeout: navigationTimeout,
     });
     await expandInteractiveSections(page);
@@ -90,7 +94,7 @@ export async function exploreAndParsePages(
   const nextChildren: ParsedNode[] = [];
   let newLinksCount = 0;
   for (const href of anchors) {
-    const normalized = UrlWithoutAnchors(href);
+    const normalized = RemoveAnchorsFromUrl(href);
     const visitCandidate = buildVisitKey(normalized);
     if (parsedPages.has(visitCandidate) || scheduledPages.has(visitCandidate))
       continue;
@@ -172,8 +176,9 @@ export function buildVisitKey(rawUrl: string): string {
     const url = new URL(rawUrl);
     url.hash = "";
     url.hostname = url.hostname.replace(/^www\./, "");
-    return UrlWithoutAnchors(url.toString());
-  } catch (_error) {
+    return RemoveAnchorsFromUrl(url.toString());
+  } catch (error) {
+    console.warn(`Failed to build visit key for URL: ${rawUrl}`, error);
     return rawUrl;
   }
 }

apps/parser/src/parser.ts

@@ -9,7 +9,7 @@ import { expandInteractiveSections } from "./modules/dom-actions";
 import { ParsedNode, ParsedMetadata } from "./modules/types";
 import {
   sanitizeUrlAsFilename,
-  UrlWithoutAnchors,
+  RemoveAnchorsFromUrl,
 } from "./helpers/url-handling";
 import { assertReachable } from "./modules/network";
 import { toIsoOrNull } from "./helpers/date-format";
@@ -18,7 +18,7 @@ puppeteer.use(StealthPlugin());
 
 const NAVIGATION_TIMEOUT_MS = 30_000;
 const REQUEST_TIMEOUT_MS = 10_000;
-const FILENAME_LENGTH_THRESHOLD = 255;
+const FILENAME_LENGTH_THRESHOLD = 250;
 
 const env = resolveEnv();
 const parsedPages = new Map<string, ParsedMetadata>();
@@ -32,7 +32,7 @@ void (async () => {
     const root: ParsedNode = { url: env.baseUrl };
     const baseUrlObject = new URL(env.baseUrl);
     const baseOrigin = baseUrlObject.origin;
-    const baseScope = UrlWithoutAnchors(env.baseUrl);
+    const baseScope = RemoveAnchorsFromUrl(env.baseUrl);
     const baseHostToken = baseUrlObject.hostname
       .replace(/^www\./, "")
       .toLowerCase();
@@ -72,7 +72,7 @@ async function parsePageFn(
     await expandInteractiveSections(page);
     const rawMetadata = await page.evaluate(extractDocumentMetadata);
     const snapshot = serializeMetadata(rawMetadata);
-    await persistSnapshot(snapshot, FILENAME_LENGTH_THRESHOLD);
+    await persistSnapshot(snapshot);
     return snapshot;
   } catch (error) {
     console.error(`Error while parsing ${url}:`, (error as Error).message);
@@ -84,10 +84,7 @@ async function parsePageFn(
   }
 }
 
-async function persistSnapshot(
-  snapshot: ParsedMetadata,
-  FILENAME_LENGTH_THRESHOLD: number,
-): Promise<void> {
+async function persistSnapshot(snapshot: ParsedMetadata): Promise<void> {
   const finalName = sanitizeUrlAsFilename(snapshot.url, {
     replacement: "-",
     lengthThreshold: FILENAME_LENGTH_THRESHOLD,