Rename parsePages to exploreAndParsePages. Expand sanitizeUrlAsFilename function so that for root it returns the hostname

Author: anemone008

apps/parser/src/helpers/url-handling.ts

@@ -18,7 +18,12 @@ export function sanitizeUrlAsFilename(
   let filenameBase = url;
   try {
     const urlObj = new URL(url);
-    filenameBase = `${urlObj.pathname}${urlObj.search}`;
+    const pathAndSearch = `${urlObj.pathname}${urlObj.search}`;
+    if (pathAndSearch === "/" || pathAndSearch === "") {
+      filenameBase = urlObj.hostname;
+    } else {
+      filenameBase = pathAndSearch;
+    }
   } catch (_error) {
     // If it's not a valid URL, use as-is
   }

apps/parser/src/modules/config.ts

@@ -61,7 +61,7 @@ function generateOutputDirectoryPath(
   sanitizedBaseUrl: string,
 ): string {
   const safeBaseSegment = sanitizeUrlAsFilename(sanitizedBaseUrl, {
-    replacement: "_",
+    replacement: "-",
   });
   if (!vectorIndexName) {
     return `output/${safeBaseSegment}`;

apps/parser/src/modules/crawler.ts

@@ -3,7 +3,7 @@ import { ParsedNode, ParsedMetadata } from "./types";
 import { UrlWithoutAnchors } from "../helpers/url-handling";
 import { expandInteractiveSections } from "./dom-actions";
 
-export async function parsePages(
+export async function exploreAndParsePages(
   browser: Browser,
   node: ParsedNode,
   depth: number,
@@ -26,9 +26,7 @@ export async function parsePages(
     return;
   }
   const normalizedUrl = UrlWithoutAnchors(node.url);
-  if (
-    !isWithinScope(normalizedUrl, baseScope, validDomainVariants)
-  ) {
+  if (!isWithinScope(normalizedUrl, baseScope, validDomainVariants)) {
     return;
   }
   const metadata = await parsePageFn(browser, node.url);
@@ -100,9 +98,7 @@ export async function parsePages(
     if (baseHostToken && !lowerNormalized.includes(baseHostToken)) {
       continue;
     }
-    if (
-      !isWithinScope(normalized, baseScope, validDomainVariants)
-    ) {
+    if (!isWithinScope(normalized, baseScope, validDomainVariants)) {
       continue;
     }
     scheduledPages.add(visitCandidate);
@@ -120,7 +116,7 @@ export async function parsePages(
   );
   if (!node.children || depth >= maxDepth) return;
   for (const child of node.children) {
-    await parsePages(
+    await exploreAndParsePages(
       browser,
       child,
       depth + 1,

apps/parser/src/parser.ts

@@ -4,7 +4,7 @@ import { Browser, Page } from "puppeteer";
 import { resolveEnv } from "./modules/config";
 import { ensureDirectory, saveMetadata } from "./modules/output";
 import { handleError } from "./modules/errors";
-import { buildVisitKey, parsePages } from "./modules/crawler";
+import { buildVisitKey, exploreAndParsePages } from "./modules/crawler";
 import { expandInteractiveSections } from "./modules/dom-actions";
 import { ParsedNode, ParsedMetadata } from "./modules/types";
 import {
@@ -37,7 +37,7 @@ void (async () => {
       .replace(/^www\./, "")
       .toLowerCase();
     scheduledPages.add(buildVisitKey(env.baseUrl));
-    await parsePages(
+    await exploreAndParsePages(
       browser,
       root,
       0,