Enhance URL handling and configuration:

- Update sanitizeUrlAsFilename to support length threshold and hash suffix for long URLs.
- Modify resolveEnv to parse validDomainVariants from environment variables.
- Refactor parsePages to utilize validDomainVariants for scope checking.
- Update EnvConfig type to include validDomainVariants.
- Improve tests for sanitizeUrlAsFilename to cover new functionality.

Author: anemone008

apps/parser/src/helpers/url-handling.ts

@@ -1,4 +1,5 @@
 import { SanitizeOptions } from "../modules/types";
+import crypto from "crypto";
 
 const ILLEGAL_RE = /[\/\?<>\\:\*\|"]/g;
 const CONTROL_RE = /[\x00-\x1f\x80-\x9f]/g;
@@ -8,16 +9,24 @@ const WINDOWS_TRAILING_RE = /[\. ]+$/;
 const DEFAULT_REPLACEMENT = "-";
 
 export function sanitizeUrlAsFilename(
-  input: string,
+  url: string,
   options?: SanitizeOptions,
 ): string {
-  if (!input) {
+  if (!url) {
     return DEFAULT_REPLACEMENT;
   }
+  let filenameBase = url;
+  try {
+    const urlObj = new URL(url);
+    filenameBase = `${urlObj.pathname}${urlObj.search}`;
+  } catch (_error) {
+    // If it's not a valid URL, use as-is
+  }
   const replacement = validReplacementOrDefault(
     options?.replacement ?? DEFAULT_REPLACEMENT,
   );
-  let sanitized = input
+  let sanitized = filenameBase
+    .replace(/\/$/, "")
     .replace(ILLEGAL_RE, replacement)
     .replace(CONTROL_RE, replacement)
     .replace(RESERVED_RE, replacement)
@@ -27,7 +36,20 @@ export function sanitizeUrlAsFilename(
   if (sanitized.length === 0) {
     return replacement;
   }
-  return sanitized.slice(0, 255);
+  const trimmedName = sanitized.replace(/^[-_]+/, "") || sanitized;
+  if (
+    options?.lengthThreshold &&
+    trimmedName.length > options.lengthThreshold
+  ) {
+    const hash = crypto
+      .createHash("sha1")
+      .update(url)
+      .digest("hex")
+      .slice(0, 9);
+    const prefix = trimmedName.slice(0, options.lengthThreshold - 10);
+    return `${prefix}_${hash}`;
+  }
+  return trimmedName;
 }
 
 function validReplacementOrDefault(candidate: string): string {

apps/parser/src/modules/config.ts

@@ -12,12 +12,15 @@ export function resolveEnv(): EnvConfig {
   let baseUrl = process.env.URL?.trim();
   let depth = process.env.DEPTH?.trim();
   let vectorIndexName = process.env.CHB_INDEX_ID?.trim();
+  let validDomainVariants = process.env.validDomainVariants?.trim();
   if (!baseUrl || !depth || !vectorIndexName) {
     const parserHome = path.resolve(__dirname, "../../");
     dotenv.config({ path: path.join(parserHome, ".env") });
     baseUrl = baseUrl || process.env.URL?.trim();
     depth = depth || process.env.DEPTH?.trim();
     vectorIndexName = vectorIndexName || process.env.CHB_INDEX_ID?.trim();
+    validDomainVariants =
+      validDomainVariants || process.env.validDomainVariants?.trim();
   }
   if (!baseUrl) {
     throw new Error(
@@ -33,7 +36,24 @@ export function resolveEnv(): EnvConfig {
     vectorIndexName,
     sanitizedBaseUrl,
   );
-  return { baseUrl, sanitizedBaseUrl, outputDirectory, maxDepth };
+  let parsedValidDomainVariants: string[] = [];
+  if (validDomainVariants) {
+    try {
+      parsedValidDomainVariants = JSON.parse(validDomainVariants);
+      if (!Array.isArray(parsedValidDomainVariants)) {
+        parsedValidDomainVariants = [];
+      }
+    } catch (_error) {
+      parsedValidDomainVariants = [];
+    }
+  }
+  return {
+    baseUrl,
+    sanitizedBaseUrl,
+    outputDirectory,
+    maxDepth,
+    validDomainVariants: parsedValidDomainVariants,
+  };
 }
 
 function generateOutputDirectoryPath(

apps/parser/src/modules/crawler.ts

@@ -17,6 +17,7 @@ export async function parsePages(
   baseOrigin: string,
   baseScope: string,
   baseHostToken: string,
+  validDomainVariants: string[] = [],
   navigationTimeout = 30000,
 ): Promise<void> {
   const visitKey = buildVisitKey(node.url);
@@ -25,7 +26,9 @@ export async function parsePages(
     return;
   }
   const normalizedUrl = UrlWithoutAnchors(node.url);
-  if (!isWithinScope(normalizedUrl, baseScope, baseHostToken)) {
+  if (
+    !isWithinScope(normalizedUrl, baseScope, validDomainVariants)
+  ) {
     return;
   }
   const metadata = await parsePageFn(browser, node.url);
@@ -90,19 +93,21 @@ export async function parsePages(
   let newLinksCount = 0;
   for (const href of anchors) {
     const normalized = UrlWithoutAnchors(href);
-    const visitCandidate = buildVisitKey(href);
+    const visitCandidate = buildVisitKey(normalized);
     if (parsedPages.has(visitCandidate) || scheduledPages.has(visitCandidate))
       continue;
     const lowerNormalized = normalized.toLowerCase();
     if (baseHostToken && !lowerNormalized.includes(baseHostToken)) {
       continue;
     }
-    if (!isWithinScope(normalized, baseScope, baseHostToken)) {
+    if (
+      !isWithinScope(normalized, baseScope, validDomainVariants)
+    ) {
       continue;
     }
     scheduledPages.add(visitCandidate);
     newLinksCount += 1;
-    nextChildren.push({ url: href });
+    nextChildren.push({ url: normalized });
   }
   node.children = nextChildren;
   const totalKnown = parsedPages.size + scheduledPages.size;
@@ -126,33 +131,50 @@ export async function parsePages(
       baseOrigin,
       baseScope,
       baseHostToken,
+      validDomainVariants,
     );
   }
 }
 
-function isWithinScope(url: string, scope: string, hostToken: string): boolean {
-  if (hostToken && url.toLowerCase().includes(hostToken)) {
-    return true;
-  }
+function isWithinScope(
+  url: string,
+  scope: string,
+  validDomainVariants: string[] = [],
+): boolean {
   if (!scope) {
     return true;
   }
-  const lowerUrl = url.toLowerCase();
-  const lowerScope = scope.toLowerCase();
-  if (lowerUrl === lowerScope) {
-    return true;
-  }
-  if (!lowerUrl.startsWith(lowerScope)) {
+  try {
+    const urlObj = new URL(url);
+    const scopeObj = new URL(scope);
+    const urlDomain = urlObj.hostname.replace(/^www\./, "");
+    const scopeDomain = scopeObj.hostname.replace(/^www\./, "");
+    if (urlDomain === scopeDomain) {
+      return true;
+    }
+    const urlParts = urlDomain.split(".");
+    const scopeParts = scopeDomain.split(".");
+    if (urlParts.length > scopeParts.length) {
+      const subdomain = urlParts[0];
+      const domainWithoutSubdomain = urlParts.slice(1).join(".");
+      if (
+        domainWithoutSubdomain === scopeDomain &&
+        validDomainVariants.includes(subdomain)
+      ) {
+        return true;
+      }
+    }
+    return false;
+  } catch (_error) {
     return false;
   }
-  const nextChar = lowerUrl.charAt(lowerScope.length);
-  return nextChar === "/" || nextChar === "?" || nextChar === "#";
 }
 
 export function buildVisitKey(rawUrl: string): string {
   try {
     const url = new URL(rawUrl);
     url.hash = "";
+    url.hostname = url.hostname.replace(/^www\./, "");
     return UrlWithoutAnchors(url.toString());
   } catch (_error) {
     return rawUrl;

apps/parser/src/modules/types.ts

@@ -3,10 +3,12 @@ export type EnvConfig = {
   readonly sanitizedBaseUrl: string;
   readonly outputDirectory: string;
   readonly maxDepth: number;
+  readonly validDomainVariants?: string[];
 };
 
 export type SanitizeOptions = {
   readonly replacement?: string;
+  readonly lengthThreshold?: number;
 };
 
 export type ParsedMetadata = {

apps/parser/src/parser.ts

@@ -10,11 +10,9 @@ import { ParsedNode, ParsedMetadata } from "./modules/types";
 import {
   sanitizeUrlAsFilename,
   UrlWithoutAnchors,
-  deriveSubPath,
 } from "./helpers/url-handling";
 import { assertReachable } from "./modules/network";
 import { toIsoOrNull } from "./helpers/date-format";
-import crypto from "crypto";
 
 puppeteer.use(StealthPlugin());
 
@@ -50,6 +48,7 @@ void (async () => {
       baseOrigin,
       baseScope,
       baseHostToken,
+      env.validDomainVariants || [],
       NAVIGATION_TIMEOUT_MS,
     );
     await browser.close();
@@ -89,27 +88,10 @@ async function persistSnapshot(
   snapshot: ParsedMetadata,
   FILENAME_LENGTH_THRESHOLD: number,
 ): Promise<void> {
-  const subPath = deriveSubPath(
-    snapshot.url,
-    env.baseUrl,
-    env.sanitizedBaseUrl,
-  );
-  const preferredName = subPath === "/" ? "root" : subPath;
-  const sanitizedName = sanitizeUrlAsFilename(preferredName, {
+  const finalName = sanitizeUrlAsFilename(snapshot.url, {
     replacement: "-",
+    lengthThreshold: FILENAME_LENGTH_THRESHOLD,
   });
-  const trimmedName = sanitizedName.replace(/^[-_]+/, "") || sanitizedName;
-  let finalName = trimmedName;
-  if (trimmedName.length > FILENAME_LENGTH_THRESHOLD) {
-    const normalizedUrl = UrlWithoutAnchors(snapshot.url);
-    const hash = crypto
-      .createHash("sha1")
-      .update(normalizedUrl)
-      .digest("hex")
-      .slice(0, 10);
-    const prefix = trimmedName.slice(0, 240);
-    finalName = `${prefix}_${hash}`;
-  }
   await saveMetadata(env.outputDirectory, `${finalName}.json`, snapshot);
 }
 

apps/parser/tests/parser.sanitize-filename.test.ts

@@ -1,15 +1,21 @@
 import { sanitizeUrlAsFilename } from "../src/helpers/url-handling";
 
-describe("sanitizeFilename", () => {
-  it("replaces illegal characters", () => {
-    expect(sanitizeUrlAsFilename("file/name?with*illegal|chars.txt")).toBe(
-      "file-name-with-illegal-chars.txt",
+describe("sanitizeUrlAsFilename", () => {
+  it("sanitizes URL paths with query strings and fragments", () => {
+    expect(
+      sanitizeUrlAsFilename("/docs/guide?version=1.0&lang=en#section"),
+    ).toBe("docs-guide-version=1.0&lang=en#section");
+  });
+
+  it("sanitizes URL paths with special characters", () => {
+    expect(sanitizeUrlAsFilename("/api/v1/users:profile")).toBe(
+      "api-v1-users-profile",
     );
   });
 
-  it("replaces control characters", () => {
-    expect(sanitizeUrlAsFilename("file\u0000name\u0001.txt")).toBe(
-      "file-name-.txt",
+  it("sanitizes control characters in URL paths", () => {
+    expect(sanitizeUrlAsFilename("/path\u0000with\u0001control.html")).toBe(
+      "path-with-control.html",
     );
   });
 
@@ -41,46 +47,60 @@ describe("sanitizeFilename", () => {
     expect(sanitizeUrlAsFilename(undefined as unknown as string)).toBe("-");
   });
 
-  it("uses the replacement option", () => {
-    expect(sanitizeUrlAsFilename("file/name", { replacement: "-" })).toBe(
-      "file-name",
+  it("uses custom replacement option for URL paths", () => {
+    expect(sanitizeUrlAsFilename("/api/users/123", { replacement: "_" })).toBe(
+      "api_users_123",
     );
   });
 
-  it("falls back to default replacement for invalid replacement", () => {
-    expect(sanitizeUrlAsFilename("file/name", { replacement: "/" })).toBe(
-      "file-name",
-    );
-    expect(sanitizeUrlAsFilename("file/name", { replacement: "\u0000" })).toBe(
-      "file-name",
-    );
+  it("falls back to default replacement for invalid replacement chars", () => {
+    expect(
+      sanitizeUrlAsFilename("/docs/guide?ref=x", { replacement: "/" }),
+    ).toBe("docs-guide-ref=x");
+    expect(
+      sanitizeUrlAsFilename("/docs/guide", { replacement: "\u0000" }),
+    ).toBe("docs-guide");
   });
 
-  it("limits output to 255 characters", () => {
-    const longName = "a".repeat(300) + ".txt";
-    expect(sanitizeUrlAsFilename(longName).length).toBe(255);
+  it("appends hash suffix for URLs exceeding length threshold", () => {
+    const longUrl = "/docs/" + "very-long-path-segment/".repeat(20);
+    const result = sanitizeUrlAsFilename(longUrl, {
+      lengthThreshold: 255
+    });
+    expect(result.length).toBe(255);
   });
 
-  it("returns replacement for input with only illegal/control chars", () => {
-    expect(sanitizeUrlAsFilename("\u0000\u0001\u0002")).toBe("---");
-    expect(sanitizeUrlAsFilename("////")).toBe("----");
+  it("trims leading dashes from URL paths", () => {
+    expect(sanitizeUrlAsFilename("/////path")).toBe("path");
   });
 
-  it("returns unchanged valid filename", () => {
-    expect(sanitizeUrlAsFilename("valid-filename.txt")).toBe(
-      "valid-filename.txt",
+  it("returns unchanged valid URL path", () => {
+    expect(sanitizeUrlAsFilename("api-reference.html")).toBe(
+      "api-reference.html",
     );
   });
 
-  it("ignores illegal replacement chars", () => {
-    expect(sanitizeUrlAsFilename("file/name", { replacement: "*" })).toBe(
-      "file-name",
+  it("ignores illegal characters in replacement option", () => {
+    expect(sanitizeUrlAsFilename("/docs/guide", { replacement: "*" })).toBe(
+      "docs-guide",
+    );
+  });
+
+  it("ignores control chars in replacement option", () => {
+    expect(
+      sanitizeUrlAsFilename("/docs/guide", { replacement: "\u0000" }),
+    ).toBe("docs-guide");
+  });
+
+  it("sanitizes URL with subdomain path", () => {
+    expect(sanitizeUrlAsFilename("showcase.example.com/docs-guide")).toBe(
+      "showcase.example.com-docs-guide",
     );
   });
 
-  it("ignores control char replacement", () => {
-    expect(sanitizeUrlAsFilename("file/name", { replacement: "\u0000" })).toBe(
-      "file-name",
+  it("handles URL with numbers and hyphens", () => {
+    expect(sanitizeUrlAsFilename("/api/v2/resource-123")).toBe(
+      "api-v2-resource-123",
     );
   });
 });