Refactor parser configuration: update maxDepth to allow null for unlimited depth, implement base scope handling, and enhance URL sanitization functions for Directory names

Author: anemone008

apps/parser/.env.default

@@ -2,8 +2,8 @@
 # Root URL to start parsing from
 URL="https://example.com"
 
-# Maximum recursion depth (integer)
-DEPTH=2
+# Maximum recursion depth (integer or null for unlimited)
+DEPTH=null
 
 # Name of the vector index bucket/folder where parsed artifacts are stored
 CHB_INDEX_ID="parser-vector-index-name"

apps/parser/README.md

@@ -41,7 +41,7 @@ Create a `.env` file in the `apps/parser` directory with the following content:
 ```
 URL=https://example.com
 CHB_INDEX_ID=name_of_your_choice
-# DEPTH=2  # Optional, defaults to 2
+# DEPTH=2  # Optional, defaults to null
 ```
 
 #### b) Using command line variables

apps/parser/src/helpers/url-handling.ts

@@ -8,6 +8,12 @@ const WINDOWS_RESERVED_RE = /^(con|prn|aux|nul|com[0-9]|lpt[0-9])$/i;
 const WINDOWS_TRAILING_RE = /[\. ]+$/;
 const DEFAULT_REPLACEMENT = "-";
 
+let BASE_SCOPE: string;
+
+export function setBaseScope(scope: string): void {
+  BASE_SCOPE = scope;
+}
+
 export function sanitizeUrlAsFilename(
   url: string,
   options?: SanitizeOptions,
@@ -19,17 +25,53 @@ export function sanitizeUrlAsFilename(
     return DEFAULT_REPLACEMENT;
   }
   let filenameBase = url;
-  try {
-    const urlObj = new URL(url);
-    const pathAndSearch = `${urlObj.pathname}${urlObj.search}`;
-    if (pathAndSearch === "/" || pathAndSearch === "") {
-      filenameBase = urlObj.hostname;
+  if (filenameBase === BASE_SCOPE) {
+    filenameBase = new URL(filenameBase).hostname.replace(/^www\./, "");
+  } else {
+    const pathAndSearch = url.replace(BASE_SCOPE, "").replace(/^\/+/, "");
+    if (!pathAndSearch || pathAndSearch === "/") {
+      filenameBase = url.split("/").filter(Boolean).pop() || url;
     } else {
       filenameBase = pathAndSearch;
     }
-  } catch (_error) {
-    // If it's not a valid URL, use as-is
   }
+
+  const replacement = validReplacementOrDefault(
+    options?.replacement ?? DEFAULT_REPLACEMENT,
+  );
+  let sanitized = filenameBase
+    .replace(/\/$/, "")
+    .replace(ILLEGAL_RE, replacement)
+    .replace(CONTROL_RE, replacement)
+    .replace(RESERVED_RE, replacement)
+    .replace(WINDOWS_RESERVED_RE, replacement)
+    .replace(WINDOWS_TRAILING_RE, replacement)
+    .trim();
+  if (sanitized.length === 0) {
+    return replacement;
+  }
+  const trimmedName = sanitized.replace(/^[-_]+/, "") || sanitized;
+  if (
+    options?.lengthThreshold &&
+    trimmedName.length > options.lengthThreshold
+  ) {
+    const hash = crypto
+      .createHash("sha1")
+      .update(url)
+      .digest("hex")
+      .slice(0, 9);
+    const prefix = trimmedName.slice(0, options.lengthThreshold - 10);
+    return `${prefix}_${hash}`;
+  }
+  return trimmedName;
+}
+
+export function sanitizeUrlAsDirectoryName(
+  url: string,
+  options?: SanitizeOptions,
+): string {
+  let filenameBase = url;
+  filenameBase = filenameBase.replace(/^(https?:\/\/)?(www\.)?/, "");
   const replacement = validReplacementOrDefault(
     options?.replacement ?? DEFAULT_REPLACEMENT,
   );

apps/parser/src/main.ts

@@ -5,7 +5,11 @@ import { ensureDirectory } from "./modules/output";
 import { handleError } from "./modules/errors";
 import { exploreAndParsePages } from "./modules/parser";
 import { ParsedNode, ParsedMetadata } from "./modules/types";
-import { RemoveAnchorsFromUrl, buildVisitKey } from "./helpers/url-handling";
+import {
+  RemoveAnchorsFromUrl,
+  buildVisitKey,
+  setBaseScope,
+} from "./helpers/url-handling";
 import { assertReachable } from "./modules/network";
 
 puppeteer.use(StealthPlugin());
@@ -21,6 +25,8 @@ export const BASE_HOST_TOKEN = new URL(env.baseUrl).hostname
   .toLowerCase();
 export const VALID_DOMAIN_VARIANTS = env.validDomainVariants || [];
 
+setBaseScope(BASE_SCOPE);
+
 void (async () => {
   try {
     await assertReachable(env.baseUrl);

apps/parser/src/modules/config.ts

@@ -2,11 +2,11 @@ import path from "node:path";
 import { EnvConfig } from "./types";
 import {
   RemoveAnchorsFromUrl,
-  sanitizeUrlAsFilename,
+  sanitizeUrlAsDirectoryName,
 } from "../helpers/url-handling";
 import * as dotenv from "dotenv";
 
-const DEFAULT_DEPTH = 2;
+const DEFAULT_DEPTH = null;
 
 export function resolveEnv(): EnvConfig {
   let baseUrl = process.env.URL?.trim();
@@ -60,7 +60,7 @@ function generateOutputDirectoryPath(
   vectorIndexName: string | undefined,
   sanitizedBaseUrl: string,
 ): string {
-  const safeBaseSegment = sanitizeUrlAsFilename(sanitizedBaseUrl, {
+  const safeBaseSegment = sanitizeUrlAsDirectoryName(sanitizedBaseUrl, {
     replacement: "-",
   });
   if (!vectorIndexName) {

apps/parser/src/modules/output.ts

@@ -18,7 +18,7 @@ export async function persistSnapshot(
   let finalName = sanitizeUrlAsFilename(snapshot.url, {
     lengthThreshold: FILENAME_LENGTH_THRESHOLD,
   });
-  if (finalName === BASE_HOST_TOKEN) {
+  if (finalName.replace(/^www\./, "") === BASE_HOST_TOKEN) {
     finalName = "index";
   }
   await saveMetadata(outputDirectory, `${finalName}.json`, snapshot);

apps/parser/src/modules/parser.ts

@@ -33,7 +33,7 @@ export async function exploreAndParsePages(
 ): Promise<void> {
   const visitKey = buildVisitKey(node.url);
   scheduledPages.delete(visitKey);
-  if (parsedPages.has(visitKey) || depth > MAX_DEPTH) {
+  if (parsedPages.has(visitKey) || (MAX_DEPTH !== null && depth > MAX_DEPTH)) {
     return;
   }
   const normalizedUrl = RemoveAnchorsFromUrl(node.url);
@@ -125,7 +125,7 @@ export async function exploreAndParsePages(
       parsedPages.size
     }/${totalKnown} (${((parsedPages.size / totalKnown) * 100).toFixed(2)}%)`,
   );
-  if (!node.children || depth >= MAX_DEPTH) return;
+  if (!node.children || (MAX_DEPTH !== null && depth >= MAX_DEPTH)) return;
   for (const child of node.children) {
     await exploreAndParsePages(
       browser,

apps/parser/src/modules/types.ts

@@ -2,7 +2,7 @@ export type EnvConfig = {
   readonly baseUrl: string;
   readonly sanitizedBaseUrl: string;
   readonly outputDirectory: string;
-  readonly maxDepth: number;
+  readonly maxDepth: number | null;
   readonly validDomainVariants?: string[];
 };