Refactor code for consistency and readability; update imports, format code and add new helper for dates

Author: anemone008

apps/parser/jest.config.ts

@@ -1,12 +1,12 @@
-import type { Config } from 'jest';
+import type { Config } from "jest";
 
 const config: Config = {
   rootDir: __dirname,
-  testRegex: 'tests/.*\\.test\\.ts$',
+  testRegex: "tests/.*\\.test\\.ts$",
   transform: {
-    '^.+\\.ts$': ['ts-jest', { tsconfig: 'tsconfig.json' }],
+    "^.+\\.ts$": ["ts-jest", { tsconfig: "tsconfig.json" }],
   },
-  testEnvironment: 'node',
+  testEnvironment: "node",
   clearMocks: true,
   verbose: false,
 };

apps/parser/src/helpers/date-format.ts

@@ -0,0 +1,7 @@
+export function toIsoOrNull(value: string | null): string | null {
+  if (!value) {
+    return null;
+  }
+  const date = new Date(value);
+  return Number.isNaN(date.getTime()) ? null : date.toISOString();
+}

apps/parser/src/helpers/url-handling.ts

@@ -1,62 +1,84 @@
+import { SanitizeOptions } from "../modules/types";
+
 const ILLEGAL_RE = /[\/\?<>\\:\*\|"]/g;
 const CONTROL_RE = /[\x00-\x1f\x80-\x9f]/g;
 const RESERVED_RE = /^\.+$/;
 const WINDOWS_RESERVED_RE = /^(con|prn|aux|nul|com[0-9]|lpt[0-9])$/i;
 const WINDOWS_TRAILING_RE = /[\. ]+$/;
+const DEFAULT_REPLACEMENT = "-";
 
-export type SanitizeOptions = {
-  readonly replacement?: string;
-};
-
-const DEFAULT_REPLACEMENT = '-';
-
-export function sanitizeUrlAsFilename(input: string, options?: SanitizeOptions): string {
+export function sanitizeUrlAsFilename(
+  input: string,
+  options?: SanitizeOptions,
+): string {
   if (!input) {
     return DEFAULT_REPLACEMENT;
   }
-
-  const replacement = validReplacementOrDefault(options?.replacement ?? DEFAULT_REPLACEMENT);
+  const replacement = validReplacementOrDefault(
+    options?.replacement ?? DEFAULT_REPLACEMENT,
+  );
   let sanitized = input
     .replace(ILLEGAL_RE, replacement)
     .replace(CONTROL_RE, replacement)
     .replace(RESERVED_RE, replacement)
     .replace(WINDOWS_RESERVED_RE, replacement)
     .replace(WINDOWS_TRAILING_RE, replacement)
     .trim();
-
   if (sanitized.length === 0) {
     return replacement;
   }
-
   return sanitized.slice(0, 255);
 }
 
 function validReplacementOrDefault(candidate: string): string {
   if (!candidate) {
     return DEFAULT_REPLACEMENT;
   }
-
-  if (/[\/\?<>\\:\*\|"]/u.test(candidate) || /[\x00-\x1f\x80-\x9f]/u.test(candidate)) {
+  if (
+    /[\/\?<>\\:\*\|"]/u.test(candidate) ||
+    /[\x00-\x1f\x80-\x9f]/u.test(candidate)
+  ) {
     return DEFAULT_REPLACEMENT;
   }
-
   return candidate;
 }
 
 export const UrlWithoutAnchors = (rawUrl: string): string => {
-try {
+  try {
     const parsed = new URL(rawUrl);
-    parsed.hash = '';
-    if (parsed.pathname.length > 1 && parsed.pathname.endsWith('/')) {
-    parsed.pathname = parsed.pathname.slice(0, -1);
+    parsed.hash = "";
+    if (parsed.pathname.length > 1 && parsed.pathname.endsWith("/")) {
+      parsed.pathname = parsed.pathname.slice(0, -1);
     }
     const serialized = parsed.toString();
-    if (parsed.pathname === '/' && !parsed.search) {
-    return serialized.endsWith('/') ? serialized.slice(0, -1) : serialized;
+    if (parsed.pathname === "/" && !parsed.search) {
+      return serialized.endsWith("/") ? serialized.slice(0, -1) : serialized;
     }
     return serialized;
-} catch (error) {
+  } catch (error) {
     console.warn(`Failed to parse URL: ${rawUrl}`, error);
     return rawUrl;
-}
+  }
 };
+
+export function deriveSubPath(
+  targetUrl: string,
+  baseUrl: string,
+  sanitizedBaseUrl: string,
+): string {
+  const base = new URL(baseUrl);
+  const target = new URL(targetUrl);
+  let relPath = target.pathname;
+  if (base.pathname !== "/" && relPath.startsWith(base.pathname)) {
+    relPath = relPath.slice(base.pathname.length);
+    if (!relPath.startsWith("/")) relPath = "/" + relPath;
+  }
+  if (
+    UrlWithoutAnchors(targetUrl) === sanitizedBaseUrl ||
+    relPath === "/" ||
+    relPath === ""
+  ) {
+    return "/";
+  }
+  return `${relPath}${target.search}${target.hash}` || "/";
+}

apps/parser/src/modules/config.ts

@@ -1,51 +1,50 @@
-
-import path from 'node:path';
-import { UrlWithoutAnchors, sanitizeUrlAsFilename } from '../helpers/url-handling';
-import * as dotenv from 'dotenv';
-
-export type EnvConfig = {
-  readonly baseUrl: string;
-  readonly sanitizedBaseUrl: string;
-  readonly outputDirectory: string;
-  readonly maxDepth: number;
-};
-
+import path from "node:path";
+import { EnvConfig } from "./types";
+import {
+  UrlWithoutAnchors,
+  sanitizeUrlAsFilename,
+} from "../helpers/url-handling";
+import * as dotenv from "dotenv";
 
 const DEFAULT_DEPTH = 2;
 
-
 export function resolveEnv(): EnvConfig {
   let baseUrl = process.env.URL?.trim();
   let depth = process.env.DEPTH?.trim();
   let vectorIndexName = process.env.CHB_INDEX_ID?.trim();
-  
   if (!baseUrl || !depth || !vectorIndexName) {
-    const parserHome = path.resolve(__dirname, '../../');
-    dotenv.config({ path: path.join(parserHome, '.env') });
+    const parserHome = path.resolve(__dirname, "../../");
+    dotenv.config({ path: path.join(parserHome, ".env") });
     baseUrl = baseUrl || process.env.URL?.trim();
     depth = depth || process.env.DEPTH?.trim();
     vectorIndexName = vectorIndexName || process.env.CHB_INDEX_ID?.trim();
   }
-  
   if (!baseUrl) {
-    throw new Error('Missing required URL. Set URL in environment or .env file.');
+    throw new Error(
+      "Missing required URL. Set URL in environment or .env file.",
+    );
   }
-  
   const sanitizedBaseUrl = UrlWithoutAnchors(baseUrl);
   const parsedDepth = Number.parseInt(depth ?? `${DEFAULT_DEPTH}`, 10);
-  const maxDepth = Number.isNaN(parsedDepth) ? DEFAULT_DEPTH : Math.max(parsedDepth, 0);
-  const outputDirectory = generateOutputDirectoryPath(vectorIndexName, sanitizedBaseUrl);
-  
+  const maxDepth = Number.isNaN(parsedDepth)
+    ? DEFAULT_DEPTH
+    : Math.max(parsedDepth, 0);
+  const outputDirectory = generateOutputDirectoryPath(
+    vectorIndexName,
+    sanitizedBaseUrl,
+  );
   return { baseUrl, sanitizedBaseUrl, outputDirectory, maxDepth };
 }
 
 function generateOutputDirectoryPath(
   vectorIndexName: string | undefined,
-  sanitizedBaseUrl: string
+  sanitizedBaseUrl: string,
 ): string {
-  const safeBaseSegment = sanitizeUrlAsFilename(sanitizedBaseUrl, { replacement: '_' });
+  const safeBaseSegment = sanitizeUrlAsFilename(sanitizedBaseUrl, {
+    replacement: "_",
+  });
   if (!vectorIndexName) {
     return `output/${safeBaseSegment}`;
   }
-  return path.join(vectorIndexName, 'parsing', safeBaseSegment);
+  return path.join(vectorIndexName, "parsing", safeBaseSegment);
 }

apps/parser/src/modules/crawler.ts

@@ -1,54 +1,58 @@
-import { Browser } from 'puppeteer';
-import { ParseNode, ParseMetadata } from './types';
-import { UrlWithoutAnchors } from '../helpers/url-handling';
-import { expandInteractiveSections } from './dom-actions';
+import { Browser } from "puppeteer";
+import { ParsedNode, ParsedMetadata } from "./types";
+import { UrlWithoutAnchors } from "../helpers/url-handling";
+import { expandInteractiveSections } from "./dom-actions";
 
 export async function parsePages(
   browser: Browser,
-  node: ParseNode,
+  node: ParsedNode,
   depth: number,
   maxDepth: number,
-  parsedPages: Map<string, ParseMetadata>,
-  parsePageFn: (browser: Browser, url: string) => Promise<ParseMetadata | null>,
+  parsedPages: Map<string, ParsedMetadata>,
+  parsePageFn: (
+    browser: Browser,
+    url: string,
+  ) => Promise<ParsedMetadata | null>,
   baseOrigin: string,
   baseScope: string,
   baseHostToken: string,
-  navigationTimeout = 30000
+  navigationTimeout = 30000,
 ): Promise<void> {
   const visitKey = buildVisitKey(node.url);
   if (parsedPages.has(visitKey) || depth > maxDepth) {
     return;
   }
-
   const normalizedUrl = UrlWithoutAnchors(node.url);
   if (!isWithinScope(normalizedUrl, baseScope, baseHostToken)) {
     return;
   }
-
   const metadata = await parsePageFn(browser, node.url);
   if (!metadata) return;
-
   parsedPages.set(visitKey, metadata);
   node.title = metadata.title;
   node.bodyText = metadata.bodyText;
   node.lang = metadata.lang;
   node.keywords = metadata.keywords;
   node.datePublished = metadata.datePublished;
   node.lastModified = metadata.lastModified;
-
   let page;
   let anchors: string[] = [];
   try {
     page = await browser.newPage();
-    await page.goto(node.url, { waitUntil: 'networkidle2', timeout: navigationTimeout });
+    await page.goto(node.url, {
+      waitUntil: "networkidle2",
+      timeout: navigationTimeout,
+    });
     await expandInteractiveSections(page);
-    anchors = await page.evaluate((allowedToken: string) => {
-      const anchors = Array.from(document.querySelectorAll('a[href]'));
-      const iframeSources = Array.from(document.querySelectorAll('iframe[src]'));
+    anchors = (await page.evaluate((allowedToken: string) => {
+      const anchors = Array.from(document.querySelectorAll("a[href]"));
+      const iframeSources = Array.from(
+        document.querySelectorAll("iframe[src]"),
+      );
       const unique = new Set<string>();
       for (const anchor of anchors) {
         const href = (anchor as HTMLAnchorElement).href;
-        if (!href || !href.startsWith('http')) continue;
+        if (!href || !href.startsWith("http")) continue;
         try {
           const target = new URL(href, window.location.href);
           const normalizedHref = target.href.toLowerCase();
@@ -59,10 +63,9 @@ export async function parsePages(
           console.warn(`Failed to parse anchor href: ${href}`, error);
         }
       }
-
       for (const frame of iframeSources) {
         const src = (frame as HTMLIFrameElement).src;
-        if (!src || !src.startsWith('http')) {
+        if (!src || !src.startsWith("http")) {
           continue;
         }
         try {
@@ -75,20 +78,19 @@ export async function parsePages(
         }
       }
       return Array.from(unique);
-    }, baseHostToken) as string[];
+    }, baseHostToken)) as string[];
   } catch (error) {
     console.warn(`Failed to extract anchors from ${node.url}`, error);
   } finally {
     if (page) await page.close();
   }
-
-
   const scheduled = new Set<string>();
-  const nextChildren: ParseNode[] = [];
+  const nextChildren: ParsedNode[] = [];
   for (const href of anchors) {
     const normalized = UrlWithoutAnchors(href);
     const visitCandidate = buildVisitKey(href);
-    if (parsedPages.has(visitCandidate) || scheduled.has(visitCandidate)) continue;
+    if (parsedPages.has(visitCandidate) || scheduled.has(visitCandidate))
+      continue;
     const lowerNormalized = normalized.toLowerCase();
     if (baseHostToken && !lowerNormalized.includes(baseHostToken)) {
       continue;
@@ -100,7 +102,6 @@ export async function parsePages(
     nextChildren.push({ url: href });
   }
   node.children = nextChildren;
-
   if (!node.children || depth >= maxDepth) return;
   for (const child of node.children) {
     await parsePages(
@@ -112,7 +113,7 @@ export async function parsePages(
       parsePageFn,
       baseOrigin,
       baseScope,
-      baseHostToken
+      baseHostToken,
     );
   }
 }
@@ -133,13 +134,13 @@ function isWithinScope(url: string, scope: string, hostToken: string): boolean {
     return false;
   }
   const nextChar = lowerUrl.charAt(lowerScope.length);
-  return nextChar === '/' || nextChar === '?' || nextChar === '#';
+  return nextChar === "/" || nextChar === "?" || nextChar === "#";
 }
 
 export function buildVisitKey(rawUrl: string): string {
   try {
     const url = new URL(rawUrl);
-    url.hash = '';
+    url.hash = "";
     return UrlWithoutAnchors(url.toString());
   } catch (_error) {
     return rawUrl;