[CAI-749] Local parser

.changeset/wide-hairs-fail.md

@@ -0,0 +1,5 @@
+---
+"parser": major
+---
+
+Add script to perform parsing to parser app. Store parsed information locally. Sanitize urls in filesystem compatible format to use as file names.

apps/parser/.env.default

@@ -0,0 +1,9 @@
+# Default environment variables for the parser CLI
+# Root URL to start parsing from
+URL="https://example.com"
+
+# Maximum recursion depth (integer)
+DEPTH=2
+
+# Name of the vector index bucket/folder where parsed artifacts are stored
+CHB_INDEX_ID="parser-vector-index-name"

apps/parser/README.md

@@ -0,0 +1,92 @@
+
+# Parser Utilities
+
+This package provides a TypeScript CLI tool for recursively crawling a website, extracting structured metadata from each page, and saving the results in a predictable directory structure.
+
+## Features
+
+- **Recursive website parsing**: Visits all reachable pages up to a configurable depth.
+- **Structured output**: Saves each page's metadata as a JSON file.
+- **Configurable via environment variables or .env file**.
+
+---
+
+## Getting Started
+
+1. **Install dependencies:**
+	```bash
+	npm install
+	```
+2. **Type-check & compile**
+	```bash
+	npm run compile
+	```
+3. **Build the project:**
+	```bash
+	npm run build
+	```
+
+---
+
+## Usage
+
+### 1. Configure Environment Variables
+
+You can provide configuration in two ways:
+
+#### a) Using a `.env` file (recommended)
+
+Create a `.env` file in the `apps/parser` directory with the following content:
+
+```
+URL=https://example.com
+CHB_INDEX_ID=name_of_your_choice
+# DEPTH=2  # Optional, defaults to 2
+```
+
+#### b) Using command line variables
+
+```bash
+URL=https://example.com DEPTH=2 CHB_INDEX_ID=name_of_your_choice npm run parse
+```
+
+### 2. Run the Parser
+
+```bash
+npm run parse
+```
+
+---
+
+## Environment Variables
+
+- **`URL`** (required): The root page to start parsing from.
+- **`CHB_INDEX_ID`** (required): The base directory for storing parsed data. Output will be saved as `<CHB_INDEX_ID>/parsing/<sanitized(baseUrl)>/`.
+- **`DEPTH`** (optional, default: `2`): Maximum recursion depth for crawling links.
+
+**Note:** The parser will first look for these variables in the environment. If not found, it will load them from `.env` in the `apps/parser` directory.
+
+---
+
+## Output Structure
+
+Each visited page is saved as a JSON file:
+
+```
+<CHB_INDEX_ID>/parsing/<sanitized(baseUrl)>/<sanitized(path)>.json
+```
+
+- `<sanitized(baseUrl)>` and `<sanitized(path)>` are filesystem-safe versions of the URL components (illegal characters replaced with `-`).
+- This structure ensures output is predictable, easy to diff, and human-readable.
+
+---
+
+## Testing
+
+Run tests with:
+
+```bash
+npm run test
+```
+
+Tests will compile the project and then execute Jest to ensure the CLI behaves as expected.

apps/parser/jest.config.ts

@@ -0,0 +1,14 @@
+import type { Config } from "jest";
+
+const config: Config = {
+  rootDir: __dirname,
+  testRegex: "tests/.*\\.test\\.ts$",
+  transform: {
+    "^.+\\.ts$": ["ts-jest", { tsconfig: "tsconfig.json" }],
+  },
+  testEnvironment: "node",
+  clearMocks: true,
+  verbose: false,
+};
+
+export default config;

apps/parser/package.json

@@ -1,10 +1,28 @@
 {
   "name": "parser",
-  "version": "0.1.0",
+  "version": "1.0.0",
   "private": true,
-  "scripts": {},
+  "scripts": {
+    "clean": "shx rm -rf dist",
+    "compile": "tsc --project tsconfig.json",
+    "build": "npm run clean && tsc --project tsconfig.build.json",
+    "parse": "npm run build && node dist/parser.js",
+    "test": "npm run build && jest -i"
+  },
   "dependencies": {
-    "puppeteer": "^24.37.1"
+    "node-fetch": "^3.3.2",
+    "puppeteer": "^24.37.1",
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2",
+    "xml2js": "^0.6.2"
+  },
+  "devDependencies": {
+    "@types/jest": "^29.5.1",
+    "@types/node": "18.16.*",
+    "@types/xml2js": "^0.4.11",
+    "jest": "^29.5.0",
+    "shx": "^0.3.4",
+    "ts-jest": "^29.1.1",
+    "typescript": "5.1.6"
   }
 }
-

apps/parser/src/helpers/date-format.ts

@@ -0,0 +1,7 @@
+export function toIsoOrNull(value: string | null): string | null {
+  if (!value) {
+    return null;
+  }
+  const date = new Date(value);
+  return Number.isNaN(date.getTime()) ? null : date.toISOString();
+}

apps/parser/src/helpers/url-handling.ts

@@ -0,0 +1,84 @@
+import { SanitizeOptions } from "../modules/types";
+
+const ILLEGAL_RE = /[\/\?<>\\:\*\|"]/g;
+const CONTROL_RE = /[\x00-\x1f\x80-\x9f]/g;
+const RESERVED_RE = /^\.+$/;
+const WINDOWS_RESERVED_RE = /^(con|prn|aux|nul|com[0-9]|lpt[0-9])$/i;
+const WINDOWS_TRAILING_RE = /[\. ]+$/;
+const DEFAULT_REPLACEMENT = "-";
+
+export function sanitizeUrlAsFilename(
+  input: string,
+  options?: SanitizeOptions,
+): string {
+  if (!input) {
+    return DEFAULT_REPLACEMENT;
+  }
+  const replacement = validReplacementOrDefault(
+    options?.replacement ?? DEFAULT_REPLACEMENT,
+  );
+  let sanitized = input
+    .replace(ILLEGAL_RE, replacement)
+    .replace(CONTROL_RE, replacement)
+    .replace(RESERVED_RE, replacement)
+    .replace(WINDOWS_RESERVED_RE, replacement)
+    .replace(WINDOWS_TRAILING_RE, replacement)
+    .trim();
+  if (sanitized.length === 0) {
+    return replacement;
+  }
+  return sanitized.slice(0, 255);
+}
+
+function validReplacementOrDefault(candidate: string): string {
+  if (!candidate) {
+    return DEFAULT_REPLACEMENT;
+  }
+  if (
+    /[\/\?<>\\:\*\|"]/u.test(candidate) ||
+    /[\x00-\x1f\x80-\x9f]/u.test(candidate)
+  ) {
+    return DEFAULT_REPLACEMENT;
+  }
+  return candidate;
+}
+
+export const UrlWithoutAnchors = (rawUrl: string): string => {
+  try {
+    const parsed = new URL(rawUrl);
+    parsed.hash = "";
+    if (parsed.pathname.length > 1 && parsed.pathname.endsWith("/")) {
+      parsed.pathname = parsed.pathname.slice(0, -1);
+    }
+    const serialized = parsed.toString();
+    if (parsed.pathname === "/" && !parsed.search) {
+      return serialized.endsWith("/") ? serialized.slice(0, -1) : serialized;
+    }
+    return serialized;
+  } catch (error) {
+    console.warn(`Failed to parse URL: ${rawUrl}`, error);
+    return rawUrl;
+  }
+};
+
+export function deriveSubPath(
+  targetUrl: string,
+  baseUrl: string,
+  sanitizedBaseUrl: string,
+): string {
+  const base = new URL(baseUrl);
+  const target = new URL(targetUrl);
+  let relPath = target.pathname;
+  if (base.pathname !== "/" && relPath.startsWith(base.pathname)) {
+    relPath = relPath.slice(base.pathname.length);
+    if (!relPath.startsWith("/")) relPath = "/" + relPath;
+  }
+  if (
+    UrlWithoutAnchors(targetUrl) === sanitizedBaseUrl ||
+    relPath === "/" ||
+    relPath === ""
+  ) {
+    return "/";
+  }
+  return `${relPath}${target.search}${target.hash}` || "/";
+}

apps/parser/src/modules/config.ts

@@ -0,0 +1,50 @@
+import path from "node:path";
+import { EnvConfig } from "./types";
+import {
+  UrlWithoutAnchors,
+  sanitizeUrlAsFilename,
+} from "../helpers/url-handling";
+import * as dotenv from "dotenv";
+
+const DEFAULT_DEPTH = 2;
+
+export function resolveEnv(): EnvConfig {
+  let baseUrl = process.env.URL?.trim();
+  let depth = process.env.DEPTH?.trim();
+  let vectorIndexName = process.env.CHB_INDEX_ID?.trim();
+  if (!baseUrl || !depth || !vectorIndexName) {
+    const parserHome = path.resolve(__dirname, "../../");
+    dotenv.config({ path: path.join(parserHome, ".env") });
+    baseUrl = baseUrl || process.env.URL?.trim();
+    depth = depth || process.env.DEPTH?.trim();
+    vectorIndexName = vectorIndexName || process.env.CHB_INDEX_ID?.trim();
+  }
+  if (!baseUrl) {
+    throw new Error(
+      "Missing required URL. Set URL in environment or .env file.",
+    );
+  }
+  const sanitizedBaseUrl = UrlWithoutAnchors(baseUrl);
+  const parsedDepth = Number.parseInt(depth ?? `${DEFAULT_DEPTH}`, 10);
+  const maxDepth = Number.isNaN(parsedDepth)
+    ? DEFAULT_DEPTH
+    : Math.max(parsedDepth, 0);
+  const outputDirectory = generateOutputDirectoryPath(
+    vectorIndexName,
+    sanitizedBaseUrl,
+  );
+  return { baseUrl, sanitizedBaseUrl, outputDirectory, maxDepth };
+}
+
+function generateOutputDirectoryPath(
+  vectorIndexName: string | undefined,
+  sanitizedBaseUrl: string,
+): string {
+  const safeBaseSegment = sanitizeUrlAsFilename(sanitizedBaseUrl, {
+    replacement: "_",
+  });
+  if (!vectorIndexName) {
+    return `output/${safeBaseSegment}`;
+  }
+  return path.join(vectorIndexName, "parsing", safeBaseSegment);
+}