[CAI-749] Local parser

.changeset/wide-hairs-fail.md

@@ -0,0 +1,5 @@
+---
+"parser": major
+---
+
+Add script to perform parsing to parser app. Store parsed information locally. Sanitize urls in filesystem compatible format to use as file names.

apps/parser/.env.default

@@ -0,0 +1,13 @@
+# Default environment variables for the parser CLI
+# Root URL to start parsing from
+URL="https://example.com"
+
+# Maximum recursion depth (integer)
+DEPTH=2
+
+# Name of the vector index bucket/folder where parsed artifacts are stored
+VECTOR_INDEX_NAME="vector-index-name"
+
+# Optional absolute/relative directory override. Leave empty to use
+# <VECTOR_INDEX_NAME>/parsing/<sanitized(baseUrl)> automatically.
+OUTDIR=""

apps/parser/README.md

@@ -0,0 +1,39 @@
+## Parser utilities
+
+This package contains the following TypeScript CLI utility:
+
+- `parser` — recursively visits a website, extracts structured page metadata, and saves each page under `<VECTOR_INDEX_NAME>/parsing/<sanitized(baseUrl)>/`.
+
+
+### Getting started
+
+```bash
+npm install
+npm run build
+```
+
+### Parse a website
+
+```bash
+URL=https://example.com DEPTH=2 npm run parse
+```
+
+Environment variables:
+
+- `URL` (required): root page for the parse.
+- `DEPTH` (optional, default `2`): max depth for recursion.
+- `VECTOR_INDEX_NAME` (required unless `OUTDIR` is provided): base directory name where parsed data will be stored as `<VECTOR_INDEX_NAME>/parsing/<sanitized(baseUrl)>/`.
+- `OUTDIR` (optional): fully override the destination directory.
+
+`<sanitized(baseUrl)>` and `<sanitized(path)>` refer to filesystem-safe versions of the URL components (illegal characters replaced with `_`), ensuring predictable, human-readable filenames.
+
+Each visited page is stored as `<VECTOR_INDEX_NAME>/parsing/<sanitized(baseUrl)>/<sanitized(path)>.json` (or under `OUTDIR` if specified) with normalized metadata, making it easy to diff between runs.
+
+
+### Tests
+
+```bash
+npm test
+```
+
+Tests compile the project before executing Jest to ensure the CLI behaves exactly like the production build.

apps/parser/jest.config.ts

@@ -0,0 +1,14 @@
+import type { Config } from 'jest';
+
+const config: Config = {
+  rootDir: __dirname,
+  testRegex: 'tests/.*\\.test\\.ts$',
+  transform: {
+    '^.+\\.ts$': ['ts-jest', { tsconfig: 'tsconfig.json' }],
+  },
+  testEnvironment: 'node',
+  clearMocks: true,
+  verbose: false,
+};
+
+export default config;

apps/parser/package.json

@@ -1,10 +1,28 @@
 {
   "name": "parser",
-  "version": "0.1.0",
+  "version": "1.0.0",
   "private": true,
-  "scripts": {},
+  "scripts": {
+    "clean": "shx rm -rf dist",
+    "compile": "tsc --project tsconfig.json",
+    "build": "npm run clean && tsc --project tsconfig.build.json",
+    "parse": "npm run build && node dist/parser.js",
+    "test": "npm run build && jest -i"
+  },
   "dependencies": {
-    "puppeteer": "^24.37.1"
+    "node-fetch": "^3.3.2",
+    "puppeteer": "^24.37.1",
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2",
+    "xml2js": "^0.6.2"
+  },
+  "devDependencies": {
+    "@types/jest": "^29.5.1",
+    "@types/node": "18.16.*",
+    "@types/xml2js": "^0.4.11",
+    "jest": "^29.5.0",
+    "shx": "^0.3.4",
+    "ts-jest": "^29.1.1",
+    "typescript": "5.1.6"
   }
 }
-

apps/parser/src/modules/config.ts

@@ -0,0 +1,37 @@
+import path from 'node:path';
+import { stripUrlDecorations } from '../utils/url';
+import { sanitizeFilename } from '../utils/sanitizeFilename';
+
+export type EnvConfig = {
+  readonly baseUrl: string;
+  readonly sanitizedBaseUrl: string;
+  readonly outputDirectory: string;
+  readonly maxDepth: number;
+};
+
+const DEFAULT_BASE_URL = 'https://news.polymer-project.org/';
+const DEFAULT_DEPTH = 2;
+
+export function resolveEnv(): EnvConfig {
+  const baseUrl = process.env.URL?.trim().length ? process.env.URL : DEFAULT_BASE_URL;
+  const sanitizedBaseUrl = stripUrlDecorations(baseUrl);
+  const parsedDepth = Number.parseInt(process.env.DEPTH ?? `${DEFAULT_DEPTH}`, 10);
+  const maxDepth = Number.isNaN(parsedDepth) ? DEFAULT_DEPTH : Math.max(parsedDepth, 0);
+  const vectorIndexName = process.env.VECTOR_INDEX_NAME?.trim();
+  const derivedOutput = buildDefaultOutputDirectory(vectorIndexName, sanitizedBaseUrl);
+  const outputDirectory = process.env.OUTDIR?.trim().length
+    ? process.env.OUTDIR
+    : derivedOutput;
+  return { baseUrl, sanitizedBaseUrl, outputDirectory, maxDepth };
+}
+
+function buildDefaultOutputDirectory(
+  vectorIndexName: string | undefined,
+  sanitizedBaseUrl: string
+): string {
+  const safeBaseSegment = sanitizeFilename(sanitizedBaseUrl, { replacement: '_' });
+  if (!vectorIndexName) {
+    return `output/${safeBaseSegment}`;
+  }
+  return path.join(vectorIndexName, 'parsing', safeBaseSegment);
+}

apps/parser/src/modules/crawler.ts

@@ -0,0 +1,142 @@
+import { Browser } from 'puppeteer';
+import { ParseNode, ParseMetadata } from './types';
+import { normalizeUrl } from '../utils/url';
+import { expandInteractiveSections } from './domActions';
+
+export async function parsePages(
+  browser: Browser,
+  node: ParseNode,
+  depth: number,
+  maxDepth: number,
+  parsedPages: Map<string, ParseMetadata>,
+  parsePageFn: (browser: Browser, url: string) => Promise<ParseMetadata | null>,
+  baseOrigin: string,
+  baseScope: string,
+  baseHostToken: string
+): Promise<void> {
+  const visitKey = buildVisitKey(node.url);
+  if (parsedPages.has(visitKey) || depth > maxDepth) {
+    return;
+  }
+
+  const normalizedUrl = normalizeUrl(node.url);
+  if (!isWithinScope(normalizedUrl, baseScope, baseHostToken)) {
+    return;
+  }
+
+  const metadata = await parsePageFn(browser, node.url);
+  if (!metadata) return;
+
+  parsedPages.set(visitKey, metadata);
+  node.title = metadata.title;
+  node.bodyText = metadata.bodyText;
+  node.lang = metadata.lang;
+  node.keywords = metadata.keywords;
+  node.datePublished = metadata.datePublished;
+  node.lastModified = metadata.lastModified;
+
+  let page;
+  let anchors: string[] = [];
+  try {
+    page = await browser.newPage();
+    await page.goto(node.url, { waitUntil: 'networkidle2', timeout: 45000 });
+    await expandInteractiveSections(page);
+    anchors = await page.evaluate((allowedToken: string) => {
+      const anchors = Array.from(document.querySelectorAll('a[href]'));
+      const iframeSources = Array.from(document.querySelectorAll('iframe[src]'));
+      const unique = new Set<string>();
+      for (const anchor of anchors) {
+        const href = (anchor as HTMLAnchorElement).href;
+        if (!href || !href.startsWith('http')) continue;
+        try {
+          const target = new URL(href, window.location.href);
+          const normalizedHref = target.href.toLowerCase();
+          if (allowedToken && !normalizedHref.includes(allowedToken)) continue;
+          if (target.href === window.location.href) continue;
+          unique.add(target.href);
+        } catch (_) {}
+      }
+
+      for (const frame of iframeSources) {
+        const src = (frame as HTMLIFrameElement).src;
+        if (!src || !src.startsWith('http')) {
+          continue;
+        }
+        try {
+          const target = new URL(src, window.location.href);
+          const normalizedSrc = target.href.toLowerCase();
+          if (allowedToken && !normalizedSrc.includes(allowedToken)) continue;
+          unique.add(target.href);
+        } catch (_) {}
+      }
+      return Array.from(unique);
+    }, baseHostToken) as string[];
+  } catch (error) {
+    // Ignore anchor extraction errors
+  } finally {
+    if (page) await page.close();
+  }
+
+
+  const scheduled = new Set<string>();
+  const nextChildren: ParseNode[] = [];
+  for (const href of anchors) {
+    const normalized = normalizeUrl(href);
+    const visitCandidate = buildVisitKey(href);
+    if (parsedPages.has(visitCandidate) || scheduled.has(visitCandidate)) continue;
+    const lowerNormalized = normalized.toLowerCase();
+    if (baseHostToken && !lowerNormalized.includes(baseHostToken)) {
+      continue;
+    }
+    if (!isWithinScope(normalized, baseScope, baseHostToken)) {
+      continue;
+    }
+    scheduled.add(visitCandidate);
+    nextChildren.push({ url: href });
+  }
+  node.children = nextChildren;
+
+  if (!node.children || depth >= maxDepth) return;
+  for (const child of node.children) {
+    await parsePages(
+      browser,
+      child,
+      depth + 1,
+      maxDepth,
+      parsedPages,
+      parsePageFn,
+      baseOrigin,
+      baseScope,
+      baseHostToken
+    );
+  }
+}
+
+function isWithinScope(url: string, scope: string, hostToken: string): boolean {
+  if (hostToken && url.toLowerCase().includes(hostToken)) {
+    return true;
+  }
+  if (!scope) {
+    return true;
+  }
+  const lowerUrl = url.toLowerCase();
+  const lowerScope = scope.toLowerCase();
+  if (lowerUrl === lowerScope) {
+    return true;
+  }
+  if (!lowerUrl.startsWith(lowerScope)) {
+    return false;
+  }
+  const nextChar = lowerUrl.charAt(lowerScope.length);
+  return nextChar === '/' || nextChar === '?' || nextChar === '#';
+}
+
+export function buildVisitKey(rawUrl: string): string {
+  try {
+    const url = new URL(rawUrl);
+    const normalizedBase = normalizeUrl(url.toString());
+    return url.hash ? `${normalizedBase}${url.hash}` : normalizedBase;
+  } catch (_error) {
+    return rawUrl;
+  }
+}

apps/parser/src/modules/domActions.ts

@@ -0,0 +1,43 @@
+import type { Page } from 'puppeteer';
+
+const TOGGLE_SELECTORS = [
+  '[data-toggle]'
+  ,'[data-testid="accordion-toggle"]'
+  ,'[aria-expanded]'
+  ,'.accordion button'
+  ,'.accordion-toggle'
+  ,'.accordion-trigger'
+  ,'.faq-item button'
+  ,'.collapse-toggle'
+  ,'.MuiButtonBase-root[aria-expanded]'
+];
+
+export async function expandInteractiveSections(page: Page): Promise<void> {
+  await page.evaluate((selectors) => {
+    document.querySelectorAll('details').forEach((element) => {
+      (element as HTMLDetailsElement).open = true;
+    });
+
+    selectors.forEach((selector) => {
+      document.querySelectorAll(selector).forEach((node) => {
+        const target = node as HTMLElement;
+        if (!target || target.getAttribute('data-expanded') === 'true') {
+          return;
+        }
+
+        const ariaExpanded = target.getAttribute('aria-expanded');
+        const isToggleButton =
+          target.tagName === 'BUTTON' || target.getAttribute('role') === 'button';
+        const isCollapsed =
+          ariaExpanded === 'false' || target.classList.contains('collapsed');
+
+        if (isToggleButton || isCollapsed || selector === '[data-toggle]') {
+          target.click();
+          target.setAttribute('data-expanded', 'true');
+        }
+      });
+    });
+  }, TOGGLE_SELECTORS);
+
+  await new Promise((resolve) => setTimeout(resolve, 250));
+}

apps/parser/src/modules/errors.ts

@@ -0,0 +1,4 @@
+export function handleError(error: unknown) {
+  console.error('Parser terminated with an error:', error);
+  process.exitCode = 1;
+}

apps/parser/src/modules/output.ts

@@ -0,0 +1,10 @@
+import { mkdirSync } from 'node:fs';
+import { writeFile } from 'node:fs/promises';
+
+export function ensureDirectory(dir: string) {
+  mkdirSync(dir, { recursive: true });
+}
+
+export async function saveMetadata(dir: string, filename: string, metadata: object) {
+  await writeFile(`${dir}/${filename}`, JSON.stringify(metadata, null, 2));
+}

apps/parser/src/modules/types.ts

@@ -0,0 +1,20 @@
+export type ParseMetadata = {
+  readonly url: string;
+  readonly title: string;
+  readonly bodyText: string;
+  readonly lang: string | null;
+  readonly keywords: string | null;
+  readonly datePublished: string | null;
+  readonly lastModified: string | null;
+};
+
+export type ParseNode = {
+  readonly url: string;
+  title?: string;
+  bodyText?: string;
+  lang?: string | null;
+  keywords?: string | null;
+  datePublished?: string | null;
+  lastModified?: string | null;
+  children?: ParseNode[];
+};