fix(parity): enforce fail-hard cross-engine parity and sync security docs

- tests/unit/parity.test.ts: Python engine invocation now throws on
  execution or parse failure instead of silently passing; shared
  assertCrossEngineParity() helper deduplicates both fixtures;
  REPODNA_ALLOW_PYTHON_PARITY_SKIP=1 remains a local-only opt-out.
- ci.yml: add dedicated fail-hard parity job (Node 22 + Python 3.12,
  pip install -e ., vitest parity suite); visualizer unit tests set the
  local skip env since that job has no Python runtime.
- README.md: limits table split into browser/private vs public durable
  columns; compression-ratio row now documents the actual
  quarantine-and-continue behavior; added cross-engine parity
  enforcement section.
- SECURITY.md + docs/threat-model.md: replace stale abort/413
  SUSPICIOUS_COMPRESSION_RATIO wording with the implemented quarantine
  semantics and public durable limits. No behavior changes.

Author: AnasBabari

.github/workflows/ci.yml

@@ -31,10 +31,34 @@ jobs:
       - run: npm ci
       - run: npm run lint
       - run: npm run typecheck
-      - run: npm run test:unit
+      # The dedicated `parity` job enforces cross-engine parity fail-hard with
+      # the Python engine installed; the visualizer job has no Python runtime.
+      - name: Unit tests (cross-engine parity enforced in dedicated parity job)
+        run: npm run test:unit
+        env:
+          REPODNA_ALLOW_PYTHON_PARITY_SKIP: '1'
       - run: npm run build
       - run: npm run build:vercel
 
+  parity:
+    name: Cross-engine parity (TS + Python)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0
+        with:
+          node-version: '22'
+          cache: npm
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: '3.12'
+          cache: pip
+      - run: npm ci
+      - run: python -m pip install -e .
+      # Fail-hard cross-engine contract: a missing Python engine, a crash, or
+      # any TypeScript/Python output drift fails CI. No skip env var is set.
+      - run: npx vitest run tests/unit/parity.test.ts
+
   browser-smoke:
     runs-on: ubuntu-latest
     needs: visualizer

README.md

@@ -94,20 +94,18 @@ Exports use a private seven-day Vercel Blob cache for public commit-addressed an
 
 ## Enforced Resource Limits & Quotas
 
-| Limit | Maximum | Action upon breach |
-|---|---|---|
-| **Public Rate Limit** | 5 analyses / 10 min per IP | Returns `429 RATE_LIMITED` |
-| **Authenticated Rate Limit** | 20 analyses / 10 min per user | Returns `429 RATE_LIMITED` |
-| **Total archive entries** | 20,000 entries | Returns `413 TOO_MANY_ARCHIVE_ENTRIES` (header bomb defense) |
-| **Candidate repository files** | 10,000 files | Returns `413 TOO_MANY_FILES` |
-| **Individual file size** | 1 MB (1,000,000 bytes) | Skipped with diagnostic (`exceeds_file_size_limit`) |
-| **Declared compression ratio** | 200:1 (entries > 256 KB) | Returns `413 SUSPICIOUS_COMPRESSION_RATIO` |
-| **Private/browser compressed archive** | 25 MB (26,214,400 bytes) | Returns `413 ARCHIVE_TOO_LARGE` |
-| **Private/browser extracted content** | 100 MB (104,857,600 bytes) | Returns `413 EXTRACTED_TOO_LARGE` (ZIP bomb protection) |
-| **Public durable compressed archive** | 128 MB before Git tree fallback | Uses bounded archive or Git tree acquisition |
-| **Public durable extracted content** | 192 MB | Produces an honest partial inventory when bounded limits are reached |
-| **Interactive graph** | 8,000 nodes / 12,000 edges in the v2 artifact; 240 rendered nodes / edges in the live canvas | Preserves full inventory and reports compaction diagnostics |
-| **GitHub fetch timeout** | 20 seconds private/browser; 60 seconds public durable | Returns a controlled upstream timeout/failure |
+| Limit | Browser / private analysis | Public durable analysis | Action upon breach |
+|---|---|---|---|
+| **Public rate limit** | 5 analyses / 10 min per IP | Same | Returns `429 RATE_LIMITED` |
+| **Authenticated rate limit** | 20 analyses / 10 min per user | Same | Returns `429 RATE_LIMITED` |
+| **Candidate repository files** | 10,000 files | 20,000 files | Browser/private returns `413 TOO_MANY_FILES`; public durable produces an honest partial inventory (`max_files_limit`) |
+| **Total archive entries** | 20,000 entries | 100,000 entries | Returns `413 TOO_MANY_ARCHIVE_ENTRIES` (header bomb defense) |
+| **Individual file size** | 1 MB (1,000,000 bytes) | Same | Skipped with diagnostic (`exceeds_file_size_limit`) |
+| **Declared compression ratio** | 200:1 (entries emitting > 256 KB decompressed) | Same | Entry is **quarantined, not fatal**: its stream is terminated and buffers discarded, it is skipped with a `suspicious_compression_ratio` diagnostic, and analysis of the remaining safe files continues. Every emitted byte—including quarantined entries—still counts toward the cumulative extracted-bytes cap. |
+| **Compressed archive download** | 25 MB (26,214,400 bytes) | 128 MB before Git-tree fallback | Browser/private returns `413 ARCHIVE_TOO_LARGE`; public durable uses bounded archive or Git-tree acquisition |
+| **Extracted content** | 100 MB (104,857,600 bytes) | 192 MB | Browser/private returns `413 EXTRACTED_TOO_LARGE` (ZIP bomb protection); public durable produces an honest partial inventory when bounded limits are reached |
+| **Interactive graph** | 8,000 nodes / 12,000 edges in the v2 artifact; 240 rendered nodes / edges in the live canvas | Same | Preserves full inventory and reports compaction diagnostics |
+| **GitHub fetch timeout** | 20 seconds | 60 seconds | Returns a controlled upstream timeout/failure |
 
 ---
 
@@ -158,6 +156,21 @@ npm test
 npm run build:vercel
 ```
 
+### Cross-Engine Parity Enforcement
+
+The `fastapi-basic` and `express-basic` parity cases run both the TypeScript
+engine and the Python engine (`core/repodna`) over the same fixtures and
+**fail hard** when the Python engine is unavailable, crashes, emits
+unparseable output, or diverges from the TypeScript engine. A missing Python
+implementation can never silently pass.
+
+- Locally, contributors without a Python environment can set
+  `REPODNA_ALLOW_PYTHON_PARITY_SKIP=1` to skip the Python half of the parity
+  cases instead of failing.
+- CI never sets that variable: a dedicated `parity` job installs the Python
+  engine (`python -m pip install -e .`) and enforces the contract fail-hard
+  on every push and pull request.
+
 ---
 
 ## Canonical Contract & Security

SECURITY.md

@@ -29,7 +29,7 @@ RepoDNA is architected around **Zero Code Execution**:
 
 4. **Resource Bounds & Streaming Ingestion Defenses**:
    - Bounded download sizes with streaming network byte caps (25 MB default archive limit) that cancel downstream connections immediately upon breach.
-   - Streaming Decompression Defense via `fflate`: Archive buffers are fed in bounded 64 KiB chunks with a hard all-entry cap (20,000 max entries) against header bombs, candidate file limits (10,000 max), path traversal rejection, null-byte path validation, path depth limits (32 max), per-entry uncompressed byte caps (1 MB max, skipping oversized entries early), declared compression-ratio heuristic guards (aborting on >200:1 ratio past 256 KB floor), and cumulative extracted content limits (100 MB max, counting all emitted bytes).
+   - Streaming Decompression Defense via `fflate`: Archive buffers are fed in bounded 64 KiB chunks with a hard all-entry cap (20,000 max entries) against header bombs, candidate file limits (10,000 max), path traversal rejection, null-byte path validation, path depth limits (32 max), per-entry uncompressed byte caps (1 MB max, skipping oversized entries early), declared compression-ratio quarantine (>200:1 ratio past a 256 KB floor terminates and discards that entry, skipped with a `suspicious_compression_ratio` diagnostic, while analysis of the remaining safe files continues; every emitted byte—including quarantined entries—still counts toward the cumulative caps), and cumulative extracted content limits (100 MB max, counting all emitted bytes). Public durable analyses raise these bounds (20,000 files / 100,000 entries / 128 MB archive / 192 MB extracted) with honest partial inventories instead of hard failures.
    - Syntax parsers operate under AST depth caps (`MAX_AST_DEPTH = 128`), node budgets (`MAX_AST_NODES = 25000`), item collection limits, and automatic Tree-sitter WASM cleanup (`tree.delete()`).
 
 ---

docs/threat-model.md

@@ -8,7 +8,7 @@ This document outlines the threat vectors considered in the design of RepoDNA an
 
 | Threat Vector | Description | Severity | Mitigation in RepoDNA |
 |---|---|---|---|
-| **Malicious Archive / ZIP Bomb** | Attacker crafts a ZIP with high compression ratio (e.g. 5 MB expanding to 10 GB), millions of files, or duplicate paths. | **Critical** | Streaming decompression via `fflate` with bounded ingestion: 64 KiB chunk-fed `Unzip` (`UnzipInflate`/`UnzipPassThrough`), EOCD structural validation, `maxArchiveEntries = 20,000` header-bomb cap (`TOO_MANY_ARCHIVE_ENTRIES`), `maxFiles = 10,000` candidate cap (`TOO_MANY_FILES`), duplicate normalized-path rejection, per-entry live cap `maxFileBytes = 1 MB` with early `terminate()` and discarded buffers, cumulative `maxTotalExtractedBytes = 100 MB` counting every emitted byte including skipped oversized files (`EXTRACTED_TOO_LARGE`), declared ratio heuristic >200:1 after 256 KB floor (`SUSPICIOUS_COMPRESSION_RATIO`), and `maxArchiveBytes = 25 MB` network cap with streaming `reader.cancel()`. Local folder byte limits use `File.size` before decoding so UTF-16 string length cannot under-count input. |
+| **Malicious Archive / ZIP Bomb** | Attacker crafts a ZIP with high compression ratio (e.g. 5 MB expanding to 10 GB), millions of files, or duplicate paths. | **Critical** | Streaming decompression via `fflate` with bounded ingestion: 64 KiB chunk-fed `Unzip` (`UnzipInflate`/`UnzipPassThrough`), EOCD structural validation, `maxArchiveEntries = 20,000` header-bomb cap (`TOO_MANY_ARCHIVE_ENTRIES`), `maxFiles = 10,000` candidate cap (`TOO_MANY_FILES`), duplicate normalized-path rejection, per-entry live cap `maxFileBytes = 1 MB` with early `terminate()` and discarded buffers, cumulative `maxTotalExtractedBytes = 100 MB` counting every emitted byte including skipped oversized files (`EXTRACTED_TOO_LARGE`), declared ratio quarantine >200:1 after 256 KB floor (entry stream `terminate()`d and buffers discarded, skipped with a `suspicious_compression_ratio` diagnostic while analysis of safe files continues—no `413` is raised for a quarantined entry), and `maxArchiveBytes = 25 MB` network cap with streaming `reader.cancel()`. These are the browser/private defaults; public durable analyses raise them to 20,000 files / 100,000 entries / 128 MB archive / 192 MB extracted with honest partial inventories (`allowPartialOnFileLimit`). Local folder byte limits use `File.size` before decoding so UTF-16 string length cannot under-count input. |
 | **Path Traversal / Arbitrary File Overwrite** | Archive entries contain `../../etc/passwd`, absolute paths (`/root`), or Windows drive letters (`C:\`). | **Critical** | `validatePath`/`normalizeArchivePath` reject absolute paths, drive letters, `..` segments and null bytes before extraction; path depth is capped at 32 segments and paths longer than 4,096 characters are skipped. `PATH_TRAVERSAL` fails closed at the incremental `onfile` boundary, not post-extraction. |
 | **Malformed or hostile analysis artifact** | A user imports a huge or schema-invalid RepoDNA JSON file, or a cached Blob is corrupted. | **High** | Imported artifacts are capped at 128 MB before reading and validated through the version-aware schema loader. The CSP-safe browser validator checks every graph node and edge with a bounded error list. Cached public artifacts are size checked and schema validated before use; malformed entries are evicted. |
 | **Malicious Syntax / Parser Bomb** | Deeply nested ASTs, 1 MB single-line JS strings, or recursive macros designed to crash memory/CPU. | **High** | Tree-sitter budgets `MAX_AST_DEPTH = 128`, `MAX_AST_NODES = 25,000` with item collection limits and guaranteed `tree.delete()` cleanup. Syntax parsing runs in try-catch with graceful degradation (`parseSuccessRate` reduction and diagnostics) without failing the whole repo. Future: wall-clock analysis budget / Worker isolation. |

tests/unit/parity.test.ts

@@ -5,6 +5,7 @@ import { describe, expect, it } from 'vitest';
 import { analyzeRepositoryFiles } from '../../app/lib/analyzer';
 import { validateRepoDNAProject } from '../../app/lib/schema/validator';
 import type { DiscoveredFile } from '../../app/lib/analyzer/types';
+import type { RepoDNAProject } from '../../app/lib/types';
 
 function readFixtureFiles(fixtureDir: string): DiscoveredFile[] {
   const baseDir = path.resolve(process.cwd(), fixtureDir);
@@ -33,99 +34,102 @@ function readFixtureFiles(fixtureDir: string): DiscoveredFile[] {
   return results;
 }
 
-describe('Cross-Engine Conformance & Schema Parity Suite', () => {
-  it('verifies schema validity and parity on fastapi-basic fixture', async () => {
-    const fixturePath = 'tests/fixtures/fastapi-basic';
-    const files = readFixtureFiles(fixturePath);
-
-    // 1. TypeScript Engine Analysis
-    const tsResult = await analyzeRepositoryFiles(
-      { name: 'fastapi-basic', source: `file://${fixturePath}`, files, skipped: [] },
-      {}
-    );
-
-    // Validate TS schema conformance
-    const tsValidation = validateRepoDNAProject(tsResult);
-    expect(tsValidation.valid).toBe(true);
-    expect(tsValidation.errors).toEqual([]);
-
-    // 2. Python Engine Analysis via CLI
-    let pyResult: RepoDNAProject | null = null;
-    try {
-      const pyJson = execSync(
-        `python -c "import sys, json; sys.path.insert(0, 'core'); from repodna.engine import analyze_repository; res = analyze_repository('${fixturePath}'); print(json.dumps(res.to_dict()))"`,
-        { encoding: 'utf-8' }
-      );
-      pyResult = JSON.parse(pyJson);
-    } catch (e) {
-      console.warn('Python engine parity check skipped or failed:', e);
-    }
-
-    if (pyResult) {
-      // Validate Python schema conformance
-      const pyValidation = validateRepoDNAProject(pyResult);
-      expect(pyValidation.valid).toBe(true);
-      expect(pyValidation.errors).toEqual([]);
-
-      // 3. Structural Parity Assertions
-      expect(tsResult.schemaVersion).toBe(pyResult.schemaVersion);
-      expect(tsResult.repository.sourceFileCount).toBe(pyResult.repository.sourceFileCount);
-
-      // Verify routes parity
-      const tsRoutes = tsResult.routes.map((r) => `${r.method} ${r.path}`).sort();
-      const pyRoutes = pyResult.routes.map((r) => `${r.method} ${r.path}`).sort();
-      expect(tsRoutes).toEqual(pyRoutes);
+/**
+ * Cross-engine parity is a fail-hard architectural invariant: if the Python
+ * engine cannot be executed, crashes, emits unparseable output, or diverges
+ * from the TypeScript engine, the parity case FAILS.
+ *
+ * Local development escape hatch only: contributors without a Python
+ * environment may set REPODNA_ALLOW_PYTHON_PARITY_SKIP=1 to skip the Python
+ * half of a parity case instead of failing. CI never sets this variable —
+ * the dedicated `parity` workflow job enforces the contract fail-hard.
+ */
+const ALLOW_PYTHON_PARITY_SKIP = process.env.REPODNA_ALLOW_PYTHON_PARITY_SKIP === '1';
+
+function runPythonEngineAnalysis(fixturePath: string): RepoDNAProject {
+  const pyJson = execSync(
+    `python -c "import sys, json; sys.path.insert(0, 'core'); from repodna.engine import analyze_repository; res = analyze_repository('${fixturePath}'); print(json.dumps(res.to_dict()))"`,
+    { encoding: 'utf-8' }
+  );
+  return JSON.parse(pyJson) as RepoDNAProject;
+}
 
-      // Verify framework detection parity
-      expect(tsResult.repository.fingerprint.frameworks).toContain('FastAPI');
-      expect(pyResult.repository.fingerprint.frameworks).toContain('FastAPI');
+interface ParityCase {
+  fixtureName: string;
+  expectedFramework: string;
+  expectArchitectureComponents: boolean;
+}
 
-      // Verify architecture component count
-      expect(tsResult.architecture.components.length).toBeGreaterThan(0);
-      expect(pyResult.architecture.components.length).toBeGreaterThan(0);
+async function assertCrossEngineParity(parityCase: ParityCase, ctx: { skip: () => void }): Promise<void> {
+  const fixturePath = `tests/fixtures/${parityCase.fixtureName}`;
+  const files = readFixtureFiles(fixturePath);
+
+  // 1. TypeScript Engine Analysis
+  const tsResult = await analyzeRepositoryFiles(
+    { name: parityCase.fixtureName, source: `file://${fixturePath}`, files, skipped: [] },
+    {}
+  );
+
+  // Validate TS schema conformance
+  const tsValidation = validateRepoDNAProject(tsResult);
+  expect(tsValidation.valid).toBe(true);
+  expect(tsValidation.errors).toEqual([]);
+
+  // 2. Python Engine Analysis via CLI — fail-hard on any invocation or parse
+  // failure so a missing Python engine can never silently pass parity.
+  let pyResult: RepoDNAProject;
+  try {
+    pyResult = runPythonEngineAnalysis(fixturePath);
+  } catch (e) {
+    if (ALLOW_PYTHON_PARITY_SKIP) {
+      console.warn(`REPODNA_ALLOW_PYTHON_PARITY_SKIP=1: skipping Python engine parity for '${parityCase.fixtureName}':`, e);
+      ctx.skip();
     }
-  });
+    throw new Error(
+      `Python engine invocation failed for fixture '${parityCase.fixtureName}'. ` +
+      'Cross-engine parity is enforced fail-hard; install the Python engine (python -m pip install -e .) ' +
+      'or set REPODNA_ALLOW_PYTHON_PARITY_SKIP=1 to skip locally.',
+      { cause: e }
+    );
+  }
 
-  it('verifies schema validity and parity on express-basic fixture', async () => {
-    const fixturePath = 'tests/fixtures/express-basic';
-    const files = readFixtureFiles(fixturePath);
+  // Validate Python schema conformance
+  const pyValidation = validateRepoDNAProject(pyResult);
+  expect(pyValidation.valid).toBe(true);
+  expect(pyValidation.errors).toEqual([]);
 
-    // 1. TypeScript Engine Analysis
-    const tsResult = await analyzeRepositoryFiles(
-      { name: 'express-basic', source: `file://${fixturePath}`, files, skipped: [] },
-      {}
-    );
+  // 3. Structural Parity Assertions
+  expect(tsResult.schemaVersion).toBe(pyResult.schemaVersion);
+  expect(tsResult.repository.sourceFileCount).toBe(pyResult.repository.sourceFileCount);
 
-    const tsValidation = validateRepoDNAProject(tsResult);
-    expect(tsValidation.valid).toBe(true);
-    expect(tsValidation.errors).toEqual([]);
-
-    // 2. Python Engine Analysis
-    let pyResult: RepoDNAProject | null = null;
-    try {
-      const pyJson = execSync(
-        `python -c "import sys, json; sys.path.insert(0, 'core'); from repodna.engine import analyze_repository; res = analyze_repository('${fixturePath}'); print(json.dumps(res.to_dict()))"`,
-        { encoding: 'utf-8' }
-      );
-      pyResult = JSON.parse(pyJson);
-    } catch (e) {
-      console.warn('Python engine parity check skipped or failed:', e);
-    }
+  // Verify routes parity
+  const tsRoutes = tsResult.routes.map((r) => `${r.method} ${r.path}`).sort();
+  const pyRoutes = pyResult.routes.map((r) => `${r.method} ${r.path}`).sort();
+  expect(tsRoutes).toEqual(pyRoutes);
 
-    if (pyResult) {
-      const pyValidation = validateRepoDNAProject(pyResult);
-      expect(pyValidation.valid).toBe(true);
-      expect(pyValidation.errors).toEqual([]);
+  // Verify framework detection parity
+  expect(tsResult.repository.fingerprint.frameworks).toContain(parityCase.expectedFramework);
+  expect(pyResult.repository.fingerprint.frameworks).toContain(parityCase.expectedFramework);
 
-      expect(tsResult.schemaVersion).toBe(pyResult.schemaVersion);
-      expect(tsResult.repository.sourceFileCount).toBe(pyResult.repository.sourceFileCount);
+  // Verify architecture component count
+  if (parityCase.expectArchitectureComponents) {
+    expect(tsResult.architecture.components.length).toBeGreaterThan(0);
+    expect(pyResult.architecture.components.length).toBeGreaterThan(0);
+  }
+}
 
-      const tsRoutes = tsResult.routes.map((r) => `${r.method} ${r.path}`).sort();
-      const pyRoutes = pyResult.routes.map((r) => `${r.method} ${r.path}`).sort();
-      expect(tsRoutes).toEqual(pyRoutes);
+describe('Cross-Engine Conformance & Schema Parity Suite', () => {
+  it('verifies schema validity and parity on fastapi-basic fixture', async (ctx) => {
+    await assertCrossEngineParity(
+      { fixtureName: 'fastapi-basic', expectedFramework: 'FastAPI', expectArchitectureComponents: true },
+      ctx
+    );
+  });
 
-      expect(tsResult.repository.fingerprint.frameworks).toContain('Express');
-      expect(pyResult.repository.fingerprint.frameworks).toContain('Express');
-    }
+  it('verifies schema validity and parity on express-basic fixture', async (ctx) => {
+    await assertCrossEngineParity(
+      { fixtureName: 'express-basic', expectedFramework: 'Express', expectArchitectureComponents: false },
+      ctx
+    );
   });
 });