fix: reject LLM output that modifies links in defrag (#485)

fix: reject LLM output that modifies links in defrag passes

The sentence and terminology passes tell the LLM not to modify links,
but it did anyway (e.g. inventing /client/sdk/typescript). Added a
programmatic guard that compares link URLs before and after, rejecting
any output where links were added, removed, or changed.

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: kronosapiens

scripts/defrag-sentence.mjs

@@ -21,6 +21,7 @@ import {
     collectDocFiles,
     loadTextFile,
     callClaude,
+    checkLinksPreserved,
 } from "./lib/defrag-utils.mjs";
 
 const DRY_RUN = process.argv.includes("--dry-run");
@@ -107,6 +108,15 @@ async function main() {
                 continue;
             }
 
+            const linkCheck = checkLinksPreserved(original, normalized);
+            if (!linkCheck.ok) {
+                console.warn(`    REJECTED: LLM modified links in ${file.rel}`);
+                if (linkCheck.added.length) console.warn(`      Added: ${linkCheck.added.join(", ")}`);
+                if (linkCheck.removed.length) console.warn(`      Removed: ${linkCheck.removed.join(", ")}`);
+                filesErrored++;
+                continue;
+            }
+
             if (!DRY_RUN) {
                 writeFileSync(file.path, normalized, "utf-8");
             }

scripts/defrag-terminology.mjs

@@ -23,6 +23,7 @@ import {
     collectDocFiles,
     loadTextFile,
     callClaude,
+    checkLinksPreserved,
 } from "./lib/defrag-utils.mjs";
 import { join } from "path";
 
@@ -102,6 +103,15 @@ async function main() {
                 continue;
             }
 
+            const linkCheck = checkLinksPreserved(original, normalized);
+            if (!linkCheck.ok) {
+                console.warn(`    REJECTED: LLM modified links in ${file.rel}`);
+                if (linkCheck.added.length) console.warn(`      Added: ${linkCheck.added.join(", ")}`);
+                if (linkCheck.removed.length) console.warn(`      Removed: ${linkCheck.removed.join(", ")}`);
+                filesErrored++;
+                continue;
+            }
+
             if (!DRY_RUN) {
                 writeFileSync(file.path, normalized, "utf-8");
             }

scripts/lib/defrag-utils.mjs

@@ -51,6 +51,50 @@ export function getValidRoutes() {
     return files.map((f) => "/" + f.rel.replace(/\.(md|mdx)$/, "").replace(/\/index$/, ""));
 }
 
+/**
+ * Extract all markdown link URLs from content (ignoring code blocks).
+ */
+export function extractLinkUrls(content) {
+    const urls = [];
+    let inCodeBlock = false;
+    for (const line of content.split("\n")) {
+        if (line.trimStart().startsWith("```")) {
+            inCodeBlock = !inCodeBlock;
+            continue;
+        }
+        if (inCodeBlock) continue;
+        const linkRegex = /\[([^\]]*)\]\(([^)]+)\)/g;
+        let match;
+        while ((match = linkRegex.exec(line)) !== null) {
+            urls.push(match[2]);
+        }
+    }
+    return urls;
+}
+
+/**
+ * Check that the LLM output preserved all original links and added none.
+ * Returns { ok, added, removed } where added/removed are URL arrays.
+ */
+export function checkLinksPreserved(original, corrected) {
+    const originalUrls = extractLinkUrls(original);
+    const correctedUrls = extractLinkUrls(corrected);
+    const origCounts = new Map();
+    for (const u of originalUrls) origCounts.set(u, (origCounts.get(u) || 0) + 1);
+    const corrCounts = new Map();
+    for (const u of correctedUrls) corrCounts.set(u, (corrCounts.get(u) || 0) + 1);
+
+    const added = [];
+    const removed = [];
+    const allUrls = new Set([...origCounts.keys(), ...corrCounts.keys()]);
+    for (const u of allUrls) {
+        const diff = (corrCounts.get(u) || 0) - (origCounts.get(u) || 0);
+        if (diff > 0) for (let i = 0; i < diff; i++) added.push(u);
+        if (diff < 0) for (let i = 0; i < -diff; i++) removed.push(u);
+    }
+    return { ok: added.length === 0 && removed.length === 0, added, removed };
+}
+
 /**
  * Check if any single diff hunk exceeds the size limit.
  */