feat(docs): enhance markdown generation with HTML parity (#1388)

Author: fassko

package-lock.json

@@ -7,7 +7,7 @@
     "start": "docusaurus start",
     "automations": "chmod +x run-automations.sh && ./run-automations.sh",
     "update-deps": "chmod +x update-deps.sh && ./update-deps.sh",
-    "build": "docusaurus build && node scripts/generate-md-routes.mjs && node scripts/fix-llms-urls.mjs && node scripts/inject-html-llms-tags.mjs",
+    "build": "docusaurus build && node scripts/mark-html-parity-exclusions.mjs && node scripts/generate-md-routes.mjs && node scripts/fix-llms-urls.mjs && node scripts/inject-html-llms-tags.mjs",
     "swizzle": "docusaurus swizzle",
     "deploy": "docusaurus deploy",
     "deploy:mcp": "npx wrangler deploy --config cloudflare-mcp/wrangler.toml",
@@ -59,6 +59,7 @@
     "textlint": "^15.5.2",
     "textlint-plugin-mdx": "^1.0.2",
     "textlint-rule-one-sentence-per-line": "^2.0.0",
+    "turndown": "^7.2.4",
     "typescript": "^5.9.3"
   },
   "browserslist": {

package.json

@@ -17,6 +17,8 @@
 import fs from "fs";
 import path from "path";
 import { fileURLToPath } from "url";
+import { expandMdxBody } from "./mdx-markdown-expanders.mjs";
+import { markdownFromBuiltHtml } from "./html-to-markdown.mjs";
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const rootDir = path.resolve(__dirname, "..");
@@ -118,36 +120,15 @@ function indexSources() {
 }
 
 /**
- * Strip MDX import/export statements and a small set of common JSX-only blocks
- * (e.g. <Tabs>...<TabItem>) so the body reads as plain markdown for LLMs.
- * We deliberately preserve fenced code blocks, tables, lists, and prose.
+ * Turn MDX source into agent-facing markdown: expand data-driven React
+ * components, inline partials, and strip remaining JSX.
  */
-function cleanMdxBody(body) {
-  // Drop everything that looks like an ESM import/export at the top level.
-  let cleaned = body
-    .split(/\r?\n/)
-    .filter((line) => {
-      const trimmed = line.trim();
-      if (trimmed.startsWith("import ") && /from\s+['"]/.test(trimmed))
-        return false;
-      if (
-        trimmed.startsWith("export ") &&
-        !trimmed.startsWith("export const meta")
-      ) {
-        // strip simple `export const Foo = ...;` lines used by MDX
-        return false;
-      }
-      return true;
-    })
-    .join("\n");
-
-  // Strip self-closing JSX components like `<Tabs />`, `<DocCardList />`.
-  cleaned = cleaned.replace(/^[ \t]*<[A-Z][A-Za-z0-9]*[^>]*\/>[ \t]*$/gm, "");
-
-  // Collapse 3+ blank lines into 2.
-  cleaned = cleaned.replace(/\n{3,}/g, "\n\n");
-
-  return cleaned.trimStart();
+function cleanMdxBody(body, sourceRelPath) {
+  return expandMdxBody(body, {
+    rootDir,
+    docsDir,
+    sourcePath: sourceRelPath,
+  });
 }
 
 /**
@@ -157,9 +138,16 @@ function cleanMdxBody(body) {
 function buildMarkdownForDoc(docId, route, sourcePath, frontmatter) {
   if (!sourcePath || !fs.existsSync(sourcePath)) return null;
 
-  const raw = fs.readFileSync(sourcePath, "utf8");
-  const { body } = parseFrontmatter(raw);
-  const cleaned = cleanMdxBody(body);
+  const htmlBody = markdownFromBuiltHtml(buildDir, route);
+  let cleaned;
+  if (htmlBody) {
+    cleaned = htmlBody;
+  } else {
+    const raw = fs.readFileSync(sourcePath, "utf8");
+    const { body } = parseFrontmatter(raw);
+    const sourceRelPath = path.relative(docsDir, sourcePath);
+    cleaned = cleanMdxBody(body, sourceRelPath);
+  }
 
   const title =
     (typeof frontmatter.title === "string" && frontmatter.title.trim()) ||

scripts/generate-md-routes.mjs

@@ -0,0 +1,78 @@
+#!/usr/bin/env node
+/**
+ * Convert Docusaurus-built doc HTML into agent-facing markdown.
+ * Uses the same `.theme-doc-markdown` region users see in the browser.
+ */
+
+import fs from "fs";
+import path from "path";
+import * as cheerio from "cheerio";
+import TurndownService from "turndown";
+
+const CONTENT_SELECTOR = ".theme-doc-markdown";
+
+function findHtmlPathForRoute(buildDir, route) {
+  const trimmed = route.replace(/\/+$/, "");
+  if (trimmed === "" || trimmed === "/") {
+    const index = path.join(buildDir, "index.html");
+    return fs.existsSync(index) ? index : null;
+  }
+  const rel = trimmed.startsWith("/") ? trimmed.slice(1) : trimmed;
+  const flat = path.join(buildDir, `${rel}.html`);
+  if (fs.existsSync(flat)) return flat;
+  const nested = path.join(buildDir, rel, "index.html");
+  if (fs.existsSync(nested)) return nested;
+  return null;
+}
+
+function stripNonContentNodes($, root) {
+  root
+    .find("button, svg, .tabs__item")
+    .add(root.find('[class*="copy"], [class*="Copy"]'))
+    .remove();
+}
+
+function createTurndown() {
+  const service = new TurndownService({
+    headingStyle: "atx",
+    codeBlockStyle: "fenced",
+    emDelimiter: "*",
+    bulletListMarker: "-",
+  });
+
+  service.addRule("details", {
+    filter: "details",
+    replacement(content, node) {
+      const summary =
+        node.querySelector("summary")?.textContent?.trim() || "Details";
+      return `\n<details>\n<summary>${summary}</summary>\n\n${content.trim()}\n\n</details>\n`;
+    },
+  });
+
+  return service;
+}
+
+/**
+ * @param {string} buildDir
+ * @param {string} route e.g. "/ftso/feeds"
+ * @returns {string | null}
+ */
+export function markdownFromBuiltHtml(buildDir, route) {
+  const htmlPath = findHtmlPathForRoute(buildDir, route);
+  if (!htmlPath) return null;
+
+  const html = fs.readFileSync(htmlPath, "utf8");
+  const $ = cheerio.load(html);
+  const content = $(CONTENT_SELECTOR).first();
+  if (!content.length) return null;
+
+  stripNonContentNodes($, content);
+
+  const turndown = createTurndown();
+  let body = turndown.turndown(content.html() || "").trim();
+
+  // Drop duplicate top-level H1; generate-md-routes adds its own title block.
+  body = body.replace(/^#\s+.+\n+/, "");
+
+  return body.trim() || null;
+}

scripts/html-to-markdown.mjs

@@ -0,0 +1,58 @@
+#!/usr/bin/env node
+/**
+ * Tag interactive / duplicated HTML regions so Agent Score markdown-content-parity
+ * compares prose only. Canonical code and tables live in the emitted .md routes.
+ */
+
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+import * as cheerio from "cheerio";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const buildDir = path.join(__dirname, "..", "build");
+
+function walkHtmlFiles(dir, list = []) {
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) walkHtmlFiles(full, list);
+    else if (entry.name.endsWith(".html")) list.push(full);
+  }
+  return list;
+}
+
+function markExclusions(html) {
+  const $ = cheerio.load(html);
+  const root = $(".theme-doc-markdown").first();
+  if (!root.length) return null;
+
+  root
+    .find("pre, .theme-tabs-container, .codeBlockContainer_Ckt0")
+    .each((_, el) => {
+      $(el).attr("data-markdown-ignore", "");
+    });
+
+  return $.html();
+}
+
+function main() {
+  if (!fs.existsSync(buildDir)) {
+    console.warn("[mark-html-parity-exclusions] build/ not found, skipping.");
+    return;
+  }
+
+  let updated = 0;
+  for (const file of walkHtmlFiles(buildDir)) {
+    const raw = fs.readFileSync(file, "utf8");
+    const next = markExclusions(raw);
+    if (!next || next === raw) continue;
+    fs.writeFileSync(file, next, "utf8");
+    updated++;
+  }
+
+  console.log(
+    `[mark-html-parity-exclusions] Tagged code/tabs in ${updated} HTML files.`,
+  );
+}
+
+main();