Refactor LLMs generation and add static root index

Switch from auto-generating a monolithic llms.txt/llms-full.txt to a hand-curated static/llms.txt and per-section indexes. Wrap docusaurus-plugin-llms with an llms-html-injector wrapper so generation runs before post-processing (normalizing .md layout, rewriting llms*.txt URLs, and injecting <link rel="alternate">). Update docusaurus.config.js to disable the classic docs slot, disable root/full LLM generators, and adjust path transformations. Update scripts/verify-llms-output.js to mirror the new behavior (customLLMFiles list, disabled root generators, post-process step, and copy static/llms.txt into the verify outDir). Add static/llms.txt, add vercel redirects for the removed whats-new page, remove docs/whats-new.md, and tweak a navbar CSS comment. Export postProcessLlmsOutput for reuse by the verify script.

Author: shahbaz17

docs/whats-new.md

@@ -162,16 +162,13 @@ const config = {
     [
       'classic',
       {
-        docs: {
-          id: 'docs',
-          path: 'docs',
-          routeBasePath: '/',
-          editUrl: 'https://github.com/MetaMask/metamask-docs/edit/main/',
-          sidebarPath: false,
-          breadcrumbs: false,
-          remarkPlugins,
-          rehypePlugins,
-        },
+        // The classic preset's `docs` plugin is disabled. It previously
+        // rendered a single orphan page (`docs/whats-new.md`) at `/whats-new/`
+        // that pointed offsite for release notes. The page had no incoming
+        // internal links and is redirected to `/` in `vercel.json`. Per-product
+        // documentation is served by dedicated `plugin-content-docs` instances
+        // configured below in `plugins`, not by this preset slot.
+        docs: false,
         pages: {
           path: 'src/pages',
           routeBasePath: '/',
@@ -323,39 +320,32 @@ const config = {
         containerId: 'GTM-5FGPLC2Q',
       },
     ],
+    // The llms-html-injector plugin wraps docusaurus-plugin-llms so that
+    // generation (per-section `llms-<product>.txt` files and per-page `.md`
+    // files) runs sequentially before our post-processing (path normalization,
+    // index URL rewrites, and `<link rel="alternate" type="text/markdown">`
+    // injection). Wrapping is required because Docusaurus 3.x executes
+    // `postBuild` hooks concurrently via `Promise.all`, so registering both
+    // plugins separately would let the injector race against the generator.
     [
-      'docusaurus-plugin-llms',
+      './src/plugins/llms-html-injector',
       {
         // Set docsDir to site root to collect from all directories
         docsDir: '.',
-        // Generate a comprehensive root llms.txt + llms-full.txt covering every page
-        // (Agent Score: llms-txt-coverage check). The per-section files below remain
-        // useful as scoped indexes for AI tools.
-        generateLLMsTxt: true,
-        generateLLMsFullTxt: true,
-        // Emit an individual .md file next to every doc page so .md URLs return 200
-        // (Agent Score: markdown-url-support and llms-txt-directive-md checks). The
-        // companion llms-html-injector plugin (registered below) normalizes these
+        // The root llms.txt is hand-curated at static/llms.txt to stay under
+        // the AFDocs 50,000-char `llms-txt-size` threshold. The aggregate
+        // walker descends one level into the per-section indexes listed below,
+        // so coverage is preserved without dumping every page into a flat
+        // index. Likewise, a monolithic llms-full.txt would exceed every
+        // agent's context window (and crashes browser tabs at multi-MB
+        // sizes); the per-section *-full.txt files cover the bulk content.
+        generateLLMsTxt: false,
+        generateLLMsFullTxt: false,
+        // Emit an individual .md file next to every doc page so .md URLs
+        // return 200 (Agent Score: markdown-url-support and
+        // llms-txt-directive-md checks). The wrapper below normalizes these
         // files to URL-aligned paths and injects <link rel="alternate"> tags.
         generateMarkdownFiles: true,
-        // Custom framing for the root index, preserving the prose previously
-        // hand-maintained in static/llms.txt.
-        rootContent: `> MetaMask is the leading self-custodial cryptocurrency wallet and Web3 gateway, enabling developers to build dapps that connect to the MetaMask browser extension and mobile app across EVM and Solana ecosystems.
-
-This file follows the [llmstxt.org](https://llmstxt.org/) specification. Section-scoped indexes are also available:
-
-- [MetaMask Connect](https://docs.metamask.io/llms-metamask-connect.txt) ([full](https://docs.metamask.io/llms-metamask-connect-full.txt))
-- [Embedded Wallets](https://docs.metamask.io/llms-embedded-wallets.txt) ([full](https://docs.metamask.io/llms-embedded-wallets-full.txt))
-- [Smart Accounts Kit](https://docs.metamask.io/llms-smart-accounts-kit.txt) ([full](https://docs.metamask.io/llms-smart-accounts-kit-full.txt))
-- [Services](https://docs.metamask.io/llms-services.txt) ([full](https://docs.metamask.io/llms-services-full.txt))
-- [Developer dashboard](https://docs.metamask.io/llms-dashboard.txt) ([full](https://docs.metamask.io/llms-dashboard-full.txt))
-- [Snaps](https://docs.metamask.io/llms-snaps.txt) ([full](https://docs.metamask.io/llms-snaps-full.txt))
-- [Tutorials](https://docs.metamask.io/llms-tutorials.txt) ([full](https://docs.metamask.io/llms-tutorials-full.txt))
-`,
-        fullRootContent: `> MetaMask is the leading self-custodial cryptocurrency wallet and Web3 gateway, enabling developers to build dapps that connect to the MetaMask browser extension and mobile app across EVM and Solana ecosystems.
-
-This file contains the complete MetaMask developer documentation in a single markdown document, following the [llmstxt.org](https://llmstxt.org/) specification.
-`,
         // Ignore common non-doc directories
         // Note: src/pages/** is not ignored so tutorials can be collected
         // Each customLLMFiles entry filters by includePatterns, so only matching files are included
@@ -397,10 +387,13 @@ This file contains the complete MetaMask developer documentation in a single mar
         ],
         excludeImports: true,
         removeDuplicateHeadings: true,
-        // Path transformation to fix URL construction
-        // Since docsDir is '.', we need to remove 'docs/' prefix and handle src/pages paths
+        // Strip `src/pages` from generated URLs so a file at
+        // `src/pages/quickstart/foo.md` resolves to `/quickstart/foo/` to
+        // match the public route. `docsDir` is `.` (the site root) so any
+        // future top-level prefixes that should be hidden from URLs would
+        // be added here.
         pathTransformation: {
-          ignorePaths: ['docs', 'src/pages'],
+          ignorePaths: ['src/pages'],
         },
         // Generate separate files for each section
         customLLMFiles: [
@@ -519,11 +512,6 @@ This file contains the complete MetaMask developer documentation in a single mar
         ],
       },
     ],
-    // Runs in postBuild after docusaurus-plugin-llms generates files. Normalizes
-    // .md files to URL-aligned paths, rewrites llms*.txt URLs to match, and
-    // injects <link rel="alternate" type="text/markdown"> into every doc page's
-    // <head> (Agent Score: llms-txt-directive-html / llms-txt-directive-md).
-    './src/plugins/llms-html-injector',
   ],
   clientModules: [require.resolve('./src/client/scroll-fix.js')],
   themeConfig:

docusaurus.config.js

@@ -3,16 +3,136 @@
  * Dev-only sanity check that invokes docusaurus-plugin-llms against the current
  * docs source tree, using the same options we configured in docusaurus.config.js.
  *
- * This lets us verify the new generateMarkdownFiles output and llms.txt coverage
- * without doing a full `docusaurus build` (which is currently blocked by an
- * unrelated swizzled-Tabs incompatibility on this machine).
+ * After the move to a hand-curated root `static/llms.txt`, the standard
+ * llms.txt / llms-full.txt generators are disabled in production. This script
+ * mirrors that: both flags are off, customLLMFiles drives every per-section
+ * file, and the static root is copied into the verify outDir so the post-
+ * injector summary reflects what end users actually see at /llms.txt.
  *
  * Usage: node scripts/verify-llms-output.js [outDir]
  */
 
 const path = require('path')
 const fs = require('fs/promises')
 
+// Keep this list in sync with the customLLMFiles array in docusaurus.config.js.
+// Section indexes (no fullContent) stay under the AFDocs 50KB recommendation
+// for individual indexes; -full.txt files contain the bulk content per product.
+const CUSTOM_LLM_FILES = [
+  {
+    filename: 'llms-embedded-wallets.txt',
+    includePatterns: ['embedded-wallets/**/*.{md,mdx}'],
+    fullContent: false,
+    title: 'MetaMask Embedded Wallets documentation',
+    description: 'Documentation links for MetaMask Embedded Wallets',
+  },
+  {
+    filename: 'llms-embedded-wallets-full.txt',
+    includePatterns: ['embedded-wallets/**/*.{md,mdx}'],
+    fullContent: true,
+    title: 'MetaMask Embedded Wallets documentation',
+    description: 'Complete documentation for MetaMask Embedded Wallets',
+  },
+  {
+    filename: 'llms-metamask-connect.txt',
+    includePatterns: ['metamask-connect/**/*.{md,mdx}'],
+    fullContent: false,
+    title: 'MetaMask Connect documentation',
+    description: 'Documentation links for MetaMask Connect',
+  },
+  {
+    filename: 'llms-metamask-connect-full.txt',
+    includePatterns: ['metamask-connect/**/*.{md,mdx}'],
+    fullContent: true,
+    title: 'MetaMask Connect documentation',
+    description: 'Complete documentation for MetaMask Connect',
+  },
+  {
+    filename: 'llms-smart-accounts-kit.txt',
+    includePatterns: ['smart-accounts-kit/**/*.{md,mdx}'],
+    fullContent: false,
+    title: 'MetaMask Smart Accounts Kit documentation',
+    description: 'Documentation links for MetaMask Smart Accounts Kit',
+  },
+  {
+    filename: 'llms-smart-accounts-kit-full.txt',
+    includePatterns: ['smart-accounts-kit/**/*.{md,mdx}'],
+    fullContent: true,
+    title: 'MetaMask Smart Accounts Kit documentation',
+    description: 'Complete documentation for MetaMask Smart Accounts Kit',
+  },
+  {
+    filename: 'llms-snaps.txt',
+    includePatterns: ['snaps/**/*.{md,mdx}'],
+    fullContent: false,
+    title: 'Snaps documentation',
+    description: 'Documentation links for Snaps',
+  },
+  {
+    filename: 'llms-snaps-full.txt',
+    includePatterns: ['snaps/**/*.{md,mdx}'],
+    fullContent: true,
+    title: 'Snaps documentation',
+    description: 'Complete documentation for Snaps',
+  },
+  {
+    filename: 'llms-wallet.txt',
+    includePatterns: ['wallet/**/*.{md,mdx}'],
+    fullContent: false,
+    title: 'Wallet API documentation',
+    description: 'Documentation links for Wallet API',
+  },
+  {
+    filename: 'llms-wallet-full.txt',
+    includePatterns: ['wallet/**/*.{md,mdx}'],
+    fullContent: true,
+    title: 'Wallet API documentation',
+    description: 'Complete documentation for Wallet API',
+  },
+  {
+    filename: 'llms-tutorials.txt',
+    includePatterns: ['src/pages/tutorials/**/*.{md,mdx}'],
+    fullContent: false,
+    title: 'Tutorials',
+    description: 'Documentation links for MetaMask tutorials',
+  },
+  {
+    filename: 'llms-tutorials-full.txt',
+    includePatterns: ['src/pages/tutorials/**/*.{md,mdx}'],
+    fullContent: true,
+    title: 'Tutorials',
+    description: 'Complete documentation for MetaMask tutorials',
+  },
+  {
+    filename: 'llms-dashboard.txt',
+    includePatterns: ['developer-tools/dashboard/**/*.{md,mdx}'],
+    fullContent: false,
+    title: 'Developer dashboard documentation',
+    description: 'Documentation links for MetaMask Developer dashboard',
+  },
+  {
+    filename: 'llms-dashboard-full.txt',
+    includePatterns: ['developer-tools/dashboard/**/*.{md,mdx}'],
+    fullContent: true,
+    title: 'Developer dashboard documentation',
+    description: 'Complete documentation for MetaMask Developer dashboard',
+  },
+  {
+    filename: 'llms-services.txt',
+    includePatterns: ['services/**/*.md'],
+    fullContent: false,
+    title: 'Services documentation',
+    description: 'Documentation links for MetaMask services',
+  },
+  {
+    filename: 'llms-services-full.txt',
+    includePatterns: ['services/**/*.md'],
+    fullContent: true,
+    title: 'Services documentation',
+    description: 'Complete documentation for MetaMask services',
+  },
+]
+
 async function main() {
   const siteDir = path.resolve(__dirname, '..')
   const outDir = path.resolve(siteDir, process.argv[2] || '.llms-verify')
@@ -26,12 +146,12 @@ async function main() {
 
   const options = {
     docsDir: '.',
-    generateLLMsTxt: true,
-    generateLLMsFullTxt: true,
+    generateLLMsTxt: false,
+    generateLLMsFullTxt: false,
     generateMarkdownFiles: true,
     excludeImports: true,
     removeDuplicateHeadings: true,
-    pathTransformation: { ignorePaths: ['docs', 'src/pages'] },
+    pathTransformation: { ignorePaths: ['src/pages'] },
     ignoreFiles: [
       'node_modules/**',
       'build/**',
@@ -66,7 +186,7 @@ async function main() {
       'LICENSE*',
       'gator_versioned_docs/**',
     ],
-    customLLMFiles: [],
+    customLLMFiles: CUSTOM_LLM_FILES,
   }
 
   const context = {
@@ -84,17 +204,38 @@ async function main() {
   console.log(`Collected ${allDocFiles.length} source markdown files`)
 
   await generator.generateStandardLLMFiles(context, allDocFiles)
+  await generator.generateCustomLLMFiles(context, allDocFiles)
 
   console.log('\n--- Pre-injector summary ---')
   printSummary(await summarize(outDir))
 
-  const injector = require(path.join(siteDir, 'src/plugins/llms-html-injector'))()
-  await injector.postBuild({ outDir, routesPaths: [] })
+  // Invoke only the post-processing stage. The injector module also exports a
+  // wrapper plugin (used in docusaurus.config.js) that internally instantiates
+  // docusaurus-plugin-llms; we've already run the generator above, so we skip
+  // straight to normalize/rewrite/inject.
+  const { postProcessLlmsOutput } = require(path.join(siteDir, 'src/plugins/llms-html-injector'))
+  await postProcessLlmsOutput(outDir)
+
+  // Mirror Docusaurus's static-asset copy step. In a real build, anything in
+  // `static/` is copied verbatim to `outDir`, so static/llms.txt becomes
+  // build/llms.txt. Replicating that here lets the post-injector summary show
+  // the curated root file end users will receive.
+  await copyStaticLlms(siteDir, outDir)
 
   console.log('\n--- Post-injector summary ---')
   printSummary(await summarize(outDir))
 }
 
+async function copyStaticLlms(siteDir, outDir) {
+  const src = path.join(siteDir, 'static', 'llms.txt')
+  const dest = path.join(outDir, 'llms.txt')
+  try {
+    await fs.copyFile(src, dest)
+  } catch (err) {
+    console.warn(`[verify-llms-output] Could not copy static/llms.txt: ${err.message}`)
+  }
+}
+
 function printSummary(summary) {
   for (const [key, value] of Object.entries(summary)) {
     console.log(`${key}: ${Array.isArray(value) ? value.join(', ') : value}`)
@@ -129,7 +270,21 @@ async function summarize(outDir) {
     const s = await fs.stat(llmsFull)
     summary.llmsFullTxtBytes = s.size
   } catch {
-    summary.llmsFullTxt = 'MISSING'
+    summary.llmsFullTxt = 'ABSENT (expected — generation disabled)'
+  }
+  // Per-section files
+  const sectionEntries = await fs.readdir(outDir, { withFileTypes: true })
+  const sectionFiles = sectionEntries
+    .filter(e => e.isFile() && /^llms-[^.]+\.txt$/.test(e.name))
+    .map(e => e.name)
+    .sort()
+  if (sectionFiles.length > 0) {
+    const sizes = []
+    for (const name of sectionFiles) {
+      const s = await fs.stat(path.join(outDir, name))
+      sizes.push(`${name}=${s.size}`)
+    }
+    summary.sectionFiles = sizes
   }
   summary.sampleMdFiles = mdFiles
     .filter(f => /\.md$/.test(f) && !/llms[^/]*\.(md|txt)$/.test(f))