[docs-infra] Create llms.txt

.gitignore

@@ -27,3 +27,7 @@ size-snapshot.json
 .github/styles/
 .nx/cache
 .nx/workspace-data
+
+# Remove if you want to pregenerate the docs md files
+docs/public/llms.txt
+docs/public/react

docs/package.json

@@ -9,7 +9,8 @@
     "deploy": "git push -f upstream master:docs-v1",
     "serve": "serve ./export -l 3010",
     "typescript": "tsc -b tsconfig.json",
-    "link-check": "tsx ./scripts/reportBrokenLinks.mts"
+    "link-check": "tsx ./scripts/reportBrokenLinks.mts",
+    "generate-llms": "node ./scripts/generateLlmTxt/index.mjs"
   },
   "dependencies": {
     "@base-ui-components/react": "workspace:*",
@@ -18,7 +19,6 @@
     "@emotion/server": "^11.11.0",
     "@emotion/styled": "^11.14.0",
     "@mdx-js/loader": "^3.1.0",
-    "@mdx-js/mdx": "^3.1.0",
     "@mdx-js/react": "^3.1.0",
     "@mui/system": "7.0.2",
     "@next/mdx": "^15.3.1",
@@ -55,11 +55,11 @@
     "scroll-into-view-if-needed": "3.1.0",
     "shiki": "^3.3.0",
     "to-vfile": "^8.0.0",
-    "unist-util-visit": "^5.0.0",
     "unist-util-visit-parents": "^6.0.1",
     "vfile-matter": "^5.0.1"
   },
   "devDependencies": {
+    "@mdx-js/mdx": "^3.1.0",
     "@mui/internal-docs-utils": "^2.0.1",
     "@mui/internal-scripts": "^2.0.7",
     "@mui/internal-test-utils": "^2.0.7",
@@ -76,10 +76,14 @@
     "mdast-util-mdx-jsx": "^3.2.0",
     "motion": "^12.9.4",
     "prettier": "^3.5.3",
+    "react-reconciler": "^0.32.0",
+    "remark-parse": "^11.0.0",
+    "remark-stringify": "^11.0.0",
     "rimraf": "^6.0.1",
     "serve": "^14.2.4",
     "tailwindcss": "4.1.5",
     "unified": "^11.0.5",
+    "unist-util-visit": "^5.0.0",
     "webpack-bundle-analyzer": "^4.10.2",
     "yargs": "^17.7.2",
     "zod": "^3.24.3"

docs/scripts/generateLlmTxt/demoProcessor.mjs

@@ -0,0 +1,121 @@
+/**
+ * demoProcessor.mjs - Process demo component directories
+ *
+ * This module handles loading and converting demo code examples
+ * into markdown code blocks for documentation.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import * as mdx from './mdxNodeHelpers.mjs';
+
+/**
+ * Read all files from a directory
+ * @param {string} directory - The directory to read
+ * @returns {Array<string>} Array of file paths
+ */
+function readDirFiles(directory) {
+  return fs
+    .readdirSync(directory)
+    .filter((file) => !fs.statSync(path.join(directory, file)).isDirectory())
+    .map((file) => path.join(directory, file));
+}
+
+/**
+ * Create a code block for a file with a comment header
+ * @param {string} filePath - Path to the file
+ * @param {string} relativePath - Relative path to show in the comment
+ * @returns {Object} Code node with the file content
+ */
+function createFileCodeBlock(filePath, relativePath) {
+  const content = fs.readFileSync(filePath, 'utf-8');
+  const extension = path.extname(filePath).slice(1);
+
+  // Add comment header with filename
+  const commentedContent = `/* ${relativePath} */\n${content}`;
+
+  // Create code block with appropriate language
+  return mdx.code(commentedContent, extension);
+}
+
+/**
+ * Transforms a Demo component into markdown code blocks
+ * @param {Object} node - The Demo JSX node from MDX
+ * @param {string} mdxFilePath - Path to the MDX file containing the Demo component
+ * @returns {Array} Array of markdown nodes to replace the Demo component
+ */
+export function processDemo(node, mdxFilePath) {
+  // Extract path attribute
+  const pathAttr = node.attributes?.find((attr) => attr.name === 'path')?.value;
+
+  if (!pathAttr) {
+    throw new Error('Missing "path" prop on the "<Demo />" component.');
+  }
+
+  // Resolve demo path relative to the MDX file
+  const mdxDir = path.dirname(mdxFilePath);
+  const demoPath = path.resolve(mdxDir, pathAttr);
+
+  // Check if the demo folder exists
+  if (!fs.existsSync(demoPath)) {
+    throw new Error(`Demo folder not found at "${demoPath}"`);
+  }
+
+  // Define implementation types and their configurations
+  const implementationTypes = [
+    {
+      id: 'css-modules',
+      title: 'CSS Modules',
+      description: 'This example shows how to implement the component using CSS Modules.',
+    },
+    {
+      id: 'tailwind',
+      title: 'Tailwind',
+      description: 'This example shows how to implement the component using Tailwind CSS.',
+    },
+  ];
+
+  // Find which implementation types exist in the demo folder
+  const availableImplementations = implementationTypes.filter((type) => {
+    const typePath = path.join(demoPath, type.id);
+    return fs.existsSync(typePath);
+  });
+
+  // Throw error if no implementation types are found
+  if (availableImplementations.length === 0) {
+    throw new Error(
+      `No implementation types found at "${demoPath}". Expected one of: ${implementationTypes.map((t) => t.id).join(', ')}`,
+    );
+  }
+
+  const result = [];
+
+  // Add main Demo heading
+  result.push(mdx.heading(2, 'Demo'));
+
+  /**
+   * Process a specific implementation type
+   * @param {string} folderPath - Path to the implementation folder
+   * @param {string} title - Title for the section heading
+   * @param {string} description - Description text for the section
+   */
+  function processImplementation(folderPath, title, description) {
+    result.push(mdx.heading(3, title));
+    result.push(mdx.paragraph(description));
+
+    const files = readDirFiles(folderPath);
+
+    files.forEach((file) => {
+      const relativePath = path.relative(folderPath, file);
+      result.push(createFileCodeBlock(file, relativePath));
+    });
+  }
+
+  // Process each available implementation type
+  availableImplementations.forEach((impl) => {
+    const implPath = path.join(demoPath, impl.id);
+    processImplementation(implPath, impl.title, impl.description);
+  });
+
+  return result;
+}

docs/scripts/generateLlmTxt/index.mjs

@@ -0,0 +1,202 @@
+#!/usr/bin/env node
+/* eslint-disable no-await-in-loop */
+/* eslint-disable no-console */
+
+import fs from 'fs/promises';
+import path from 'path';
+import glob from 'fast-glob';
+import * as prettier from 'prettier';
+import { mdxToMarkdown } from './mdxToMarkdown.mjs';
+
+const PROJECT_ROOT = path.resolve(import.meta.dirname, '../..');
+const MDX_SOURCE_DIR = path.join(PROJECT_ROOT, 'src/app/(public)/(content)/react');
+const OUTPUT_BASE_DIR = path.join(PROJECT_ROOT, 'public');
+const OUTPUT_REACT_DIR = path.join(OUTPUT_BASE_DIR, 'react');
+
+/**
+ * Generate llms.txt and markdown files from MDX content
+ */
+async function generateLlmsTxt() {
+  console.log('Generating llms.txt and markdown files...');
+
+  try {
+    // Create output directories if they don't exist
+    await fs.mkdir(OUTPUT_BASE_DIR, { recursive: true });
+    await fs.mkdir(OUTPUT_REACT_DIR, { recursive: true });
+
+    // Store metadata for each section
+    const metadataBySection = {
+      overview: [],
+      handbook: [],
+      components: [],
+      utils: [],
+    };
+
+    // Process files from a specific section
+    const processSection = async (sectionName) => {
+      console.log(`Processing ${sectionName} section...`);
+
+      // Find all MDX files in this section
+      const sectionPath = path.join(MDX_SOURCE_DIR, sectionName);
+      const mdxFiles = await glob('**/*/page.mdx', {
+        cwd: sectionPath,
+        absolute: true,
+      });
+
+      console.log(`Found ${mdxFiles.length} files in ${sectionName}`);
+
+      for (const mdxFile of mdxFiles) {
+        const relativePath = path.relative(MDX_SOURCE_DIR, mdxFile);
+        const dirPath = path.dirname(relativePath);
+        const urlPath = dirPath.replace(/\\/g, '/');
+        const outputFilePath = path.join(OUTPUT_REACT_DIR, `${dirPath}.md`);
+
+        const mdxContent = await fs.readFile(mdxFile, 'utf-8');
+
+        const { markdown, title, subtitle, description } = await mdxToMarkdown(mdxContent, mdxFile);
+
+        // Create directories for output if needed
+        await fs.mkdir(path.dirname(outputFilePath), { recursive: true });
+
+        // Create markdown content with frontmatter
+        let content = [
+          '---',
+          `title: ${title || 'Untitled'}`,
+          subtitle ? `subtitle: ${subtitle}` : '',
+          description ? `description: ${description}` : '',
+          '---',
+          '',
+          markdown,
+        ]
+          .filter(Boolean)
+          .join('\n');
+
+        // Format markdown with frontmatter using prettier
+        const prettierOptions = await prettier.resolveConfig(outputFilePath);
+
+        content = await prettier.format(content, {
+          ...prettierOptions,
+          filepath: outputFilePath,
+          parser: 'markdown',
+        });
+
+        // Write formatted markdown file
+        await fs.writeFile(outputFilePath, content, 'utf-8');
+
+        // Extract the filename without extension to use as id
+        const fileId = path.basename(dirPath);
+
+        // Store metadata for this file in the appropriate section
+        metadataBySection[sectionName].push({
+          id: fileId,
+          title: title || 'Untitled',
+          subtitle: subtitle || '',
+          description: description || '',
+          urlPath: `./react/${urlPath}.md`,
+        });
+
+        console.log(`Processed: ${relativePath}`);
+      }
+    };
+
+    // Process each section
+    await processSection('overview');
+    await processSection('handbook');
+    await processSection('components');
+    await processSection('utils');
+
+    // Build structured content for llms.txt
+    const sections = ['# Base UI', ''];
+
+    sections.push(
+      'This is the documentation for the `@base-ui-components/react` package.',
+      'It contains a collection of components and utilities for building user interfaces in React.',
+      'The library is designed to be composable and styling agnostic.',
+      '',
+    );
+
+    // Create formatted sections in specific order
+    const formatSection = (items, title) => {
+      if (items.length > 0) {
+        sections.push(`## ${title}`, '');
+
+        // Add each item as a link with description, starting with a bullet (-)
+        items.forEach((item) => {
+          sections.push(`- [${item.title}](${item.urlPath}): ${item.description}`);
+        });
+
+        sections.push(''); // Add empty line after section
+      }
+    };
+
+    // Define specific orders for sections
+    const overviewOrder = ['quick-start', 'accessibility', 'releases', 'about'];
+    const handbookOrder = ['styling', 'animation', 'composition'];
+
+    // Validate that all expected overview items exist
+    overviewOrder.forEach((id) => {
+      if (!metadataBySection.overview.some((item) => item.id === id)) {
+        throw new Error(`Missing expected overview item: ${id}`);
+      }
+    });
+
+    // Validate that all expected handbook items exist
+    handbookOrder.forEach((id) => {
+      if (!metadataBySection.handbook.some((item) => item.id === id)) {
+        throw new Error(`Missing expected handbook item: ${id}`);
+      }
+    });
+
+    // Sort overview by predefined order
+    const sortedOverview = [...metadataBySection.overview].sort((a, b) => {
+      return overviewOrder.indexOf(a.id) - overviewOrder.indexOf(b.id);
+    });
+
+    // Sort handbook by predefined order
+    const sortedHandbook = [...metadataBySection.handbook].sort((a, b) => {
+      return handbookOrder.indexOf(a.id) - handbookOrder.indexOf(b.id);
+    });
+
+    // Sort components and utilities alphabetically by id
+    const sortedComponents = [...metadataBySection.components].sort((a, b) =>
+      a.id.localeCompare(b.id),
+    );
+    const sortedUtils = [...metadataBySection.utils].sort((a, b) => a.id.localeCompare(b.id));
+
+    // Add sections in the required order
+    formatSection(sortedOverview, 'Overview');
+    formatSection(sortedHandbook, 'Handbook');
+    formatSection(sortedComponents, 'Components');
+    formatSection(sortedUtils, 'Utilities');
+
+    // Create llms.txt content and format with prettier
+    let llmsTxtContent = sections.join('\n');
+
+    // Apply prettier formatting using the project's configuration
+    const llmsFilePath = path.join(OUTPUT_BASE_DIR, 'llms.txt');
+    const prettierOptions = await prettier.resolveConfig(llmsFilePath);
+
+    llmsTxtContent = await prettier.format(llmsTxtContent, {
+      ...prettierOptions,
+      filepath: llmsFilePath,
+      parser: 'markdown',
+    });
+
+    await fs.writeFile(path.join(OUTPUT_BASE_DIR, 'llms.txt'), llmsTxtContent, 'utf-8');
+
+    // Calculate the total number of files processed
+    const totalFiles =
+      metadataBySection.overview.length +
+      metadataBySection.handbook.length +
+      metadataBySection.components.length +
+      metadataBySection.utils.length;
+
+    console.log(`Successfully generated ${totalFiles} markdown files and llms.txt`);
+  } catch (error) {
+    console.error('Error generating llms.txt:', error);
+    process.exit(1);
+  }
+}
+
+// Run the generator
+generateLlmsTxt();