diff --git a/.gitignore b/.gitignore index 854e577c5f..21d91a2a7f 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,7 @@ size-snapshot.json .github/styles/ .nx/cache .nx/workspace-data + +# Remove if you want to pregenerate the docs md files +docs/public/llms.txt +docs/public/react diff --git a/docs/package.json b/docs/package.json index 3952577fc8..e5dab36f0a 100644 --- a/docs/package.json +++ b/docs/package.json @@ -9,7 +9,8 @@ "deploy": "git push -f upstream master:docs-v1", "serve": "serve ./export -l 3010", "typescript": "tsc -b tsconfig.json", - "link-check": "tsx ./scripts/reportBrokenLinks.mts" + "link-check": "tsx ./scripts/reportBrokenLinks.mts", + "generate-llms": "node ./scripts/generateLlmTxt/index.mjs" }, "dependencies": { "@base-ui-components/react": "workspace:*", @@ -18,7 +19,6 @@ "@emotion/server": "^11.11.0", "@emotion/styled": "^11.14.0", "@mdx-js/loader": "^3.1.0", - "@mdx-js/mdx": "^3.1.0", "@mdx-js/react": "^3.1.0", "@mui/system": "7.0.2", "@next/mdx": "^15.3.3", @@ -55,11 +55,11 @@ "scroll-into-view-if-needed": "3.1.0", "shiki": "^3.3.0", "to-vfile": "^8.0.0", - "unist-util-visit": "^5.0.0", "unist-util-visit-parents": "^6.0.1", "vfile-matter": "^5.0.1" }, "devDependencies": { + "@mdx-js/mdx": "^3.1.0", "@mui/internal-docs-utils": "^2.0.1", "@mui/internal-scripts": "^2.0.7", "@mui/internal-test-utils": "^2.0.7", @@ -76,10 +76,14 @@ "mdast-util-mdx-jsx": "^3.2.0", "motion": "^12.9.4", "prettier": "^3.5.3", + "react-reconciler": "^0.32.0", + "remark-parse": "^11.0.0", + "remark-stringify": "^11.0.0", "rimraf": "^6.0.1", "serve": "^14.2.4", "tailwindcss": "4.1.8", "unified": "^11.0.5", + "unist-util-visit": "^5.0.0", "webpack-bundle-analyzer": "^4.10.2", "yargs": "^17.7.2", "zod": "^3.24.3" diff --git a/docs/scripts/generateLlmTxt/demoProcessor.mjs b/docs/scripts/generateLlmTxt/demoProcessor.mjs new file mode 100644 index 0000000000..7e6d283415 --- /dev/null +++ b/docs/scripts/generateLlmTxt/demoProcessor.mjs @@ -0,0 +1,121 @@ +/** + * demoProcessor.mjs - Process demo component directories + * + * This module handles loading and converting demo code examples + * into markdown code blocks for documentation. + */ + +import fs from 'fs'; +import path from 'path'; +import * as mdx from './mdxNodeHelpers.mjs'; + +/** + * Read all files from a directory + * @param {string} directory - The directory to read + * @returns {Array} Array of file paths + */ +function readDirFiles(directory) { + return fs + .readdirSync(directory) + .filter((file) => !fs.statSync(path.join(directory, file)).isDirectory()) + .map((file) => path.join(directory, file)); +} + +/** + * Create a code block for a file with a comment header + * @param {string} filePath - Path to the file + * @param {string} relativePath - Relative path to show in the comment + * @returns {Object} Code node with the file content + */ +function createFileCodeBlock(filePath, relativePath) { + const content = fs.readFileSync(filePath, 'utf-8'); + const extension = path.extname(filePath).slice(1); + + // Add comment header with filename + const commentedContent = `/* ${relativePath} */\n${content}`; + + // Create code block with appropriate language + return mdx.code(commentedContent, extension); +} + +/** + * Transforms a Demo component into markdown code blocks + * @param {Object} node - The Demo JSX node from MDX + * @param {string} mdxFilePath - Path to the MDX file containing the Demo component + * @returns {Array} Array of markdown nodes to replace the Demo component + */ +export function processDemo(node, mdxFilePath) { + // Extract path attribute + const pathAttr = node.attributes?.find((attr) => attr.name === 'path')?.value; + + if (!pathAttr) { + throw new Error('Missing "path" prop on the "" component.'); + } + + // Resolve demo path relative to the MDX file + const mdxDir = path.dirname(mdxFilePath); + const demoPath = path.resolve(mdxDir, pathAttr); + + // Check if the demo folder exists + if (!fs.existsSync(demoPath)) { + throw new Error(`Demo folder not found at "${demoPath}"`); + } + + // Define implementation types and their configurations + const implementationTypes = [ + { + id: 'css-modules', + title: 'CSS Modules', + description: 'This example shows how to implement the component using CSS Modules.', + }, + { + id: 'tailwind', + title: 'Tailwind', + description: 'This example shows how to implement the component using Tailwind CSS.', + }, + ]; + + // Find which implementation types exist in the demo folder + const availableImplementations = implementationTypes.filter((type) => { + const typePath = path.join(demoPath, type.id); + return fs.existsSync(typePath); + }); + + // Throw error if no implementation types are found + if (availableImplementations.length === 0) { + throw new Error( + `No implementation types found at "${demoPath}". Expected one of: ${implementationTypes.map((t) => t.id).join(', ')}`, + ); + } + + const result = []; + + // Add main Demo heading + result.push(mdx.heading(2, 'Demo')); + + /** + * Process a specific implementation type + * @param {string} folderPath - Path to the implementation folder + * @param {string} title - Title for the section heading + * @param {string} description - Description text for the section + */ + function processImplementation(folderPath, title, description) { + result.push(mdx.heading(3, title)); + result.push(mdx.paragraph(description)); + + const files = readDirFiles(folderPath); + + files.forEach((file) => { + const relativePath = path.relative(folderPath, file); + result.push(createFileCodeBlock(file, relativePath)); + }); + } + + // Process each available implementation type + availableImplementations.forEach((impl) => { + const implPath = path.join(demoPath, impl.id); + processImplementation(implPath, impl.title, impl.description); + }); + + return result; +} diff --git a/docs/scripts/generateLlmTxt/index.mjs b/docs/scripts/generateLlmTxt/index.mjs new file mode 100755 index 0000000000..72eacc3f7e --- /dev/null +++ b/docs/scripts/generateLlmTxt/index.mjs @@ -0,0 +1,202 @@ +#!/usr/bin/env node +/* eslint-disable no-await-in-loop */ +/* eslint-disable no-console */ + +import fs from 'fs/promises'; +import path from 'path'; +import glob from 'fast-glob'; +import * as prettier from 'prettier'; +import { mdxToMarkdown } from './mdxToMarkdown.mjs'; + +const PROJECT_ROOT = path.resolve(import.meta.dirname, '../..'); +const MDX_SOURCE_DIR = path.join(PROJECT_ROOT, 'src/app/(public)/(content)/react'); +const OUTPUT_BASE_DIR = path.join(PROJECT_ROOT, 'public'); +const OUTPUT_REACT_DIR = path.join(OUTPUT_BASE_DIR, 'react'); + +/** + * Generate llms.txt and markdown files from MDX content + */ +async function generateLlmsTxt() { + console.log('Generating llms.txt and markdown files...'); + + try { + // Create output directories if they don't exist + await fs.mkdir(OUTPUT_BASE_DIR, { recursive: true }); + await fs.mkdir(OUTPUT_REACT_DIR, { recursive: true }); + + // Store metadata for each section + const metadataBySection = { + overview: [], + handbook: [], + components: [], + utils: [], + }; + + // Process files from a specific section + const processSection = async (sectionName) => { + console.log(`Processing ${sectionName} section...`); + + // Find all MDX files in this section + const sectionPath = path.join(MDX_SOURCE_DIR, sectionName); + const mdxFiles = await glob('**/*/page.mdx', { + cwd: sectionPath, + absolute: true, + }); + + console.log(`Found ${mdxFiles.length} files in ${sectionName}`); + + for (const mdxFile of mdxFiles) { + const relativePath = path.relative(MDX_SOURCE_DIR, mdxFile); + const dirPath = path.dirname(relativePath); + const urlPath = dirPath.replace(/\\/g, '/'); + const outputFilePath = path.join(OUTPUT_REACT_DIR, `${dirPath}.md`); + + const mdxContent = await fs.readFile(mdxFile, 'utf-8'); + + const { markdown, title, subtitle, description } = await mdxToMarkdown(mdxContent, mdxFile); + + // Create directories for output if needed + await fs.mkdir(path.dirname(outputFilePath), { recursive: true }); + + // Create markdown content with frontmatter + let content = [ + '---', + `title: ${title || 'Untitled'}`, + subtitle ? `subtitle: ${subtitle}` : '', + description ? `description: ${description}` : '', + '---', + '', + markdown, + ] + .filter(Boolean) + .join('\n'); + + // Format markdown with frontmatter using prettier + const prettierOptions = await prettier.resolveConfig(outputFilePath); + + content = await prettier.format(content, { + ...prettierOptions, + filepath: outputFilePath, + parser: 'markdown', + }); + + // Write formatted markdown file + await fs.writeFile(outputFilePath, content, 'utf-8'); + + // Extract the filename without extension to use as id + const fileId = path.basename(dirPath); + + // Store metadata for this file in the appropriate section + metadataBySection[sectionName].push({ + id: fileId, + title: title || 'Untitled', + subtitle: subtitle || '', + description: description || '', + urlPath: `./react/${urlPath}.md`, + }); + + console.log(`Processed: ${relativePath}`); + } + }; + + // Process each section + await processSection('overview'); + await processSection('handbook'); + await processSection('components'); + await processSection('utils'); + + // Build structured content for llms.txt + const sections = ['# Base UI', '']; + + sections.push( + 'This is the documentation for the `@base-ui-components/react` package.', + 'It contains a collection of components and utilities for building user interfaces in React.', + 'The library is designed to be composable and styling agnostic.', + '', + ); + + // Create formatted sections in specific order + const formatSection = (items, title) => { + if (items.length > 0) { + sections.push(`## ${title}`, ''); + + // Add each item as a link with description, starting with a bullet (-) + items.forEach((item) => { + sections.push(`- [${item.title}](${item.urlPath}): ${item.description}`); + }); + + sections.push(''); // Add empty line after section + } + }; + + // Define specific orders for sections + const overviewOrder = ['quick-start', 'accessibility', 'releases', 'about']; + const handbookOrder = ['styling', 'animation', 'composition']; + + // Validate that all expected overview items exist + overviewOrder.forEach((id) => { + if (!metadataBySection.overview.some((item) => item.id === id)) { + throw new Error(`Missing expected overview item: ${id}`); + } + }); + + // Validate that all expected handbook items exist + handbookOrder.forEach((id) => { + if (!metadataBySection.handbook.some((item) => item.id === id)) { + throw new Error(`Missing expected handbook item: ${id}`); + } + }); + + // Sort overview by predefined order + const sortedOverview = [...metadataBySection.overview].sort((a, b) => { + return overviewOrder.indexOf(a.id) - overviewOrder.indexOf(b.id); + }); + + // Sort handbook by predefined order + const sortedHandbook = [...metadataBySection.handbook].sort((a, b) => { + return handbookOrder.indexOf(a.id) - handbookOrder.indexOf(b.id); + }); + + // Sort components and utilities alphabetically by id + const sortedComponents = [...metadataBySection.components].sort((a, b) => + a.id.localeCompare(b.id), + ); + const sortedUtils = [...metadataBySection.utils].sort((a, b) => a.id.localeCompare(b.id)); + + // Add sections in the required order + formatSection(sortedOverview, 'Overview'); + formatSection(sortedHandbook, 'Handbook'); + formatSection(sortedComponents, 'Components'); + formatSection(sortedUtils, 'Utilities'); + + // Create llms.txt content and format with prettier + let llmsTxtContent = sections.join('\n'); + + // Apply prettier formatting using the project's configuration + const llmsFilePath = path.join(OUTPUT_BASE_DIR, 'llms.txt'); + const prettierOptions = await prettier.resolveConfig(llmsFilePath); + + llmsTxtContent = await prettier.format(llmsTxtContent, { + ...prettierOptions, + filepath: llmsFilePath, + parser: 'markdown', + }); + + await fs.writeFile(path.join(OUTPUT_BASE_DIR, 'llms.txt'), llmsTxtContent, 'utf-8'); + + // Calculate the total number of files processed + const totalFiles = + metadataBySection.overview.length + + metadataBySection.handbook.length + + metadataBySection.components.length + + metadataBySection.utils.length; + + console.log(`Successfully generated ${totalFiles} markdown files and llms.txt`); + } catch (error) { + console.error('Error generating llms.txt:', error); + process.exit(1); + } +} + +// Run the generator +generateLlmsTxt(); diff --git a/docs/scripts/generateLlmTxt/mdxNodeHelpers.mjs b/docs/scripts/generateLlmTxt/mdxNodeHelpers.mjs new file mode 100644 index 0000000000..e314f4d89b --- /dev/null +++ b/docs/scripts/generateLlmTxt/mdxNodeHelpers.mjs @@ -0,0 +1,199 @@ +/** + * mdxNodeHelpers.mjs - Helper functions for creating MDX AST nodes + * + * This module provides utility functions to create nodes for MDX/Markdown + * abstract syntax trees, making transformer code more readable and maintainable. + */ + +/** + * Create a text node + * @param {string} value - The text content + * @returns {Object} A text node + */ +export function text(value) { + return { + type: 'text', + value: value || '', + }; +} + +/** + * Helper to normalize children (handles string, node, or array) + * @param {string|Object|Array} children - Child content + * @returns {Array} Normalized array of nodes + */ +function normalizeChildren(children) { + // Handle empty or undefined + if (!children) { + return []; + } + + // Convert to array if not already + const childArray = Array.isArray(children) ? children : [children]; + + // Convert strings to text nodes + return childArray.map((child) => (typeof child === 'string' ? text(child) : child)); +} + +/** + * Create a paragraph node + * @param {string|Object|Array} children - Child node, string, or array of nodes/strings + * @returns {Object} A paragraph node + */ +export function paragraph(children) { + return { + type: 'paragraph', + children: normalizeChildren(children), + }; +} + +/** + * Create an emphasis (italic) node + * @param {string|Object|Array} children - Child node, string, or array of nodes/strings + * @returns {Object} An emphasis node + */ +export function emphasis(children) { + return { + type: 'emphasis', + children: normalizeChildren(children), + }; +} + +/** + * Create a strong (bold) node + * @param {string|Object|Array} children - Child node, string, or array of nodes/strings + * @returns {Object} A strong node + */ +export function strong(children) { + return { + type: 'strong', + children: normalizeChildren(children), + }; +} + +/** + * Create a heading node + * @param {number} depth - Heading level (1-6) + * @param {string|Object|Array} children - Child node, string, or array of nodes/strings + * @returns {Object} A heading node + */ +export function heading(depth, children) { + return { + type: 'heading', + depth: depth || 1, + children: normalizeChildren(children), + }; +} + +/** + * Create a code block node + * @param {string} value - Code content + * @param {string} lang - Language for syntax highlighting + * @returns {Object} A code node + */ +export function code(value, lang) { + return { + type: 'code', + lang: lang || null, + value: value || '', + }; +} + +/** + * Create an inline code node + * @param {string} value - Code content + * @returns {Object} An inline code node + */ +export function inlineCode(value) { + return { + type: 'inlineCode', + value: value || '', + }; +} + +/** + * Creates a table cell node + * @param {string|Object} content - Cell content + * @returns {Object} Table cell node + */ +function tableCell(content) { + return { + type: 'tableCell', + children: normalizeChildren(content), + }; +} + +/** + * Creates a table row node + * @param {Array} cells - Array of cell contents + * @returns {Object} Table row node + */ +function tableRow(cells) { + return { + type: 'tableRow', + children: cells.map((cell) => tableCell(cell)), + }; +} + +/** + * Creates a markdown table node (GFM) + * @param {Array} headers - Array of header strings or nodes + * @param {Array>} rows - Array of row data, each row is an array of cell content + * @param {Array} [alignment] - Optional array of alignments ('left', 'center', 'right') for each column + * @returns {Object} A table node + */ +export function table(headers, rows, alignment = null) { + // Convert alignment strings to AST format + const align = headers.map((_, index) => { + if (!alignment || !alignment[index]) { + return null; + } + + switch (alignment[index]) { + case 'center': + return 'center'; + case 'right': + return 'right'; + default: + return 'left'; + } + }); + + // Create header row + const headerRow = tableRow(headers); + + // Create data rows + const dataRows = rows.map((row) => tableRow(row)); + + // Return table node + return { + type: 'table', + align, + children: [headerRow, ...dataRows], + }; +} + +/** + * Function to extract all text from a node and its children recursively + * @param {Object} node - AST node + * @returns {string} Extracted text content + */ +export function textContent(node) { + if (!node) { + return ''; + } + + if (typeof node === 'string') { + return node; + } + + if (node.type === 'text') { + return node.value || ''; + } + + if (node.children && Array.isArray(node.children)) { + return node.children.map(textContent).join(''); + } + + return ''; +} diff --git a/docs/scripts/generateLlmTxt/mdxToMarkdown.mjs b/docs/scripts/generateLlmTxt/mdxToMarkdown.mjs new file mode 100644 index 0000000000..5ea9e9d444 --- /dev/null +++ b/docs/scripts/generateLlmTxt/mdxToMarkdown.mjs @@ -0,0 +1,216 @@ +/** + * mdxToMarkdown.mjs - Converts MDX content to Markdown + * + * This module transforms MDX content to Markdown format + * using remark and remark-mdx plugin. + */ + +import { unified } from 'unified'; +import remarkParse from 'remark-parse'; +import remarkMdx from 'remark-mdx'; +import remarkGfm from 'remark-gfm'; +import remarkStringify from 'remark-stringify'; +import { visit } from 'unist-util-visit'; +import { processReference } from './referenceProcessor.mjs'; +import { processDemo } from './demoProcessor.mjs'; +import { processPropsReferenceTable } from './propsReferenceTableProcessor.mjs'; +import * as mdx from './mdxNodeHelpers.mjs'; + +/** + * Plugin to extract metadata from the MDX content + */ +function extractMetadata() { + return (tree, file) => { + // Initialize metadata in file.data + file.data.metadata = { + title: '', + subtitle: '', + description: '', + }; + + // Extract title from first h1 + visit(tree, 'heading', (node) => { + if (node.depth === 1 && node.children?.[0]?.value) { + file.data.metadata.title = mdx.textContent(node); + } + }); + + // Extract from MDX components + visit(tree, ['mdxJsxFlowElement', 'mdxFlowExpression', 'mdxJsxTextElement'], (node) => { + // Extract from Subtitle component + if (node.name === 'Subtitle') { + file.data.metadata.subtitle = mdx.textContent(node); + } + // Extract from Meta component + else if (node.name === 'Meta') { + const nameAttr = node.attributes?.find( + (attr) => attr.name === 'name' && attr.value === 'description', + ); + const contentAttr = node.attributes?.find((attr) => attr.name === 'content'); + + if (nameAttr && contentAttr) { + file.data.metadata.description = contentAttr.value; + } + } + }); + + return tree; + }; +} + +/** + * Plugin to transform JSX elements to markdown or remove them from the tree + */ +function transformJsx() { + return (tree, file) => { + // Handle JSX flow elements (block-level JSX) + visit( + tree, + [ + 'mdxJsxFlowElement', + 'mdxjsEsm', + 'mdxFlowExpression', + 'mdxTextExpression', + 'mdxJsxTextElement', + ], + (node, index, parent) => { + // Process different component types + switch (node.name) { + case 'Demo': { + // Get the file path for context + const filePath = file.path || ''; + + // Process the demo component using our dedicated processor + const demoContent = processDemo(node, filePath); + + // Replace the demo component with the generated content + parent.children.splice(index, 1, ...demoContent); + return visit.CONTINUE; + } + + case 'Reference': { + // Process the reference component using our dedicated processor + const tables = processReference(node, parent, index); + + // Replace the reference component with the generated tables + parent.children.splice(index, 1, ...tables); + + return visit.CONTINUE; + } + + case 'PropsReferenceTable': { + // Process the PropsReferenceTable component using our dedicated processor + const tables = processPropsReferenceTable(node); + + // Replace the PropsReferenceTable component with the generated tables + parent.children.splice(index, 1, ...tables); + + return visit.CONTINUE; + } + + case 'Subtitle': { + parent.children.splice(index, 1); + return visit.CONTINUE; + } + + case 'Meta': { + // Check if it's a description meta tag + const nameAttr = node.attributes?.find( + (attr) => attr.name === 'name' && attr.value === 'description', + ); + const contentAttr = node.attributes?.find((attr) => attr.name === 'content'); + + if (nameAttr && contentAttr && contentAttr.value) { + // Replace with a paragraph containing the description + parent.children.splice(index, 1, mdx.paragraph(contentAttr.value)); + return visit.CONTINUE; + } + + // Remove other Meta tags + parent.children.splice(index, 1); + return [visit.SKIP, index]; + } + + case 'a': + case 'abbr': + case 'b': + case 'br': + case 'code': + case 'del': + case 'em': + case 'i': + case 'img': + case 'kbd': + case 'mark': + case 's': + case 'span': + case 'strong': + case 'sub': + case 'sup': + case 'time': { + // Support some HTML elements from GitHub flavored markdown + return visit.CONTINUE; + } + + case 'link': { + // Ignore some hidden elements + parent.children.splice(index, 1); + return [visit.SKIP, index]; + } + + default: { + throw new Error(`Unknown component: ${node.name}`); + } + } + }, + ); + + return tree; + }; +} + +/** + * Converts MDX content to markdown and extracts metadata + * @param {string} mdxContent - The MDX content to convert + * @param {string} mdxFilePath - Optional path to the MDX file for context + * @returns {Promise} An object containing the markdown and metadata + */ +export async function mdxToMarkdown(mdxContent, mdxFilePath) { + // Process the MDX content and include file path for context + const vfile = { + path: mdxFilePath, + value: mdxContent, + }; + + const file = await unified() + .use(remarkParse) + .use(remarkMdx) + .use(remarkGfm) // Add GitHub Flavored Markdown support + .use(extractMetadata) + .use(transformJsx) + .use(remarkStringify, { + bullet: '-', + emphasis: '*', + strong: '*', + fence: '`', + fences: true, + listItemIndent: 'one', + rule: '-', + commonmark: true, + gfm: true, + }) + .process(vfile); + + // Get markdown content as string + const markdown = String(file); + + // Extract metadata from the file's data + const { title = '', subtitle = '', description = '' } = file.data.metadata || {}; + + return { + markdown, + title, + subtitle, + description, + }; +} diff --git a/docs/scripts/generateLlmTxt/propsReferenceTableProcessor.mjs b/docs/scripts/generateLlmTxt/propsReferenceTableProcessor.mjs new file mode 100644 index 0000000000..c4a52e3f75 --- /dev/null +++ b/docs/scripts/generateLlmTxt/propsReferenceTableProcessor.mjs @@ -0,0 +1,174 @@ +/** + * propsReferenceTableProcessor.mjs - Process inline PropsReferenceTable components + * + * This module handles converting inline props reference data from MDX PropsReferenceTable + * components into markdown tables for documentation. + */ + +import { unified } from 'unified'; +import remarkParse from 'remark-parse'; +import * as mdx from './mdxNodeHelpers.mjs'; + +/** + * Parse a markdown string into an AST + * @param {string} markdown - Markdown string to parse + * @returns {Object} The root content node of the parsed AST + */ +function parseMarkdown(markdown) { + // Parse markdown into an AST + const processor = unified().use(remarkParse); + const result = processor.parse(markdown); + return result.children; +} + +/** + * Recursively convert an estree expression into a JavaScript object + * @param {Object} estree - The estree node to convert + * @returns {Object} The converted JavaScript object + */ +function convertEstreeToObject(estree) { + // Get the main expression from the estree program body + if (!estree || !estree.body || !estree.body[0] || !estree.body[0].expression) { + throw new Error('Invalid estree structure - missing expression'); + } + + const expression = estree.body[0].expression; + return convertExpressionNode(expression); +} + +/** + * Convert an expression node to a JavaScript value + * @param {Object} node - The expression node + * @returns {any} The converted JavaScript value + */ +function convertExpressionNode(node) { + if (!node || !node.type) { + throw new Error('Invalid expression node - missing type'); + } + + switch (node.type) { + case 'ObjectExpression': { + const obj = {}; + // Convert each property in the object + for (const prop of node.properties) { + if (prop.type !== 'Property') { + throw new Error(`Unsupported property type: ${prop.type}`); + } + + // Get the property key + let key; + if (prop.key.type === 'Identifier') { + key = prop.key.name; + } else if (prop.key.type === 'Literal') { + key = prop.key.value; + } else { + throw new Error(`Unsupported key type: ${prop.key.type}`); + } + + // Get the property value + const value = convertExpressionNode(prop.value); + + // Add to the object + obj[key] = value; + } + return obj; + } + case 'ArrayExpression': + // Convert each element in the array + return node.elements.map((element) => convertExpressionNode(element)); + + case 'Literal': + // Return literals directly + return node.value; + + case 'TemplateLiteral': + // For simple template literals with no expressions + if (node.quasis.length === 1 && node.expressions.length === 0) { + return node.quasis[0].value.raw; + } + // For complex template literals, return a simplified representation + return node.quasis.map((q) => q.value.raw).join('...'); + + case 'Identifier': + // For identifiers like undefined, null, etc. + return node.name; + + default: + throw new Error(`Unsupported expression type: ${node.type}`); + } +} + +/** + * Transforms a PropsReferenceTable component into a markdown table + * @param {Object} node - The PropsReferenceTable JSX node from MDX + * @returns {Array} Array of markdown nodes to replace the PropsReferenceTable component + */ +export function processPropsReferenceTable(node) { + // Extract the data attribute which contains props definitions + const dataAttr = node.attributes?.find((attr) => attr.name === 'data'); + const typeAttr = node.attributes?.find((attr) => attr.name === 'type')?.value || 'props'; + + // If no data attribute is found, throw an error + if (!dataAttr) { + throw new Error('PropsReferenceTable: No data provided'); + } + + // Process the data object from the AST + let propsData = {}; + + if (dataAttr.type === 'mdxJsxAttribute' && dataAttr.value) { + try { + if ( + dataAttr.value.type === 'mdxJsxAttributeValueExpression' && + dataAttr.value.data && + dataAttr.value.data.estree + ) { + // Convert the estree to a JavaScript object + propsData = convertEstreeToObject(dataAttr.value.data.estree); + } else { + throw new Error('PropsReferenceTable data must be a static JavaScript object'); + } + } catch (err) { + throw new Error(`Error processing PropsReferenceTable data: ${err.message}`); + } + } else { + throw new Error('PropsReferenceTable data attribute must be a valid JSX attribute'); + } + + // Generate markdown tables + const tables = []; + + // Add heading based on the type + const heading = typeAttr === 'return' ? 'Return Value' : 'Props'; + tables.push(mdx.paragraph([mdx.strong(`${heading}:`)])); + + // Convert props data to table rows + const propsRows = Object.entries(propsData).map(([propName, propDef]) => { + const row = [propName, propDef.type ? mdx.inlineCode(propDef.type) : '-']; + + // Add default column for props type + if (typeAttr === 'props') { + row.push(propDef.default ? mdx.inlineCode(propDef.default) : '-'); + } + + // Add description + row.push(parseMarkdown(propDef.description || '-')); + + return row; + }); + + // Define columns based on type + const headers = + typeAttr === 'props' + ? ['Prop', 'Type', 'Default', 'Description'] + : ['Property', 'Type', 'Description']; + + // Define column alignments + const alignments = + typeAttr === 'props' ? ['left', 'left', 'left', 'left'] : ['left', 'left', 'left']; + + const tableNode = mdx.table(headers, propsRows, alignments); + tables.push(tableNode); + + return tables; +} diff --git a/docs/scripts/generateLlmTxt/referenceProcessor.mjs b/docs/scripts/generateLlmTxt/referenceProcessor.mjs new file mode 100644 index 0000000000..e22a046074 --- /dev/null +++ b/docs/scripts/generateLlmTxt/referenceProcessor.mjs @@ -0,0 +1,158 @@ +/** + * referenceProcessor.mjs - Process component reference definitions + * + * This module handles loading and converting component reference data + * from JSON files into markdown tables for documentation. + */ + +import fs from 'fs'; +import path from 'path'; +import { unified } from 'unified'; +import remarkParse from 'remark-parse'; + +import * as mdx from './mdxNodeHelpers.mjs'; + +/** + * Parse a markdown string into an AST + * @param {string} markdown - Markdown string to parse + * @returns {Object} The root content node of the parsed AST + */ +function parseMarkdown(markdown) { + // Parse markdown into an AST + const processor = unified().use(remarkParse); + const result = processor.parse(markdown); + return result.children; +} + +/** + * Transforms a Reference component into markdown tables + * @param {Object} node - The Reference JSX node from MDX + * @param {Array} ancestors - The ancestry chain of the node + * @returns {Array} Array of markdown nodes to replace the Reference component + */ +export function processReference(node) { + // Extract component name and parts from attributes + const componentAttr = node.attributes?.find((attr) => attr.name === 'component')?.value; + const partsAttr = node.attributes?.find((attr) => attr.name === 'parts')?.value; + + if (!componentAttr) { + throw new Error('Missing "component" prop on the "" component.'); + } + + const tables = []; + + // Process each component part + const parts = partsAttr ? partsAttr.split(/,\s*/).map((p) => p.trim()) : [componentAttr]; + + // Load component definitions from JSON files + const componentDefs = []; + const kebabCase = (str) => str.replace(/([a-z])([A-Z])/g, '$1-$2').toLowerCase(); + const projectRoot = path.resolve(import.meta.dirname, '../..'); + + for (const part of parts) { + // Construct file path for this component part + let filename = `${kebabCase(componentAttr)}-${kebabCase(part)}.json`; + let filepath = path.join(projectRoot, 'reference/generated', filename); + + // If file doesn't exist, try with just the part name + if (!fs.existsSync(filepath)) { + filename = `${kebabCase(part)}.json`; + filepath = path.join(projectRoot, 'reference/generated', filename); + } + + // Read and parse JSON file + if (!fs.existsSync(filepath)) { + throw new Error(`Reference file not found for component ${componentAttr}, part ${part}`); + } + + const jsonContent = fs.readFileSync(filepath, 'utf-8'); + const componentDef = JSON.parse(jsonContent); + componentDefs.push(componentDef); + } + + // Generate markdown tables for each component + componentDefs.forEach((def, idx) => { + const part = parts[idx]; + + // Add subheading for the part + if (parts.length > 1) { + tables.push(mdx.heading(3, part)); + } + + // Add description if available + if (def.description) { + // Parse the description as markdown + const descriptionNode = parseMarkdown(def.description); + tables.push(mdx.paragraph(descriptionNode)); + } + + // Props table + if (Object.keys(def.props || {}).length > 0) { + // Create a proper heading with strong node + tables.push(mdx.paragraph([mdx.strong(`${part} Props:`)])); + + const propsRows = Object.entries(def.props).map(([propName, propDef]) => [ + propName, + propDef.type ? mdx.inlineCode(propDef.type) : '-', + propDef.default ? mdx.inlineCode(propDef.default) : '-', + parseMarkdown(propDef.description || '-'), + ]); + + // Define column alignments: prop name left-aligned, others left-aligned + const alignments = ['left', 'left', 'left', 'left']; + + const tableNode = mdx.table( + ['Prop', 'Type', 'Default', 'Description'], + propsRows, + alignments, + ); + tables.push(tableNode); + } + + // Data attributes table + if (Object.keys(def.dataAttributes || {}).length > 0) { + tables.push(mdx.paragraph([mdx.strong(`${part} Data Attributes:`)])); + + const attrRows = Object.entries(def.dataAttributes).map(([attrName, attrDef]) => [ + attrName, + attrDef.type ? mdx.inlineCode(attrDef.type) : '-', + parseMarkdown(attrDef.description || '-'), + ]); + + // Define column alignments + const alignments = ['left', 'left', 'left']; + + const tableNode = mdx.table(['Attribute', 'Type', 'Description'], attrRows, alignments); + tables.push(tableNode); + } + + // CSS variables table + if (Object.keys(def.cssVariables || {}).length > 0) { + tables.push(mdx.paragraph([mdx.strong(`${part} CSS Variables:`)])); + + const cssRows = Object.entries(def.cssVariables).map(([varName, varDef]) => [ + varName, + varDef.type ? mdx.inlineCode(varDef.type) : '-', + varDef.default ? mdx.inlineCode(varDef.default) : '-', + parseMarkdown(varDef.description || '-'), + ]); + + // Define column alignments + const alignments = ['left', 'left', 'left', 'left']; + + const tableNode = mdx.table( + ['Variable', 'Type', 'Default', 'Description'], + cssRows, + alignments, + ); + tables.push(tableNode); + } + + // Add separator between parts + if (parts.length > 1 && idx < parts.length - 1) { + tables.push(mdx.paragraph('')); + } + }); + + return tables; +} diff --git a/docs/src/components/MarkdownLink.tsx b/docs/src/components/MarkdownLink.tsx new file mode 100644 index 0000000000..39b16686a3 --- /dev/null +++ b/docs/src/components/MarkdownLink.tsx @@ -0,0 +1,19 @@ +'use client'; + +import * as React from 'react'; +import { usePathname } from 'next/navigation'; +import { MarkdownIcon } from '../icons/MarkdownIcon'; + +export function MarkdownLink() { + const pathname = usePathname(); + + return ( + + + + ); +} diff --git a/docs/src/icons/MarkdownIcon.tsx b/docs/src/icons/MarkdownIcon.tsx new file mode 100644 index 0000000000..325a6f6998 --- /dev/null +++ b/docs/src/icons/MarkdownIcon.tsx @@ -0,0 +1,21 @@ +import * as React from 'react'; + +export function MarkdownIcon(props: React.SVGProps) { + return ( + + ); +} diff --git a/docs/src/mdx-components.tsx b/docs/src/mdx-components.tsx index 33653dbe9c..2234b20dc8 100644 --- a/docs/src/mdx-components.tsx +++ b/docs/src/mdx-components.tsx @@ -12,6 +12,7 @@ import { Link } from './components/Link'; import { HeadingLink } from './components/HeadingLink'; import { Subtitle } from './components/Subtitle/Subtitle'; import { Kbd } from './components/Kbd/Kbd'; +import { MarkdownLink } from './components/MarkdownLink'; interface MDXComponents { [key: string]: React.FC | MDXComponents; @@ -24,7 +25,10 @@ export const mdxComponents: MDXComponents = { h1: (props) => ( // Do not wrap heading tags in divs, that confuses Safari Reader -

+
+

+ +

{`${getChildrenText(props.children)} ยท Base UI`} ), diff --git a/netlify.toml b/netlify.toml index 0e8efabdd4..f66c120f50 100644 --- a/netlify.toml +++ b/netlify.toml @@ -6,7 +6,7 @@ command = "pnpm docs:build" [build.environment] - NODE_VERSION = "18" + NODE_VERSION = "20" PNPM_FLAGS = "--frozen-lockfile" [[plugins]] diff --git a/package.json b/package.json index 758ce8610e..4c461d5262 100644 --- a/package.json +++ b/package.json @@ -14,12 +14,13 @@ "release:publish:dry-run": "pnpm publish --recursive --tag latest --registry=\"http://localhost:4873/\"", "release:tag": "node scripts/releaseTag.mjs --repo base-ui", "docs:api": "pnpm --filter api-docs-builder start", - "docs:build": "pnpm --filter docs build", + "docs:build": "pnpm --filter docs generate-llms && pnpm --filter docs build", "docs:deploy": "pnpm --filter docs run deploy", "docs:dev": "pnpm --filter docs dev", "docs:size-why": "cross-env DOCS_STATS_ENABLED=true pnpm docs:build", "docs:start": "pnpm --filter docs serve", "docs:link-check": "pnpm --filter docs link-check", + "docs:generate-llms": "pnpm --filter docs run generate-llms", "extract-error-codes": "cross-env MUI_EXTRACT_ERROR_CODES=true lerna run --concurrency 1 build:stable", "install:codesandbox": "pnpm install --no-frozen-lockfile", "jsonlint": "node ./scripts/jsonlint.mjs", @@ -145,6 +146,7 @@ "react-dom": "^19.1.0", "recast": "^0.23.11", "remark": "^15.0.1", + "remark-mdx": "^3.1.0", "rimraf": "^6.0.1", "serve": "^14.2.4", "style-loader": "^4.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b86d249618..5d8f4a39b0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -274,6 +274,9 @@ importers: remark: specifier: ^15.0.1 version: 15.0.1 + remark-mdx: + specifier: ^3.1.0 + version: 3.1.0 rimraf: specifier: ^6.0.1 version: 6.0.1 @@ -346,9 +349,6 @@ importers: '@mdx-js/loader': specifier: ^3.1.0 version: 3.1.0(acorn@8.14.1)(webpack@5.99.9(webpack-cli@6.0.1(webpack-bundle-analyzer@4.10.2)(webpack@5.99.9))) - '@mdx-js/mdx': - specifier: ^3.1.0 - version: 3.1.0(acorn@8.14.1) '@mdx-js/react': specifier: ^3.1.0 version: 3.1.0(@types/react@19.1.6)(react@19.1.0) @@ -457,9 +457,6 @@ importers: to-vfile: specifier: ^8.0.0 version: 8.0.0 - unist-util-visit: - specifier: ^5.0.0 - version: 5.0.0 unist-util-visit-parents: specifier: ^6.0.1 version: 6.0.1 @@ -467,6 +464,9 @@ importers: specifier: ^5.0.1 version: 5.0.1 devDependencies: + '@mdx-js/mdx': + specifier: ^3.1.0 + version: 3.1.0(acorn@8.14.1) '@mui/internal-docs-utils': specifier: ^2.0.1 version: 2.0.1 @@ -515,6 +515,12 @@ importers: prettier: specifier: ^3.5.3 version: 3.5.3 + react-reconciler: + specifier: ^0.32.0 + version: 0.32.0(react@19.1.0) + remark-stringify: + specifier: ^11.0.0 + version: 11.0.0 rimraf: specifier: ^6.0.1 version: 6.0.1 @@ -527,6 +533,9 @@ importers: unified: specifier: ^11.0.5 version: 11.0.5 + unist-util-visit: + specifier: ^5.0.0 + version: 5.0.0 webpack-bundle-analyzer: specifier: ^4.10.2 version: 4.10.2 @@ -9123,6 +9132,12 @@ packages: react-is@19.1.0: resolution: {integrity: sha512-Oe56aUPnkHyyDxxkvqtd7KkdQP5uIUfHxd5XTb3wE9d/kRnZLmKbDB0GWk919tdQ+mxxPtG6EAs6RMT6i1qtHg==} + react-reconciler@0.32.0: + resolution: {integrity: sha512-2NPMOzgTlG0ZWdIf3qG+dcbLSoAc/uLfOwckc3ofy5sSK0pLJqnQLpUFxvGcN2rlXSjnVtGeeFLNimCQEj5gOQ==} + engines: {node: '>=0.10.0'} + peerDependencies: + react: ^19.1.0 + react-refresh@0.17.0: resolution: {integrity: sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ==} engines: {node: '>=0.10.0'} @@ -21160,6 +21175,11 @@ snapshots: react-is@19.1.0: {} + react-reconciler@0.32.0(react@19.1.0): + dependencies: + react: 19.1.0 + scheduler: 0.26.0 + react-refresh@0.17.0: {} react-remove-scroll-bar@2.3.8(@types/react@19.1.6)(react@19.1.0):