Generate llms.txt file (#1304)

* Generate llms.txt file

* Share frontmatter parsing

Author: JoviDeCroock

plugins/generate-llms-txt.js

@@ -0,0 +1,106 @@
+import fs from 'fs/promises';
+import fsSync from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { parseFrontmatter, cleanReplComments } from '../src/lib/frontmatter.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+/**
+ * Read all markdown files from the guide directory
+ * @param {string} guideDir
+ * @returns {Promise<Array<{filename: string, content: string}>>}
+ */
+async function readMarkdownFiles(guideDir) {
+	const files = await fs.readdir(guideDir);
+	const markdownFiles = files.filter(file => file.endsWith('.md'));
+
+	return Promise.all(
+		markdownFiles.map(async filename => {
+			const filePath = path.join(guideDir, filename);
+			const content = await fs.readFile(filePath, 'utf-8');
+			return { filename, content };
+		})
+	);
+}
+
+/**
+ * Generate the llms.txt content
+ * @param {Array<{filename: string, content: string}>} files
+ * @returns {string}
+ */
+function generateLlmsTxt(files) {
+	const header = `# Preact Documentation
+
+This file contains comprehensive documentation for Preact v10, a fast 3kB alternative to React with the same modern API.
+Preact is a fast, lightweight alternative to React that provides the same modern API in a much smaller package. This documentation covers all aspects of Preact v10, including components, hooks, server-side rendering, TypeScript support, and more.
+
+---
+
+`;
+
+	let content = header;
+
+	files.sort((a, b) => a.filename.localeCompare(b.filename));
+
+	files.forEach(({ filename, content: fileContent }) => {
+		const { description, body } = parseFrontmatter(fileContent, filename);
+		let cleanedBody = cleanReplComments(body);
+
+		// Remove <toc></toc> tags
+		cleanedBody = cleanedBody.replace(/<toc><\/toc>/g, '');
+
+		// Clean up multiple consecutive newlines and empty lines around separators
+		cleanedBody = cleanedBody.replace(/\n{3,}/g, '\n\n');
+		cleanedBody = cleanedBody.replace(/---\s*\n\s*\n\s*---/g, '');
+
+		// Fix heading hierarchy: convert # to ## for consistency
+		cleanedBody = cleanedBody.replace(/^# /gm, '## ');
+
+		if (description) {
+			content += `**Description:** ${description}\n\n`;
+		}
+
+		content += `${cleanedBody}\n\n`;
+		content += `---\n\n`;
+	});
+
+	return content;
+}
+
+/**
+ * Vite plugin to generate llms.txt from Preact documentation
+ * @param {Object} options
+ * @param {string} [options.guideDir] - Path to the guide directory
+ * @param {string} [options.outputFile] - Path to the output file
+ * @returns {import('vite').Plugin}
+ */
+export default function generateLlmsTxtPlugin(options = {}) {
+	const guideDir =
+		options.guideDir ||
+		path.join(__dirname, '..', 'content', 'en', 'guide', 'v10');
+
+	return {
+		name: 'generate-llms-txt',
+		async buildStart() {
+			try {
+				if (!fsSync.existsSync(guideDir)) {
+					this.warn(`Guide directory not found: ${guideDir}`);
+					return;
+				}
+
+				const files = await readMarkdownFiles(guideDir);
+				const llmsTxtContent = generateLlmsTxt(files);
+
+				this.emitFile({
+					type: 'asset',
+					fileName: 'llms.txt',
+					source: llmsTxtContent
+				});
+			} catch (error) {
+				this.error(`Error generating llms.txt: ${error.message}`);
+			}
+		}
+	};
+}

plugins/precompile-markdown/index.js

@@ -1,10 +1,10 @@
-import yaml from 'yaml';
 import { marked } from 'marked';
 import Prism from 'prismjs';
 import loadLanguages from 'prismjs/components/';
 import { parse } from 'node-html-parser';
 import { replace } from './gh-emoji/index.js';
 import { textToBase64 } from '../../src/components/controllers/repl/query-encode.js';
+import { parseFrontmatter } from '../../src/lib/frontmatter.js';
 
 // Prism will always load `markup`, `css`, `clike` and `javascript` by default.
 // Any additional languages we need should be loaded here
@@ -38,41 +38,23 @@ export async function precompileMarkdown(content, path) {
  * @param {string} path
  */
 function parseContent(content, path) {
-	/** @type {import('../../src/types.d.ts').ContentMetaData} */
-	let meta = {};
-
-	// Find YAML FrontMatter preceeding a markdown document
-	const FRONT_MATTER_REG = /^\s*---\n\s*([\s\S]*?)\s*\n---\n/i;
-
-	const matches = content.match(FRONT_MATTER_REG);
-	if (!matches) throw new Error(`Missing YAML FrontMatter in ${path}`);
-	try {
-		meta = yaml.parse('---\n' + matches[1].replace(/^/gm, '  ') + '\n');
-		if (!meta.title) {
-			throw new Error(`Missing title in YAML FrontMatter for ${path}`);
-		}
-		if (!meta.description) {
-			//console.warn(`Missing description in FrontMatter for ${path}`);
-		}
-	} catch (e) {
-		throw new Error(`Error parsing YAML FrontMatter in ${path}`);
-	}
+	const { body, meta } = parseFrontmatter(content, path);
 
-	content = content.replace(FRONT_MATTER_REG, '');
+	let processedContent = body;
 
 	if (path.includes('/guide/')) {
-		meta.toc = generateToc(content);
+		meta.toc = generateToc(processedContent);
 	}
 
 	// extract tutorial setup, initial and final code blocks
 	if (/tutorial\/\d/.test(path)) {
-		const { markdown, tutorial } = extractTutorialCodeBlocks(content);
-		content = markdown;
+		const { markdown, tutorial } = extractTutorialCodeBlocks(processedContent);
+		processedContent = markdown;
 		meta.tutorial = tutorial;
 	}
 
 	return {
-		content,
+		content: processedContent,
 		meta
 	};
 }
@@ -121,10 +103,12 @@ marked.use({
 
 			Prism.languages[lang] == null
 				? console.warn(`No Prism highlighter for language: ${lang}`)
-				: text = Prism.highlight(code, Prism.languages[lang], lang);
+				: (text = Prism.highlight(code, Prism.languages[lang], lang));
 
 			const runInReplLink = runInRepl
-				? `<a class="repl-link" href="/repl?code=${encodeURIComponent(textToBase64(source))}">Run in REPL</a>`
+				? `<a class="repl-link" href="/repl?code=${encodeURIComponent(
+						textToBase64(source)
+				  )}">Run in REPL</a>`
 				: '';
 
 			return `
@@ -227,7 +211,9 @@ function highlightCodeBlocks(data) {
 	const doc = parse(data.html, { blockTextElements: { code: true } });
 
 	// Only get the pre blocks that haven't already been highlighted
-	const codeBlocks = doc.querySelectorAll('pre:not([class="highlight"]):has(> code[class])');
+	const codeBlocks = doc.querySelectorAll(
+		'pre:not([class="highlight"]):has(> code[class])'
+	);
 	for (const block of codeBlocks) {
 		const child = block.childNodes[0];
 
@@ -243,24 +229,31 @@ function highlightCodeBlocks(data) {
 		 * and switch it back to `\n` for the code content after marked is through with it.
 		 * We only do this on the home/index page at the moment.
 		 */
-		const rawCodeBlockText = unescapeHTML(child.innerText.trim().replace('<br>', '\n'));
+		const rawCodeBlockText = unescapeHTML(
+			child.innerText.trim().replace('<br>', '\n')
+		);
 		const [code, source, runInRepl] = processRepl(rawCodeBlockText);
 
 		const lang = child.getAttribute('class').replace('language-', '');
 
 		Prism.languages[lang] == null
 			? console.warn(`No Prism highlighter for language: ${lang}`)
-			: child.innerHTML = Prism.highlight(code, Prism.languages[lang], lang);
+			: (child.innerHTML = Prism.highlight(code, Prism.languages[lang], lang));
 
-		block.insertAdjacentHTML('beforebegin', '<div class="highlight-container">');
+		block.insertAdjacentHTML(
+			'beforebegin',
+			'<div class="highlight-container">'
+		);
 		const container = block.previousSibling;
 		container.appendChild(block);
 		block.setAttribute('class', 'highlight');
 
 		if (runInRepl) {
 			block.insertAdjacentHTML(
 				'afterend',
-				`<a class="repl-link" href="/repl?code=${encodeURIComponent(textToBase64(source))}">
+				`<a class="repl-link" href="/repl?code=${encodeURIComponent(
+					textToBase64(source)
+				)}">
 					Run in REPL
 				</a>`
 			);
@@ -271,7 +264,6 @@ function highlightCodeBlocks(data) {
 	return data;
 }
 
-
 /**
  * Marked escapes HTML entities, which is normally great,
  * but we want to feed the raw code into Prism for highlighting.
@@ -280,12 +272,12 @@ function highlightCodeBlocks(data) {
  * @returns {string}
  */
 function unescapeHTML(str) {
-    return str
-        .replace(/&amp;/g, '&')
-        .replace(/&lt;/g, '<')
-        .replace(/&gt;/g, '>')
-        .replace(/&quot;/g, '"')
-        .replace(/&#39;/g, "'");
+	return str
+		.replace(/&amp;/g, '&')
+		.replace(/&lt;/g, '<')
+		.replace(/&gt;/g, '>')
+		.replace(/&quot;/g, '"')
+		.replace(/&#39;/g, "'");
 }
 
 /**

src/lib/frontmatter.js

@@ -0,0 +1,52 @@
+import yaml from 'yaml';
+
+/**
+ * Parse Markdown with YAML FrontMatter
+ * @param {string} content - Raw markdown content with frontmatter
+ * @param {string} path - File path for error reporting
+ * @returns {{title: string, description: string, body: string, meta: object}}
+ */
+export function parseFrontmatter(content, path = '') {
+	const FRONT_MATTER_REG = /^\s*---\n\s*([\s\S]*?)\s*\n---\n/i;
+
+	const matches = content.match(FRONT_MATTER_REG);
+	if (!matches) {
+		throw new Error(`Missing YAML FrontMatter in ${path}`);
+	}
+
+	let meta = {};
+	try {
+		meta = yaml.parse('---\n' + matches[1].replace(/^/gm, '  ') + '\n');
+
+		if (!meta.title) {
+			throw new Error(`Missing title in YAML FrontMatter for ${path}`);
+		}
+	} catch (e) {
+		throw new Error(`Error parsing YAML FrontMatter in ${path}: ${e.message}`);
+	}
+
+	const body = content.replace(FRONT_MATTER_REG, '').trim();
+
+	return {
+		title: meta.title || '',
+		description: meta.description || '',
+		body,
+		meta
+	};
+}
+
+/**
+ * Process and clean code blocks for LLM consumption
+ * Removes --repl comments and markers that are used for REPL functionality
+ * @param {string} content - Markdown content
+ * @returns {string} - Cleaned content
+ */
+export function cleanReplComments(content) {
+	// Remove --repl comment lines and markers
+	return content
+		.replace(/^\/\/ --repl.*$/gm, '')
+		.replace(/^\/\/ --repl-before.*$/gm, '')
+		.replace(/^\/\/ --repl-after.*$/gm, '')
+		.replace(/\n\n\n+/g, '\n\n') // Clean up extra newlines
+		.trim();
+}

vite.config.js

@@ -8,6 +8,7 @@ import { netlifyPlugin } from './plugins/netlify.js';
 import { spaFallbackMiddlewarePlugin } from './plugins/spa-fallback-middleware.js';
 import { htmlRoutingMiddlewarePlugin } from './plugins/html-routing-middleware.js';
 import { rssFeedPlugin } from './plugins/rss-feed.js';
+import generateLlmsTxtPlugin from './plugins/generate-llms-txt.js';
 
 // TODO: Should we do this for all routes, rely on discovery a bit less?
 import { tutorialRoutes } from './src/lib/route-utils.js';
@@ -44,7 +45,8 @@ export default defineConfig({
 				{
 					src: './content/**/*.md',
 					dest: './',
-					rename: (_name, _fileExtension, fullPath) => path.basename(fullPath).replace(/\.md$/, '.json'),
+					rename: (_name, _fileExtension, fullPath) =>
+						path.basename(fullPath).replace(/\.md$/, '.json'),
 					transform: precompileMarkdown
 				}
 			],
@@ -74,6 +76,7 @@ export default defineConfig({
 		netlifyPlugin(),
 		spaFallbackMiddlewarePlugin(),
 		htmlRoutingMiddlewarePlugin(),
-		rssFeedPlugin()
+		rssFeedPlugin(),
+		generateLlmsTxtPlugin()
 	]
 });