Generate llms.txt file

Author: JoviDeCroock

plugins/generate-llms-txt.js

@@ -0,0 +1,129 @@
+import fs from 'fs/promises';
+import fsSync from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+/**
+ * Extract title and description from frontmatter
+ * @param {string} content
+ * @returns {{title: string, description: string, body: string}}
+ */
+function parseFrontmatter(content) {
+	const frontmatterRegex = /^---\n([\s\S]*?)\n---\n([\s\S]*)$/;
+	const match = content.match(frontmatterRegex);
+
+	if (!match) {
+		return { title: '', description: '', body: content };
+	}
+
+	const frontmatter = match[1];
+	const body = match[2];
+
+	const titleMatch = frontmatter.match(/^title:\s*(.+)$/m);
+	const descriptionMatch = frontmatter.match(/^description:\s*(.+)$/m);
+
+	return {
+		title: titleMatch ? titleMatch[1].trim() : '',
+		description: descriptionMatch ? descriptionMatch[1].trim() : '',
+		body: body.trim()
+	};
+}
+
+/**
+ * Read all markdown files from the guide directory
+ * @param {string} guideDir
+ * @returns {Promise<Array<{filename: string, content: string}>>}
+ */
+async function readMarkdownFiles(guideDir) {
+	const files = await fs.readdir(guideDir);
+	const markdownFiles = files.filter(file => file.endsWith('.md'));
+
+	return Promise.all(
+		markdownFiles.map(async filename => {
+			const filePath = path.join(guideDir, filename);
+			const content = await fs.readFile(filePath, 'utf-8');
+			return { filename, content };
+		})
+	);
+}
+
+/**
+ * Generate the llms.txt content
+ * @param {Array<{filename: string, content: string}>} files
+ * @returns {string}
+ */
+function generateLlmsTxt(files) {
+	const header = `# Preact Documentation
+
+This file contains comprehensive documentation for Preact v10, a fast 3kB alternative to React with the same modern API.
+
+Generated on: ${new Date().toISOString()}
+Source: https://github.com/preactjs/preact-www
+
+## Overview
+
+Preact is a fast, lightweight alternative to React that provides the same modern API in a much smaller package. This documentation covers all aspects of Preact v10, including components, hooks, server-side rendering, TypeScript support, and more.
+
+---
+
+`;
+
+	let content = header;
+
+	files.sort((a, b) => a.filename.localeCompare(b.filename));
+
+	files.forEach(({ filename, content: fileContent }) => {
+		const { title, description, body } = parseFrontmatter(fileContent);
+
+		content += `## ${title || filename.replace('.md', '')}\n\n`;
+
+		if (description) {
+			content += `**Description:** ${description}\n\n`;
+		}
+
+		content += `**Source:** content/en/guide/v10/${filename}\n\n`;
+		content += `${body}\n\n`;
+		content += `---\n\n`;
+	});
+
+	return content;
+}
+
+/**
+ * Vite plugin to generate llms.txt from Preact documentation
+ * @param {Object} options
+ * @param {string} [options.guideDir] - Path to the guide directory
+ * @param {string} [options.outputFile] - Path to the output file
+ * @returns {import('vite').Plugin}
+ */
+export default function generateLlmsTxtPlugin(options = {}) {
+	const guideDir =
+		options.guideDir ||
+		path.join(__dirname, '..', 'content', 'en', 'guide', 'v10');
+
+	return {
+		name: 'generate-llms-txt',
+		async buildStart() {
+			try {
+				if (!fsSync.existsSync(guideDir)) {
+					this.warn(`Guide directory not found: ${guideDir}`);
+					return;
+				}
+
+				const files = await readMarkdownFiles(guideDir);
+				const llmsTxtContent = generateLlmsTxt(files);
+
+				this.emitFile({
+					type: 'asset',
+					fileName: 'llms.txt',
+					source: llmsTxtContent
+				});
+			} catch (error) {
+				this.error(`Error generating llms.txt: ${error.message}`);
+			}
+		}
+	};
+}

vite.config.js

@@ -8,6 +8,7 @@ import { netlifyPlugin } from './plugins/netlify.js';
 import { spaFallbackMiddlewarePlugin } from './plugins/spa-fallback-middleware.js';
 import { htmlRoutingMiddlewarePlugin } from './plugins/html-routing-middleware.js';
 import { rssFeedPlugin } from './plugins/rss-feed.js';
+import generateLlmsTxtPlugin from './plugins/generate-llms-txt.js';
 
 // TODO: Should we do this for all routes, rely on discovery a bit less?
 import { tutorialRoutes } from './src/lib/route-utils.js';
@@ -44,7 +45,8 @@ export default defineConfig({
 				{
 					src: './content/**/*.md',
 					dest: './',
-					rename: (_name, _fileExtension, fullPath) => path.basename(fullPath).replace(/\.md$/, '.json'),
+					rename: (_name, _fileExtension, fullPath) =>
+						path.basename(fullPath).replace(/\.md$/, '.json'),
 					transform: precompileMarkdown
 				}
 			],
@@ -74,6 +76,7 @@ export default defineConfig({
 		netlifyPlugin(),
 		spaFallbackMiddlewarePlugin(),
 		htmlRoutingMiddlewarePlugin(),
-		rssFeedPlugin()
+		rssFeedPlugin(),
+		generateLlmsTxtPlugin()
 	]
 });