|
| 1 | +const fs = require('fs').promises; |
| 2 | +const path = require('path'); |
| 3 | +const matter = require('gray-matter'); |
| 4 | +const globby = require('globby'); |
| 5 | + |
| 6 | +const CONTENT_DIR = path.join(__dirname, '../src/content'); |
| 7 | +const OUTPUT_DIR = path.join(__dirname, '../out'); |
| 8 | + |
| 9 | +const PAGES_DIR = path.join(__dirname, '../src/pages'); |
| 10 | + |
| 11 | +async function generate() { |
| 12 | + // 1. Process Markdown files |
| 13 | + const patterns = ['**/*.en.md', '**/*.en.mdx']; |
| 14 | + const files = await globby(patterns, {cwd: CONTENT_DIR}); |
| 15 | + |
| 16 | + let llmsFull = ''; |
| 17 | + let llms = '# Documentation Index\n\n'; |
| 18 | + const seenUrls = new Set(); |
| 19 | + |
| 20 | + // Sort files for consistent order |
| 21 | + files.sort(); |
| 22 | + |
| 23 | + for (const file of files) { |
| 24 | + const filePath = path.join(CONTENT_DIR, file); |
| 25 | + const content = await fs.readFile(filePath, 'utf8'); |
| 26 | + const {data, content: body} = matter(content); |
| 27 | + |
| 28 | + const title = data.title || file; |
| 29 | + const description = data.description || ''; |
| 30 | + |
| 31 | + // Clean URL path |
| 32 | + // Remove extension (.en.md or .en.mdx) |
| 33 | + let urlPath = file.replace(/\.en\.mdx?$/, ''); |
| 34 | + // Remove "index" at the end (e.g. foo/index -> foo, or index -> /) |
| 35 | + if (urlPath === 'index' || urlPath.endsWith('/index')) { |
| 36 | + urlPath = urlPath.replace(/\/index$/, '').replace(/^index$/, ''); |
| 37 | + } |
| 38 | + // Ensure root slash if empty (though usually we want relative or absolute paths, let's keep it clean) |
| 39 | + if (!urlPath.startsWith('/')) urlPath = '/' + urlPath; |
| 40 | + |
| 41 | + seenUrls.add(urlPath); |
| 42 | + |
| 43 | + llms += `- [${title}](${urlPath}) ${ |
| 44 | + description ? '- ' + description : '' |
| 45 | + }\n`; |
| 46 | + |
| 47 | + // Append to llms-full.txt |
| 48 | + llmsFull += `\n\n# ${title}\n\n`; |
| 49 | + llmsFull += `> File: ${file}\n\n`; |
| 50 | + llmsFull += body; |
| 51 | + llmsFull += '\n\n---\n'; |
| 52 | + } |
| 53 | + |
| 54 | + // 2. Process Page files |
| 55 | + // Scan for pages, ignoring api, _*, and dynamic routes with brackets [ ] |
| 56 | + const pageFiles = await globby(['**/*.tsx'], { |
| 57 | + cwd: PAGES_DIR, |
| 58 | + ignore: ['_*', 'api/**', '**/*\\[*'], |
| 59 | + }); |
| 60 | + pageFiles.sort(); |
| 61 | + |
| 62 | + for (const file of pageFiles) { |
| 63 | + const filePath = path.join(PAGES_DIR, file); |
| 64 | + const content = await fs.readFile(filePath, 'utf8'); |
| 65 | + |
| 66 | + // Try to extract title from meta={{title: '...'}} |
| 67 | + const titleMatch = content.match( |
| 68 | + /meta=\{\{.*?title:\s*['"]([^'"]+)['"].*?\}\}/s |
| 69 | + ); |
| 70 | + let title = titleMatch |
| 71 | + ? titleMatch[1] |
| 72 | + : path.basename(file, path.extname(file)); |
| 73 | + // Capitalize if fallback |
| 74 | + if (!titleMatch && title !== 'index') { |
| 75 | + title = title.charAt(0).toUpperCase() + title.slice(1); |
| 76 | + } else if (title === 'index') { |
| 77 | + // For index files, try to use dirname as title if no meta |
| 78 | + const dir = path.dirname(file); |
| 79 | + if (dir !== '.') { |
| 80 | + title = dir.charAt(0).toUpperCase() + dir.slice(1); |
| 81 | + } else { |
| 82 | + title = 'Home'; |
| 83 | + } |
| 84 | + } |
| 85 | + |
| 86 | + let urlPath = file.replace(/\.tsx$/, ''); |
| 87 | + // Remove "index" at the end (e.g. foo/index -> foo) |
| 88 | + if (urlPath === 'index' || urlPath.endsWith('/index')) { |
| 89 | + urlPath = urlPath.replace(/\/index$/, '').replace(/^index$/, ''); |
| 90 | + } |
| 91 | + // Ensure properly formatted path |
| 92 | + if (!urlPath.startsWith('/')) urlPath = '/' + urlPath; |
| 93 | + |
| 94 | + // Avoid duplicates if markdown already covered it (unlikely for pages dir vs content dir overlap unless mapped) |
| 95 | + // But check if path already in llms (simple check) |
| 96 | + if (seenUrls.has(urlPath)) continue; |
| 97 | + |
| 98 | + llms += `- [${title}](${urlPath})\n`; |
| 99 | + } |
| 100 | + |
| 101 | + // Write files |
| 102 | + await fs.mkdir(OUTPUT_DIR, {recursive: true}); |
| 103 | + await fs.writeFile(path.join(OUTPUT_DIR, 'llms.txt'), llms); |
| 104 | + await fs.writeFile(path.join(OUTPUT_DIR, 'llms-full.txt'), llmsFull); |
| 105 | + |
| 106 | + console.log( |
| 107 | + `Successfully generated llms.txt and llms-full.txt in ${OUTPUT_DIR}` |
| 108 | + ); |
| 109 | +} |
| 110 | + |
| 111 | +generate().catch(console.error); |
0 commit comments