Skip to content

Commit 916f470

Browse files
decepulisdecepulis
authored
feat(site): split llms.txt into per-framework and blog sub-indexes (#697)
1 parent cc5b37e commit 916f470

2 files changed

Lines changed: 108 additions & 91 deletions

File tree

‎site/astro.config.mjs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,14 @@ export default defineConfig({
5757
}),
5858
mdx({ extendMarkdownConfig: true }),
5959
sitemap({
60-
customPages: [`${SITE_URL}/llms.txt`],
60+
// llms-markdown.ts auto-generates per-framework sub-indexes, but sitemap
61+
// entries are hardcoded here. Add a new line when adding a framework.
62+
customPages: [
63+
`${SITE_URL}/llms.txt`,
64+
`${SITE_URL}/blog/llms.txt`,
65+
`${SITE_URL}/docs/framework/html/llms.txt`,
66+
`${SITE_URL}/docs/framework/react/llms.txt`,
67+
],
6168
}),
6269
pagefind(),
6370
llmsMarkdown(),

‎site/integrations/llms-markdown.ts‎

Lines changed: 100 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,14 @@ import type { AstroIntegration } from 'astro';
55
import { JSDOM } from 'jsdom';
66
import TurndownService from 'turndown';
77

8+
interface PageEntry {
9+
pathname: string;
10+
title: string;
11+
description?: string;
12+
sort?: string;
13+
framework?: string;
14+
}
15+
816
export default function llmsMarkdown(): AstroIntegration {
917
return {
1018
name: 'llms-markdown',
@@ -18,9 +26,9 @@ export default function llmsMarkdown(): AstroIntegration {
1826
});
1927

2028
// Track all docs and blog pages for llms.txt index
21-
const docsPages: Array<{ pathname: string; title: string; description?: string; sort?: string }> = [];
22-
const blogPages: Array<{ pathname: string; title: string; description?: string; sort?: string }> = [];
23-
const otherPages: Array<{ pathname: string; title: string; description?: string; sort?: string }> = [];
29+
const docsPages: PageEntry[] = [];
30+
const blogPages: PageEntry[] = [];
31+
const otherPages: PageEntry[] = [];
2432

2533
logger.info('Generating LLM-optimized markdown files...');
2634

@@ -71,15 +79,18 @@ export default function llmsMarkdown(): AstroIntegration {
7179
const sortAttr = contentElements[0]?.getAttribute('data-llms-sort');
7280
const sort = sortAttr || undefined;
7381

82+
const frameworkAttr = contentElements[0]?.getAttribute('data-framework');
83+
const framework = frameworkAttr || undefined;
84+
7485
// Write markdown file as sibling to the directory
75-
// docs/framework/html/style/css/slug -> docs/framework/html/style/css/slug.md
86+
// docs/framework/html/how-to/slug -> docs/framework/html/how-to/slug.md
7687
const mdPath = join(siteDir, `${pathname}.md`);
7788
await mkdir(dirname(mdPath), { recursive: true });
7889
await writeFile(mdPath, markdown, 'utf-8');
7990

8091
// Track for llms.txt index (with leading slash for URLs)
8192
if (pathname.startsWith('docs/')) {
82-
docsPages.push({ pathname: `/${pathname}`, title, description, sort });
93+
docsPages.push({ pathname: `/${pathname}`, title, description, sort, framework });
8394
} else if (pathname.startsWith('blog/')) {
8495
blogPages.push({ pathname: `/${pathname}`, title, description, sort });
8596
} else {
@@ -90,117 +101,116 @@ export default function llmsMarkdown(): AstroIntegration {
90101
}
91102
}
92103

93-
// Generate llms.txt index file
94-
const llmsTxt = generateLlmsTxt(docsPages, blogPages, otherPages);
95-
const llmsTxtPath = join(siteDir, 'llms.txt');
96-
await writeFile(llmsTxtPath, llmsTxt, 'utf-8');
104+
// Group docs by framework
105+
const docsByFramework = new Map<string, PageEntry[]>();
106+
for (const doc of docsPages) {
107+
const fw = doc.framework ?? 'unknown';
108+
if (!docsByFramework.has(fw)) {
109+
docsByFramework.set(fw, []);
110+
}
111+
docsByFramework.get(fw)!.push(doc);
112+
}
113+
114+
// Write per-framework docs sub-indexes
115+
const frameworks: string[] = [];
116+
for (const [fw, fwPages] of docsByFramework) {
117+
frameworks.push(fw);
118+
const subIndex = generateDocsIndex(fw, fwPages);
119+
const subIndexPath = join(siteDir, 'docs', 'framework', fw, 'llms.txt');
120+
await mkdir(dirname(subIndexPath), { recursive: true });
121+
await writeFile(subIndexPath, subIndex, 'utf-8');
122+
}
123+
124+
// Write blog sub-index
125+
if (blogPages.length > 0) {
126+
const blogIndex = generateBlogIndex(blogPages);
127+
const blogIndexPath = join(siteDir, 'blog', 'llms.txt');
128+
await mkdir(dirname(blogIndexPath), { recursive: true });
129+
await writeFile(blogIndexPath, blogIndex, 'utf-8');
130+
}
131+
132+
// Write root llms.txt index
133+
const rootIndex = generateRootIndex(frameworks, blogPages.length > 0, otherPages);
134+
const rootIndexPath = join(siteDir, 'llms.txt');
135+
await writeFile(rootIndexPath, rootIndex, 'utf-8');
97136

137+
const subIndexCount = frameworks.length + (blogPages.length > 0 ? 1 : 0);
98138
logger.info(
99-
`Generated ${docsPages.length + blogPages.length + otherPages.length} markdown files and llms.txt index`
139+
`Generated ${docsPages.length + blogPages.length + otherPages.length} markdown files, llms.txt root index, and ${subIndexCount} sub-indexes`
100140
);
101141
},
102142
},
103143
};
104144
}
105145

106-
function generateLlmsTxt(
107-
docsPages: Array<{ pathname: string; title: string; description?: string; sort?: string }>,
108-
blogPages: Array<{ pathname: string; title: string; description?: string; sort?: string }>,
109-
otherPages: Array<{ pathname: string; title: string; description?: string; sort?: string }>
110-
): string {
111-
// Group docs by framework and style
112-
const docsByFrameworkStyle = new Map<
113-
string,
114-
Array<{ pathname: string; title: string; description?: string; sort?: string }>
115-
>();
116-
117-
for (const doc of docsPages) {
118-
// Extract framework and style from pathname
119-
// Pattern: /docs/framework/{framework}/style/{style}/{...slug}
120-
const match = doc.pathname.match(/^\/docs\/framework\/([^/]+)\/style\/([^/]+)\//);
121-
if (match) {
122-
const [, framework, style] = match;
123-
const key = `${framework}/${style}`;
124-
if (!docsByFrameworkStyle.has(key)) {
125-
docsByFrameworkStyle.set(key, []);
126-
}
127-
docsByFrameworkStyle.get(key)!.push(doc);
128-
}
129-
}
130-
131-
// Build llms.txt content
146+
function generateRootIndex(frameworks: string[], hasBlog: boolean, otherPages: PageEntry[]): string {
132147
let content = `# Video.js v10\n\n`;
133148
content += `> Modern video player framework with multi-platform support\n\n`;
134149

135-
// Add documentation sections grouped by framework/style
136-
if (docsByFrameworkStyle.size > 0) {
137-
content += `## Documentation\n\n`;
138-
139-
// Sort by framework/style for consistent output
140-
const sortedKeys = Array.from(docsByFrameworkStyle.keys()).sort();
141-
142-
for (const key of sortedKeys) {
143-
const [framework, style] = key.split('/');
144-
const frameworkLabel = framework.charAt(0).toUpperCase() + framework.slice(1);
145-
const styleLabel = style.toUpperCase();
146-
147-
content += `### ${frameworkLabel} + ${styleLabel}\n\n`;
150+
content += `## Documentation\n\n`;
151+
for (const fw of [...frameworks].sort()) {
152+
const label = fw.charAt(0).toUpperCase() + fw.slice(1);
153+
content += `- [${label} Docs](/docs/framework/${fw}/llms.txt)\n`;
154+
}
155+
content += `\n`;
148156

149-
const docs = docsByFrameworkStyle.get(key)!;
150-
// Sort docs by pathname for consistent output
151-
docs.sort((a, b) => a.pathname.localeCompare(b.pathname));
157+
if (hasBlog) {
158+
content += `## Blog\n\n`;
159+
content += `- [Blog Posts](/blog/llms.txt)\n\n`;
160+
}
152161

153-
for (const doc of docs) {
154-
if (doc.description) {
155-
content += `- [${doc.title}](${doc.pathname}): ${doc.description}\n`;
156-
} else {
157-
content += `- [${doc.title}](${doc.pathname})\n`;
158-
}
162+
if (otherPages.length > 0) {
163+
content += `## Other\n\n`;
164+
const sorted = [...otherPages].sort((a, b) => a.pathname.localeCompare(b.pathname));
165+
for (const page of sorted) {
166+
if (page.description) {
167+
content += `- [${page.title}](${page.pathname}.md): ${page.description}\n`;
168+
} else {
169+
content += `- [${page.title}](${page.pathname}.md)\n`;
159170
}
160-
content += `\n`;
161171
}
172+
content += `\n`;
162173
}
163174

164-
// Add blog posts section
165-
if (blogPages.length > 0) {
166-
content += `## Blog Posts\n\n`;
175+
return content;
176+
}
167177

168-
// Sort by date using data-llms-sort attribute in reverse order (newest first)
169-
const sortedBlogPages = [...blogPages].sort((a, b) => {
170-
// If both have sort attributes, compare them (reverse for newest first)
171-
if (a.sort && b.sort) {
172-
return b.sort.localeCompare(a.sort);
173-
}
174-
// Fallback to pathname comparison if sort is missing
175-
return b.pathname.localeCompare(a.pathname);
176-
});
178+
function generateDocsIndex(framework: string, pages: PageEntry[]): string {
179+
const label = framework.charAt(0).toUpperCase() + framework.slice(1);
180+
let content = `# Video.js v10 — ${label} Documentation\n\n`;
177181

178-
for (const post of sortedBlogPages) {
179-
if (post.description) {
180-
content += `- [${post.title}](${post.pathname}): ${post.description}\n`;
181-
} else {
182-
content += `- [${post.title}](${post.pathname})\n`;
183-
}
182+
const sorted = [...pages].sort((a, b) => a.pathname.localeCompare(b.pathname));
183+
for (const page of sorted) {
184+
if (page.description) {
185+
content += `- [${page.title}](${page.pathname}.md): ${page.description}\n`;
186+
} else {
187+
content += `- [${page.title}](${page.pathname}.md)\n`;
184188
}
185-
content += `\n`;
186189
}
190+
content += `\n`;
187191

188-
// Add other pages section
189-
if (otherPages.length > 0) {
190-
content += `## Other\n\n`;
192+
return content;
193+
}
191194

192-
// Sort by pathname
193-
const sortedOtherPages = [...otherPages].sort((a, b) => a.pathname.localeCompare(b.pathname));
195+
function generateBlogIndex(pages: PageEntry[]): string {
196+
let content = `# Video.js v10 — Blog\n\n`;
194197

195-
for (const page of sortedOtherPages) {
196-
if (page.description) {
197-
content += `- [${page.title}](${page.pathname}): ${page.description}\n`;
198-
} else {
199-
content += `- [${page.title}](${page.pathname})\n`;
200-
}
198+
// Newest first
199+
const sorted = [...pages].sort((a, b) => {
200+
if (a.sort && b.sort) {
201+
return b.sort.localeCompare(a.sort);
202+
}
203+
return b.pathname.localeCompare(a.pathname);
204+
});
205+
206+
for (const post of sorted) {
207+
if (post.description) {
208+
content += `- [${post.title}](${post.pathname}.md): ${post.description}\n`;
209+
} else {
210+
content += `- [${post.title}](${post.pathname}.md)\n`;
201211
}
202-
content += `\n`;
203212
}
213+
content += `\n`;
204214

205215
return content;
206216
}

0 commit comments

Comments
 (0)