Skip to content

Commit 032dc9b

Browse files
JoviDeCroockJoviDeCroock
committed
Share frontmatter parsing
1 parent 1ae470a commit 032dc9b

3 files changed

Lines changed: 111 additions & 67 deletions

File tree

‎plugins/generate-llms-txt.js‎

Lines changed: 7 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -2,36 +2,11 @@ import fs from 'fs/promises';
22
import fsSync from 'fs';
33
import path from 'path';
44
import { fileURLToPath } from 'url';
5+
import { parseFrontmatter, cleanReplComments } from '../src/lib/frontmatter.js';
56

67
const __filename = fileURLToPath(import.meta.url);
78
const __dirname = path.dirname(__filename);
89

9-
/**
10-
* Extract title and description from frontmatter
11-
* @param {string} content
12-
* @returns {{title: string, description: string, body: string}}
13-
*/
14-
function parseFrontmatter(content) {
15-
const frontmatterRegex = /^---\n([\s\S]*?)\n---\n([\s\S]*)$/;
16-
const match = content.match(frontmatterRegex);
17-
18-
if (!match) {
19-
return { title: '', description: '', body: content };
20-
}
21-
22-
const frontmatter = match[1];
23-
const body = match[2];
24-
25-
const titleMatch = frontmatter.match(/^title:\s*(.+)$/m);
26-
const descriptionMatch = frontmatter.match(/^description:\s*(.+)$/m);
27-
28-
return {
29-
title: titleMatch ? titleMatch[1].trim() : '',
30-
description: descriptionMatch ? descriptionMatch[1].trim() : '',
31-
body: body.trim()
32-
};
33-
}
34-
3510
/**
3611
* Read all markdown files from the guide directory
3712
* @param {string} guideDir
@@ -76,7 +51,11 @@ Preact is a fast, lightweight alternative to React that provides the same modern
7651
files.sort((a, b) => a.filename.localeCompare(b.filename));
7752

7853
files.forEach(({ filename, content: fileContent }) => {
79-
const { title, description, body } = parseFrontmatter(fileContent);
54+
const { title, description, body } = parseFrontmatter(
55+
fileContent,
56+
filename
57+
);
58+
const cleanedBody = cleanReplComments(body);
8059

8160
content += `## ${title || filename.replace('.md', '')}\n\n`;
8261

@@ -85,7 +64,7 @@ Preact is a fast, lightweight alternative to React that provides the same modern
8564
}
8665

8766
content += `**Source:** content/en/guide/v10/${filename}\n\n`;
88-
content += `${body}\n\n`;
67+
content += `${cleanedBody}\n\n`;
8968
content += `---\n\n`;
9069
});
9170

‎plugins/precompile-markdown/index.js‎

Lines changed: 31 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
import yaml from 'yaml';
21
import { marked } from 'marked';
32
import Prism from 'prismjs';
43
import loadLanguages from 'prismjs/components/';
54
import { parse } from 'node-html-parser';
65
import { replace } from './gh-emoji/index.js';
76
import { textToBase64 } from '../../src/components/controllers/repl/query-encode.js';
7+
import { parseFrontmatter } from '../../src/lib/frontmatter.js';
88

99
// Prism will always load `markup`, `css`, `clike` and `javascript` by default.
1010
// Any additional languages we need should be loaded here
@@ -38,41 +38,23 @@ export async function precompileMarkdown(content, path) {
3838
* @param {string} path
3939
*/
4040
function parseContent(content, path) {
41-
/** @type {import('../../src/types.d.ts').ContentMetaData} */
42-
let meta = {};
43-
44-
// Find YAML FrontMatter preceeding a markdown document
45-
const FRONT_MATTER_REG = /^\s*---\n\s*([\s\S]*?)\s*\n---\n/i;
46-
47-
const matches = content.match(FRONT_MATTER_REG);
48-
if (!matches) throw new Error(`Missing YAML FrontMatter in ${path}`);
49-
try {
50-
meta = yaml.parse('---\n' + matches[1].replace(/^/gm, ' ') + '\n');
51-
if (!meta.title) {
52-
throw new Error(`Missing title in YAML FrontMatter for ${path}`);
53-
}
54-
if (!meta.description) {
55-
//console.warn(`Missing description in FrontMatter for ${path}`);
56-
}
57-
} catch (e) {
58-
throw new Error(`Error parsing YAML FrontMatter in ${path}`);
59-
}
41+
const { body, meta } = parseFrontmatter(content, path, true);
6042

61-
content = content.replace(FRONT_MATTER_REG, '');
43+
let processedContent = body;
6244

6345
if (path.includes('/guide/')) {
64-
meta.toc = generateToc(content);
46+
meta.toc = generateToc(processedContent);
6547
}
6648

6749
// extract tutorial setup, initial and final code blocks
6850
if (/tutorial\/\d/.test(path)) {
69-
const { markdown, tutorial } = extractTutorialCodeBlocks(content);
70-
content = markdown;
51+
const { markdown, tutorial } = extractTutorialCodeBlocks(processedContent);
52+
processedContent = markdown;
7153
meta.tutorial = tutorial;
7254
}
7355

7456
return {
75-
content,
57+
content: processedContent,
7658
meta
7759
};
7860
}
@@ -121,10 +103,12 @@ marked.use({
121103

122104
Prism.languages[lang] == null
123105
? console.warn(`No Prism highlighter for language: ${lang}`)
124-
: text = Prism.highlight(code, Prism.languages[lang], lang);
106+
: (text = Prism.highlight(code, Prism.languages[lang], lang));
125107

126108
const runInReplLink = runInRepl
127-
? `<a class="repl-link" href="/repl?code=${encodeURIComponent(textToBase64(source))}">Run in REPL</a>`
109+
? `<a class="repl-link" href="/repl?code=${encodeURIComponent(
110+
textToBase64(source)
111+
)}">Run in REPL</a>`
128112
: '';
129113

130114
return `
@@ -227,7 +211,9 @@ function highlightCodeBlocks(data) {
227211
const doc = parse(data.html, { blockTextElements: { code: true } });
228212

229213
// Only get the pre blocks that haven't already been highlighted
230-
const codeBlocks = doc.querySelectorAll('pre:not([class="highlight"]):has(> code[class])');
214+
const codeBlocks = doc.querySelectorAll(
215+
'pre:not([class="highlight"]):has(> code[class])'
216+
);
231217
for (const block of codeBlocks) {
232218
const child = block.childNodes[0];
233219

@@ -243,24 +229,31 @@ function highlightCodeBlocks(data) {
243229
* and switch it back to `\n` for the code content after marked is through with it.
244230
* We only do this on the home/index page at the moment.
245231
*/
246-
const rawCodeBlockText = unescapeHTML(child.innerText.trim().replace('<br>', '\n'));
232+
const rawCodeBlockText = unescapeHTML(
233+
child.innerText.trim().replace('<br>', '\n')
234+
);
247235
const [code, source, runInRepl] = processRepl(rawCodeBlockText);
248236

249237
const lang = child.getAttribute('class').replace('language-', '');
250238

251239
Prism.languages[lang] == null
252240
? console.warn(`No Prism highlighter for language: ${lang}`)
253-
: child.innerHTML = Prism.highlight(code, Prism.languages[lang], lang);
241+
: (child.innerHTML = Prism.highlight(code, Prism.languages[lang], lang));
254242

255-
block.insertAdjacentHTML('beforebegin', '<div class="highlight-container">');
243+
block.insertAdjacentHTML(
244+
'beforebegin',
245+
'<div class="highlight-container">'
246+
);
256247
const container = block.previousSibling;
257248
container.appendChild(block);
258249
block.setAttribute('class', 'highlight');
259250

260251
if (runInRepl) {
261252
block.insertAdjacentHTML(
262253
'afterend',
263-
`<a class="repl-link" href="/repl?code=${encodeURIComponent(textToBase64(source))}">
254+
`<a class="repl-link" href="/repl?code=${encodeURIComponent(
255+
textToBase64(source)
256+
)}">
264257
Run in REPL
265258
</a>`
266259
);
@@ -271,7 +264,6 @@ function highlightCodeBlocks(data) {
271264
return data;
272265
}
273266

274-
275267
/**
276268
* Marked escapes HTML entities, which is normally great,
277269
* but we want to feed the raw code into Prism for highlighting.
@@ -280,12 +272,12 @@ function highlightCodeBlocks(data) {
280272
* @returns {string}
281273
*/
282274
function unescapeHTML(str) {
283-
return str
284-
.replace(/&amp;/g, '&')
285-
.replace(/&lt;/g, '<')
286-
.replace(/&gt;/g, '>')
287-
.replace(/&quot;/g, '"')
288-
.replace(/&#39;/g, "'");
275+
return str
276+
.replace(/&amp;/g, '&')
277+
.replace(/&lt;/g, '<')
278+
.replace(/&gt;/g, '>')
279+
.replace(/&quot;/g, '"')
280+
.replace(/&#39;/g, "'");
289281
}
290282

291283
/**

‎src/lib/frontmatter.js‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
import yaml from 'yaml';
2+
3+
/**
4+
* Parse Markdown with YAML FrontMatter
5+
* @param {string} content - Raw markdown content with frontmatter
6+
* @param {string} path - File path for error reporting
7+
* @param {boolean} strict - Whether to throw errors for missing required fields
8+
* @returns {{title: string, description: string, body: string, meta: object}}
9+
*/
10+
export function parseFrontmatter(content, path = '', strict = false) {
11+
const FRONT_MATTER_REG = /^\s*---\n\s*([\s\S]*?)\s*\n---\n/i;
12+
13+
const matches = content.match(FRONT_MATTER_REG);
14+
if (!matches) {
15+
if (strict) {
16+
throw new Error(`Missing YAML FrontMatter in ${path}`);
17+
}
18+
return {
19+
title: '',
20+
description: '',
21+
body: content.trim(),
22+
meta: {}
23+
};
24+
}
25+
26+
let meta = {};
27+
try {
28+
meta = yaml.parse('---\n' + matches[1].replace(/^/gm, ' ') + '\n');
29+
30+
if (strict) {
31+
if (!meta.title) {
32+
throw new Error(`Missing title in YAML FrontMatter for ${path}`);
33+
}
34+
if (!meta.description) {
35+
console.warn(`Missing description in FrontMatter for ${path}`);
36+
}
37+
}
38+
} catch (e) {
39+
if (strict) {
40+
throw new Error(
41+
`Error parsing YAML FrontMatter in ${path}: ${e.message}`
42+
);
43+
} else {
44+
console.warn(`Error parsing YAML FrontMatter in ${path}: ${e.message}`);
45+
meta = {};
46+
}
47+
}
48+
49+
const body = content.replace(FRONT_MATTER_REG, '').trim();
50+
51+
return {
52+
title: meta.title || '',
53+
description: meta.description || '',
54+
body,
55+
meta
56+
};
57+
}
58+
59+
/**
60+
* Process and clean code blocks for LLM consumption
61+
* Removes --repl comments and markers that are used for REPL functionality
62+
* @param {string} content - Markdown content
63+
* @returns {string} - Cleaned content
64+
*/
65+
export function cleanReplComments(content) {
66+
// Remove --repl comment lines and markers
67+
return content
68+
.replace(/^\/\/ --repl.*$/gm, '')
69+
.replace(/^\/\/ --repl-before.*$/gm, '')
70+
.replace(/^\/\/ --repl-after.*$/gm, '')
71+
.replace(/\n\n\n+/g, '\n\n') // Clean up extra newlines
72+
.trim();
73+
}

0 commit comments

Comments
 (0)